-
Notifications
You must be signed in to change notification settings - Fork 1
241 lines (230 loc) · 7.42 KB
/
cd.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
name: CD
on:
release:
types: [published]
jobs:
info:
name: Prepare release
runs-on: ubuntu-latest
if: github.event_name == 'release'
timeout-minutes: 5
outputs:
version: ${{ steps.version.outputs.version }}
steps:
- name: Checkout commit
uses: actions/checkout@v3
- name: Setup Python 3.9
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Cache PyPI
uses: actions/cache@v3
with:
key: pip-lint-${{ hashFiles('requirements/*.txt') }}
path: ~/.cache/pip
restore-keys: |
pip-lint-
- name: Install dependencies
run: |
python -m pip install pip
make setup
- name: Save the package version
id: version
run: |
echo "version=$(python setup.py --version)" >> $GITHUB_OUTPUT
- name: Show version
run: |
echo ${{ steps.version.outputs.version }}
pypi_deploy:
name: Release package
runs-on: ubuntu-latest
needs: [info]
if: github.event_name == 'release'
steps:
- name: Checkout commit
uses: actions/checkout@v3
- name: Sanity check for tag and version
run: |
export VERSION=${{ needs.info.outputs.version }}
if [ "${{ github.ref }}" != "refs/tags/v$VERSION" ]
then
echo "Git tag '${{ github.ref }}' differs from hard-coded package version 'v$VERSION'"
exit 1
else
echo "OK, git tag matches hard-coded package version: 'v$VERSION'"
fi
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Install dependencies
run: |
python -m pip install twine wheel
make setup
- name: Make dists
run: |
python setup.py sdist bdist_wheel
- name: PyPI upload
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
run: |
twine upload dist/*
dockerhub_deploy:
name: Release image if needed
runs-on: ubuntu-latest
needs: [info, pypi_deploy]
if: github.event_name == 'release'
steps:
- name: Checkout commit
uses: actions/checkout@v3
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Install dependencies
run: |
python -m pip install pip
make setup
- name: Wait until package becomes available on PyPI
timeout-minutes: 5
run: |
PYPIVERSION=$(python setup.py --version)
PYPIPACKAGE=neuro-extras==$PYPIVERSION
until python -m pip install $PYPIPACKAGE
do
echo "Waiting for the pypi package $PYPIPACKAGE ..."
sleep 1
done
- name: Build release image
run: |
export PACKAGE="neuro-extras==${{ needs.info.outputs.version }}"
docker build -t ghcr.io/neuro-inc/neuro-extras:latest \
--build-arg NEURO_EXTRAS_PACKAGE=$PACKAGE .
- name: Login to ghcr.io
uses: docker/login-action@v2.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Push release release image
id: push
run: |
export IMAGE=ghcr.io/neuro-inc/neuro-extras
export TAG=${{ needs.info.outputs.version }}
echo "image=${IMAGE}" >> $GITHUB_OUTPUT
echo "tag=${TAG}" >> $GITHUB_OUTPUT
docker tag $IMAGE:latest $IMAGE:$TAG
docker push $IMAGE:$TAG
# Push latest if it's not an alpha release:
# not alpha: TAG="20.9.29"
# alpha: TAG="20.9.29a1"
if [[ ! "$TAG" =~ ^.*a[0-9]*$ ]]; then
docker push $IMAGE:latest
fi
- name: Wait until image becomes available on DockerHub
timeout-minutes: 10
run: |
export IMAGE=${{ steps.push.outputs.image }}
export TAG=${{ steps.push.outputs.tag }}
until docker pull --quiet $IMAGE:$TAG
do
echo "Waiting for the dockerhub image $IMAGE:$TAG ..."
sleep 1
done
test:
name: Run tests against release image
needs: [dockerhub_deploy]
if: github.event_name == 'release'
strategy:
matrix:
python-version: ['3.7', '3.8', '3.9']
os: [ubuntu, macos, windows]
exclude:
# not to overload the platform, we will remove this eventually
# see https://github.com/neuro-inc/neuro-extras/pull/249
- python-version: 3.8
os: macos
- python-version: 3.8
os: windows
fail-fast: false
runs-on: ${{ matrix.os }}-latest
timeout-minutes: 30
env:
PYTHONIOENCODING: utf-8
NEURO_STAGING_URL: ${{ secrets.NEURO_STAGING_URL }}
NEURO_TOKEN: ${{ secrets.NEURO_TOKEN }}
NEURO_CLUSTER: default
#NEURO_CLUSTER_SECONDARY: cato-poc # TODO: uncomment when cato-poc cluster is fixed
AZURE_SAS_TOKEN: ${{ secrets.AZURE_SAS_TOKEN }}
steps:
- name: Checkout commit
uses: actions/checkout@v3
- name: Install python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache dir
id: pip-cache
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: Cache PyPI
uses: actions/cache@v3
with:
key: pip-ci-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }}
path: ${{ steps.pip-cache.outputs.dir }}
restore-keys: |
pip-ci-${{ runner.os }}-${{ matrix.python-version }}-
- name: Install libraries on Linux
if: matrix.os == 'ubuntu'
shell: bash
run: |
curl https://rclone.org/install.sh | sudo bash
- name: Install libraries on macOS
if: matrix.os == 'macos'
shell: bash
run: |
curl https://rclone.org/install.sh | sudo bash
- name: Install python dependencies
run: |
make setup
- name: Configure neuro
run: |
neuro config login-with-token ${{ env.NEURO_TOKEN }} ${{ env.NEURO_STAGING_URL }}
neuro config switch-cluster ${{ env.NEURO_CLUSTER }}
neuro --color=no config show
- name: Authorize GCP
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.E2E_COOKIECUTTER_GCP_SA_KEY }}
- name: Setup gcloud
uses: google-github-actions/setup-gcloud@v1
with:
export_default_credentials: true
- name: Configure AWS access
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.E2E_COOKIECUTTER_AWS_KEY_ID }}
aws-secret-access-key: ${{ secrets.E2E_COOKIECUTTER_AWS_ACCESS_KEY }}
aws-region: us-east-1
- name: Run tests on MacOS and Linux
if: matrix.os != 'windows'
env:
COLOR: 'yes'
DOCKER_CI_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
DOCKER_CI_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
run: |
unset NEURO_EXTRAS_IMAGE # clear this env var to test it in real
make test
- name: Run tests on Windows
if: matrix.os == 'windows'
shell: pwsh
env:
COLOR: 'yes'
DOCKER_CI_USERNAME: ${{ secrets.DOCKER_CI_USERNAME }}
DOCKER_CI_TOKEN: ${{ secrets.DOCKER_CI_TOKEN }}
run: |
if (Test-Path 'Env:NEURO_EXTRAS_IMAGE') { # clear this env var to test it in real
Remove-Item 'Env:NEURO_EXTRAS_IMAGE'
}
make test