-
Notifications
You must be signed in to change notification settings - Fork 317
258 lines (256 loc) · 10.2 KB
/
schedule-daily.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
name: Schedule Daily
on:
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
env:
BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/"
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
CI_COMMIT_SHA: ${{ github.sha }}
CI_JOB_NAME: ${{ github.job }}
CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
CI_PIPELINE_SOURCE: ${{ github.event_name }}
CI_PROJECT_DIR: ${{ github.workspace }}
CI_RUN_ID: ${{ github.run_id }}
BUILDEVENT_DATASET: "github-ci-dfinity"
jobs:
# bazel-test-bare-metal:
# name: Bazel Test Bare Metal
# <<: *dind-large-setup
# timeout-minutes: 120
# runs-on:
# group: zh1
# labels: dind-large
# steps:
# - <<: *checkout
# - <<: *before-script
# - <<: *docker-login
# - name: Run Bazel Launch Bare Metal
# shell: bash
# run: |
# echo "$ZH2_DLL01_CSV_SECRETS" > file1
# echo "$ZH2_FILE_SHARE_KEY" > file2 && chmod 400 file2
# # Run bare metal installation test
# # shellcheck disable=SC2046,SC2086
# bazel ${BAZEL_STARTUP_ARGS} run ${BAZEL_CI_CONFIG} \
# //ic-os/setupos/envs/dev:launch_bare_metal -- \
# --config_path "$(realpath ./ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml)" \
# --csv_filename "$(realpath file1)" \
# --file_share_ssh_key "$(realpath file2)" \
# --inject_image_pub_key "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK3gjE/2K5nxIBbk3ohgs8J5LW+XiObwA+kGtSaF5+4c" \
# --file_share_username ci_interim \
# --ci_mode
# # Run bare metal node performance benchmarks
# # shellcheck disable=SC2046,SC2086
# bazel ${BAZEL_STARTUP_ARGS} run ${BAZEL_CI_CONFIG} \
# //ic-os/setupos/envs/dev:launch_bare_metal -- \
# --config_path "$(realpath ./ic-os/dev-tools/bare_metal_deployment/zh2-dll01.yaml)" \
# --csv_filename "$(realpath file1)" \
# --file_share_ssh_key "$(realpath file2)" \
# --inject_image_pub_key "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK3gjE/2K5nxIBbk3ohgs8J5LW+XiObwA+kGtSaF5+4c" \
# --file_share_username ci_interim \
# --ci_mode \
# --benchmark
# bazel clean
# env:
# BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/"
# BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
# ZH2_DLL01_CSV_SECRETS: "${{ secrets.ZH2_DLL01_CSV_SECRETS }}"
# ZH2_FILE_SHARE_KEY: "${{ secrets.ZH2_FILE_SHARE_KEY }}"
fi-tests-nightly:
name: Bazel Test FI Nightly
runs-on:
group: zh1
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:2c6fc0aa92ada647e42790cbdac3199b27a1407d9e90ff6e5a97a69acac24041
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
timeout-minutes: 720 # 12 hours
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Before script
id: before-script
shell: bash
run: |
[ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
- name: Login to Dockerhub
shell: bash
run: ./ci/scripts/docker-login.sh
env:
DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
- name: Run FI Tests Nightly
id: bazel-test-all
uses: ./.github/actions/bazel-test-all/
with:
BAZEL_COMMAND: "test"
BAZEL_TARGETS: "//rs/ledger_suite/..."
BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=fi_tests_nightly --test_env=SSH_AUTH_SOCK --test_timeout=43200"
BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
- name: Upload bazel-bep
# runs only if previous step succeeded or failed;
# we avoid collecting artifacts of jobs that were cancelled
if: success() || failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-bep
retention-days: 14
if-no-files-found: ignore
compression-level: 9
path: |
bazel-bep.pb
profile.json
nns-tests-nightly:
name: Bazel Test NNS Nightly
runs-on:
group: zh1
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:2c6fc0aa92ada647e42790cbdac3199b27a1407d9e90ff6e5a97a69acac24041
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
timeout-minutes: 20
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Before script
id: before-script
shell: bash
run: |
[ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
- name: Login to Dockerhub
shell: bash
run: ./ci/scripts/docker-login.sh
env:
DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
- name: Run NNS Tests Nightly
id: bazel-test-all
uses: ./.github/actions/bazel-test-all/
with:
BAZEL_COMMAND: "test"
BAZEL_TARGETS: "//rs/nns/..."
BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
BAZEL_EXTRA_ARGS: "--keep_going --test_tag_filters=nns_tests_nightly --test_env=SSH_AUTH_SOCK --test_env=NNS_CANISTER_UPGRADE_SEQUENCE=all"
BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
- name: Upload bazel-bep
# runs only if previous step succeeded or failed;
# we avoid collecting artifacts of jobs that were cancelled
if: success() || failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-bep
retention-days: 14
if-no-files-found: ignore
compression-level: 9
path: |
bazel-bep.pb
profile.json
system-tests-benchmarks-nightly:
name: Bazel System Test Benchmarks
runs-on:
group: zh1
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:2c6fc0aa92ada647e42790cbdac3199b27a1407d9e90ff6e5a97a69acac24041
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
timeout-minutes: 480
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Before script
id: before-script
shell: bash
run: |
[ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
- name: Login to Dockerhub
shell: bash
run: ./ci/scripts/docker-login.sh
env:
DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
- name: Set Benchmark Targets
shell: bash
run: |
set -xeuo pipefail
echo "BENCHMARK_TARGETS=$(bazel query 'attr(tags, system_test_benchmark, //rs/...)' | tr '\n' ' ')" >> $GITHUB_ENV
- name: Test System Test Benchmarks
id: bazel-system-test-benchmarks
uses: ./.github/actions/bazel-test-all/
with:
BAZEL_COMMAND: "test"
BAZEL_TARGETS: ${{ env.BENCHMARK_TARGETS }}
BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
# note: there's just one performance cluster, so the job can't be parallelized
BAZEL_EXTRA_ARGS: "--test_tag_filters=system_test_benchmark --//bazel:enable_upload_perf_systest_results=True --keep_going --jobs 1"
BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
- name: Upload bazel-bep
# runs only if previous step succeeded or failed;
# we avoid collecting artifacts of jobs that were cancelled
if: success() || failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-bep
retention-days: 14
if-no-files-found: ignore
compression-level: 9
path: |
bazel-bep.pb
profile.json
dependency-scan-nightly:
name: Dependency Scan Nightly
runs-on:
group: zh1
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:2c6fc0aa92ada647e42790cbdac3199b27a1407d9e90ff6e5a97a69acac24041
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp
timeout-minutes: 60
permissions:
actions: write
env:
SHELL_WRAPPER: "/usr/bin/time"
CARGO_WASMPACK_VERSION: "0.12.1"
DEFAULT_NODE_VERSION: "20"
CI_PROJECT_PATH: ${{ github.repository }}
CI_PIPELINE_ID: ${{ github.run_id }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
SLACK_PSEC_BOT_OAUTH_TOKEN: ${{ secrets.SLACK_PSEC_BOT_OAUTH_TOKEN }}
GITHUB_REF: ${{ github.ref }}
REPO_NAME: ${{ github.repository }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Before script
id: before-script
shell: bash
run: |
[ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
- name: Setup environment deps
id: setup-environment-deps
shell: bash
run: |
pip3 install --ignore-installed -r requirements.txt
cargo install wasm-pack --version "${CARGO_WASMPACK_VERSION}"
source "${NVM_DIR}/nvm.sh"
nvm use ${DEFAULT_NODE_VERSION}
node --version
npm --version
- name: Run Dependency Scan Nightly
id: dependency-scan-nightly
shell: bash
run: |
set -euo pipefail
export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies
cd ci/src/dependencies/
$SHELL_WRAPPER python3 job/bazel_rust_ic_scanner_periodic_job.py
$SHELL_WRAPPER python3 job/npm_scanner_periodic_job.py
$SHELL_WRAPPER python3 job/bazel_trivy_container_ic_scanner_periodic_job.py