ci: Add GPU benchmarks and configure with just script
#4
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run final tests only when attempting to merge, shown as skipped status checks beforehand | |
| name: Merge group tests | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: [master] | |
| merge_group: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| linux-ignored: | |
| if: github.event_name != 'pull_request' || github.event.action == 'enqueued' | |
| runs-on: buildjet-16vcpu-ubuntu-2204 | |
| env: | |
| RUSTFLAGS: -D warnings | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: actions-rs/toolchain@v1 | |
| - uses: taiki-e/install-action@nextest | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Linux Tests | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-ci --run-ignored ignored-only -E 'all() - test(groth16::tests::outer_prove_recursion) - test(test_make_fcomm_examples) - test(test_functional_commitments_demo) - test(test_chained_functional_commitments_demo)' | |
| linux-arm: | |
| if: github.event_name != 'pull_request' || github.event.action == 'enqueued' | |
| runs-on: buildjet-16vcpu-ubuntu-2204-arm | |
| env: | |
| RUSTFLAGS: -D warnings | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: actions-rs/toolchain@v1 | |
| - uses: taiki-e/install-action@nextest | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Linux Tests | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-ci | |
| - name: Linux Gadget Tests w/o debug assertions | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-no-assertions -E 'test(circuit::gadgets)' | |
| mac-m1: | |
| if: github.event_name != 'pull_request' || github.event.action == 'enqueued' | |
| runs-on: macos-latest-xlarge | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: actions-rs/toolchain@v1 | |
| - uses: taiki-e/install-action@nextest | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Linux Tests | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-ci | |
| - name: Linux Gadget Tests w/o debug assertions | |
| run: | | |
| cargo nextest run --profile ci --workspace --cargo-profile dev-no-assertions -E 'test(circuit::gadgets)' | |
| # TODO: Make this a required status check | |
| # Run comparative benchmark against master, reject on regression | |
| gpu-benchmark: | |
| # [TEMPORARY] Test one run before attempting merge | |
| #if: github.event_name != 'pull_request' || github.event.action == 'enqueued' | |
| name: Run fibonacci bench on GPU | |
| runs-on: [self-hosted, gpu-bench] | |
| steps: | |
| # [TEMPORARY] Test access to Docker env vars | |
| - run: | | |
| echo ${{ env.CUSTOM_VAR }} | |
| env | grep 'CUSTOM_VAR' | |
| # TODO: Factor out GPU setup into an action or into justfile, it's used in 4 places | |
| # Set up GPU | |
| # Check we have access to the machine's Nvidia drivers | |
| - run: nvidia-smi | |
| # Check that CUDA is installed with a driver-compatible version | |
| # This must also be compatible with the GPU architecture, see above link | |
| - run: nvcc --version | |
| - uses: actions/checkout@v4 | |
| # Checkout base branch for comparative bench | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: master | |
| - name: Set base ref variable | |
| run: echo "BASE_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
| # Checkout the justfile and env of the source branch so the base can bench | |
| - run: git restore --source ${{ github.sha }} justfile bench.env | |
| # Install dependencies | |
| - uses: actions-rs/toolchain@v1 | |
| - uses: Swatinem/rust-cache@v2 | |
| - uses: taiki-e/install-action@v2 | |
| with: | |
| tool: just@1.15 | |
| # Run benchmark on base branch | |
| - name: Install criterion | |
| run: | | |
| cargo install cargo-criterion | |
| cargo install criterion-table | |
| - name: Run GPU bench | |
| run: just --dotenv-filename bench.env gpu-bench fibonacci | |
| # Switch to triggering branch and run benchmark | |
| - run: rm justfile bench.env | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.sha }} | |
| - name: Run GPU bench on source branch | |
| run: just --dotenv-filename bench.env gpu-bench fibonacci | |
| # Create a comparative `criterion-table` and write in commit comment | |
| - name: Run `criterion-table` | |
| run: cat ${{ env.BASE_REF }}.json ${{ github.sha }}.json | criterion-table > BENCHMARKS.md | |
| - name: Write comparative bench on commit comment | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| body-path: BENCHMARKS.md | |
| # TODO: Use jq for JSON parsing if needed | |
| # Check for benchmark regression based on Criterion's configured noise threshold | |
| - name: Performance regression check | |
| id: check-regression | |
| run: | | |
| echo "regress_count=$(grep -c 'Regressed' ${{ github.sha }}.json)" >> $GITHUB_OUTPUT | |
| # Fail job if regression found | |
| - uses: actions/github-script@v6 | |
| if: ${{ steps.check-regression.outputs.regress_count }} > 0 | |
| with: | |
| script: | | |
| core.setFailed('Fibonacci bench regression detected') | |