Refactor job triggers

.github/workflows/bench-deploy.yml

@@ -1,13 +1,16 @@
-name: GPU benchmarks
+name: GPU benchmark on `master`
 on:
-  workflow_dispatch:
-  release:
-    types: [published]
+  push:
+    branches:
+      - master
 
 jobs:
+  # TODO: Account for different `justfile`, `bench.env`, and hardware
+  # One option is to upload them to gh-pages for qualitative comparison
+  # TODO: Fall back to a default if `justfile`/`bench.env` not present
   benchmark:
     name: Bench and deploy
-    runs-on: [self-hosted, gpu-bench-t4]
+    runs-on: [self-hosted, gpu-bench]
     steps:
       # Install deps
       - uses: actions/checkout@v4
@@ -43,12 +46,12 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./target/criterion
           destination_dir: benchmarks/criterion
-      - name: Move benchmark json to history
+      - name: Copy benchmark json to history
         run: mkdir history; cp ${{ github.sha }}.tar.gz history/
       - name: Deploy benchmark history
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: history/
           destination_dir: benchmarks/history
-          keep_files: true
+          keep_files: true

.github/workflows/bench-pr-comment.yml

@@ -43,11 +43,11 @@ jobs:
   # `gh pr checkout {{ github.event.issue.number}}` with `env: GH_TOKEN`
   gpu-benchmark:
     name: run fibonacci benchmark on GPU
-    runs-on: [self-hosted, gpu-bench-t4]
+    runs-on: [self-hosted, gpu-bench]
     if:
       github.event.issue.pull_request
       && github.event.issue.state == 'open'
-      && contains(github.event.comment.body, '!benchmark')
+      && contains(github.event.comment.body, '!gpu-benchmark')
       && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
     steps:
       # Set up GPU

.github/workflows/gpu-bench.yml

@@ -0,0 +1,70 @@
+# Run `fibonacci` GPU benchmark on a given commit
+name: GPU benchmark on commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      label:
+        description: "Git reference"
+        required: true
+        default: "master"
+
+env:
+  CARGO_TERM_COLOR: always
+  # Disable incremental compilation.
+  #
+  # Incremental compilation is useful as part of an edit-build-test-edit cycle,
+  # as it lets the compiler avoid recompiling code that hasn't changed. However,
+  # on CI, we're not making small edits; we're almost always building the entire
+  # project from scratch. Thus, incremental compilation on CI actually
+  # introduces *additional* overhead to support making future builds
+  # faster...but no future builds will ever occur in any given CI environment.
+  #
+  # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
+  # for details.
+  CARGO_INCREMENTAL: 0
+  # Allow more retries for network requests in cargo (downloading crates) and
+  # rustup (installing toolchains). This should help to reduce flaky CI failures
+  # from transient network timeouts or other issues.
+  CARGO_NET_RETRY: 10
+  RUSTUP_MAX_RETRIES: 10
+  # Don't emit giant backtraces in the CI logs.
+  RUST_BACKTRACE: short
+  RUSTFLAGS: -D warnings
+
+jobs:
+  # TODO: Fall back to a default if `justfile`/`bench.env` not present
+  gpu-benchmark:
+    name: Run fibonacci bench on GPU
+    runs-on: [self-hosted, gpu-bench]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: {{ inputs.label.description }}
+      # Set up GPU
+      # Check we have access to the machine's Nvidia drivers
+      - run: nvidia-smi
+      # Check that CUDA is installed with a driver-compatible version
+      # This must also be compatible with the GPU architecture, see above link
+      - run: nvcc --version
+      # Install dependencies
+      - uses: actions-rs/toolchain@v1
+      - uses: Swatinem/rust-cache@v2
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: just@1.15
+      # Run benchmark
+      - name: Install criterion
+        run: |
+          cargo install cargo-criterion
+          cargo install criterion-table
+      - name: Run GPU bench
+        run: just --dotenv-filename bench.env gpu-bench fibonacci
+      # TODO: Create `tables.toml` to format/describe the .md file
+      # Create a `criterion-table` and write to commit comment
+      - name: Run `criterion-table`
+        run: cat ${{ github.sha }}.json | criterion-table > BENCHMARKS.md
+      - name: Write comparative bench on commit comment
+        uses: peter-evans/commit-comment@v3
+        with:
+          body-path: BENCHMARKS.md

.github/workflows/gpu.yml → .github/workflows/gpu-ci.yml

@@ -2,9 +2,10 @@
 name: GPU tests
 
 on:
-  push:
-    branches:
-      - master
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    branches: [master]
+  merge_group:
 
 env:
   CARGO_TERM_COLOR: always
@@ -36,6 +37,7 @@ concurrency:
 jobs:
   cuda:
     name: Rust tests on CUDA
+    if: github.event_name != 'pull_request' || github.event.action == 'enqueued'
     runs-on: [self-hosted, gpu-ci]
     env:
       NVIDIA_VISIBLE_DEVICES: all
@@ -68,6 +70,7 @@ jobs:
 
   opencl:
     name: Rust tests on OpenCL
+    if: github.event_name != 'pull_request' || github.event.action == 'enqueued'
     runs-on: [self-hosted, gpu-ci]
     env:
       NVIDIA_VISIBLE_DEVICES: all

.github/workflows/merge-group.yml → .github/workflows/merge-tests.yml

@@ -6,8 +6,6 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
     branches: [master]
   merge_group:
-  # Manual trigger for early signal on local branches
-  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -67,15 +65,18 @@ jobs:
           cargo nextest run --profile ci --workspace --cargo-profile dev-no-assertions -E 'test(circuit::gadgets)'
 
   # TODO: Make this a required status check
-  # TODO: Cache successful bench run from PR branch on master, keyed on commit hash
-  # Run comparative benchmark against master
+  # Run comparative benchmark against master, reject on regression
   gpu-benchmark:
     # [TEMPORARY] Test one run before attempting merge
     #if: github.event_name != 'pull_request' || github.event.action == 'enqueued'
     name: Run fibonacci bench on GPU
-    runs-on: [self-hosted, gpu-bench-t4]
+    runs-on: [self-hosted, gpu-bench]
     steps:
-      # TODO: Factor this out into an action or into justfile, it's used in 4 places
+      # [TEMPORARY] Test access to Docker env vars
+      - run: |
+          echo ${{ env.CUSTOM_VAR }}
+          env | grep 'CUSTOM_VAR'
+      # TODO: Factor out GPU setup into an action or into justfile, it's used in 4 places
       # Set up GPU
       # Check we have access to the machine's Nvidia drivers
       - run: nvidia-smi
@@ -87,18 +88,16 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: master
-      - run: ls -a
       - name: Set base ref variable
         run: echo "BASE_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
       # Checkout the justfile and env of the source branch so the base can bench
       - run: git restore --source ${{ github.sha }} justfile bench.env
-      - run: ls -a
       # Install dependencies
       - uses: actions-rs/toolchain@v1
       - uses: Swatinem/rust-cache@v2
       - uses: taiki-e/install-action@v2
         with:
-          tool: just@1
+          tool: just@1.15
       # Run benchmark on base branch
       - name: Install criterion
         run: |
@@ -132,3 +131,4 @@ jobs:
         with:
           script: |
             core.setFailed('Fibonacci bench regression detected')
+

bench.env

@@ -6,4 +6,4 @@ LURK_BENCH_NOISE_THRESHOLD=0.10
 # CUDA config
 NVIDIA_VISIBLE_DEVICES=all
 NVIDIA_DRIVER_CAPABILITITES=compute,utility
-EC_GPU_FRAMEWORK=cuda
+EC_GPU_FRAMEWORK=cuda

justfile

@@ -1,5 +1,5 @@
 # Install with `cargo install just`
-# Usage: `just --dotenv-filename /path/to/file.env <bench|gpu-bench>`
+# Usage: `just --dotenv-filename /path/to/file.env <bench|gpu-bench> <args>`
 # TODO: Move dotenv-filename into justfile once the feature is available
 set dotenv-load