Add CUDA tests to release pipeline.

These run in compatibility-checking mode, meaning that they run tests which
are expected to fail and ensure that these tests still fail. This takes
even longer than when not checking this, so these run only as part of the
release pipeline.

PiperOrigin-RevId: 692355383

.buildkite/pipeline.yaml

@@ -205,6 +205,18 @@ steps:
       - make cuda-tests
     agents:
       queue: gpu
+  - label: ":fish: CUDA tests (NOSUBMIT)"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around 30 minutes to run.
+    parallelism: 32
+    timeout_in_minutes: 120
+    retry:
+      <<: *retry_settings
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make cuda-tests ARGS="--cuda_verify_compatibility=true"
+    agents:
+      queue: gpu
   - <<: *common
     <<: *source_test_continuous
     label: ":screwdriver: All GPU Drivers Test"

.buildkite/release.yaml

@@ -67,6 +67,19 @@ steps:
       - make gpu-all-tests
     agents:
       queue: gpu
+  - <<: *source_test_continuous
+    label: ":fish: CUDA tests"
+    # This is its own test rather than being part of the GPU tests,
+    # because it takes around 30 minutes to run.
+    parallelism: 32
+    timeout_in_minutes: 120
+    retry:
+      <<: *retry_settings
+    commands:
+      - make sudo TARGETS=//tools/gpu:main ARGS="install --latest" || cat /var/log/nvidia-installer.log
+      - make cuda-tests ARGS="--cuda_verify_compatibility=true"
+    agents:
+      queue: gpu
   - <<: *common
     label: ":screwdriver: All GPU Drivers Test"
     commands: