Enable useSparkShuffleFetch in Github CI.

Signed-off-by: Pascal Spörri <psp@zurich.ibm.com>
IBM · Sep 13, 2023 · cf5840f · cf5840f
1 parent aaa5968
commit cf5840f
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 2 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -62,11 +62,18 @@ jobs:
  distribution: temurin
  java-version: 11
  cache: sbt
- - name: Test
+ - name: Test Default Shuffle Fetch
  shell: bash
  if: startsWith(matrix.scala, '2.12.')
  run: |
  sbt test
+ - name: Test Spark Shuffle Fetch
+ shell: bash
+ if: startsWith(matrix.scala, '2.12.') && (startsWith(matrix.spark, '3.3.') || startsWith(matrix.spark, '3.3.'))
+ env:
+ USE_SPARK_SHUFFLE_FETCH: "true"
+ run: |
+ sbt test
  - name: Package
  shell: bash
  run: |

diff --git a/src/test/scala-2.12/org/apache/spark/shuffle/S3ShuffleManagerTest.scala b/src/test/scala-2.12/org/apache/spark/shuffle/S3ShuffleManagerTest.scala
@@ -238,7 +238,7 @@ class S3ShuffleManagerTest {
  def newSparkConf(): SparkConf = new SparkConf()
  .setAppName("testApp")
  .setMaster(s"local[2]")
- .set("spark.shuffle.s3.useSparkShuffleFetch", "false")
+ .set("spark.shuffle.s3.useSparkShuffleFetch", scala.util.Properties.envOrElse("USE_SPARK_SHUFFLE_FETCH", "false"))
  .set("spark.ui.enabled", "false")
  .set("spark.driver.allowMultipleContexts", "true")
  .set("spark.app.id", "app-" + UUID.randomUUID())