From cf5840f1c308507bfc23674d708178034e0f784d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pascal=20Spo=CC=88rri?= Date: Wed, 13 Sep 2023 11:15:30 +0200 Subject: [PATCH] Enable useSparkShuffleFetch in Github CI. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Pascal SpoĢˆrri --- .github/workflows/ci.yml | 9 ++++++++- .../org/apache/spark/shuffle/S3ShuffleManagerTest.scala | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d5f2e5..60d2192 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,11 +62,18 @@ jobs: distribution: temurin java-version: 11 cache: sbt - - name: Test + - name: Test Default Shuffle Fetch shell: bash if: startsWith(matrix.scala, '2.12.') run: | sbt test + - name: Test Spark Shuffle Fetch + shell: bash + if: startsWith(matrix.scala, '2.12.') && (startsWith(matrix.spark, '3.3.') || startsWith(matrix.spark, '3.3.')) + env: + USE_SPARK_SHUFFLE_FETCH: "true" + run: | + sbt test - name: Package shell: bash run: | diff --git a/src/test/scala-2.12/org/apache/spark/shuffle/S3ShuffleManagerTest.scala b/src/test/scala-2.12/org/apache/spark/shuffle/S3ShuffleManagerTest.scala index 882a646..d5b7424 100644 --- a/src/test/scala-2.12/org/apache/spark/shuffle/S3ShuffleManagerTest.scala +++ b/src/test/scala-2.12/org/apache/spark/shuffle/S3ShuffleManagerTest.scala @@ -238,7 +238,7 @@ class S3ShuffleManagerTest { def newSparkConf(): SparkConf = new SparkConf() .setAppName("testApp") .setMaster(s"local[2]") - .set("spark.shuffle.s3.useSparkShuffleFetch", "false") + .set("spark.shuffle.s3.useSparkShuffleFetch", scala.util.Properties.envOrElse("USE_SPARK_SHUFFLE_FETCH", "false")) .set("spark.ui.enabled", "false") .set("spark.driver.allowMultipleContexts", "true") .set("spark.app.id", "app-" + UUID.randomUUID())