From d10e76448326d9659f45db7f4d130fec7afab3a8 Mon Sep 17 00:00:00 2001 From: Kazuyuki Tanimura Date: Mon, 20 Apr 2026 18:22:00 -0700 Subject: [PATCH 1/3] test: run more Spark 4 tests --- .../comet/CometArrayExpressionSuite.scala | 2 -- .../apache/comet/CometExpressionSuite.scala | 6 ----- .../comet/CometJsonExpressionSuite.scala | 2 -- .../exec/CometColumnarShuffleSuite.scala | 24 ------------------- .../apache/comet/exec/CometExecSuite.scala | 4 ---- 5 files changed, 38 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index 322dc4c4ef..cb52e7e6c8 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -561,8 +561,6 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp } test("array_compact") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => withTempView("t1") { diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index c3d66340c4..9091b871c5 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -2271,8 +2271,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("to_json") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[StructsToJson]) -> "true") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable( @@ -2298,8 +2296,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("to_json escaping of field names and string values") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[StructsToJson]) -> "true") { val gen = new DataGenerator(new Random(42)) val chars = "\\'\"abc\t\r\n\f\b" @@ -2329,8 +2325,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("to_json unicode") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[StructsToJson]) -> "true") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable( diff --git a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala index 64c330dbdd..c2c1ce3663 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala @@ -30,7 +30,6 @@ import org.apache.spark.sql.catalyst.expressions.{JsonToStructs, StructsToJson} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions._ -import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.CometStructsToJson import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} @@ -48,7 +47,6 @@ class CometJsonExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe } test("to_json - all supported types") { - assume(!isSpark40Plus) withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") val filename = path.toString diff --git a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala index 7d865829b6..71104a3a80 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala @@ -344,78 +344,54 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar } test("columnar shuffle on map [bool]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(true, false)) } test("columnar shuffle on map [byte]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toByte, 1.toByte)) } test("columnar shuffle on map [short]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toShort, 1.toShort)) } test("columnar shuffle on map [int]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0, 1)) } test("columnar shuffle on map [long]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toLong, 1.toLong)) } test("columnar shuffle on map [float]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toFloat, 1.toFloat)) } test("columnar shuffle on map [double]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toDouble, 1.toDouble)) } test("columnar shuffle on map [date]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(new java.sql.Date(0.toLong), new java.sql.Date(1.toLong))) } test("columnar shuffle on map [timestamp]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest( 50, Seq(new java.sql.Timestamp(0.toLong), new java.sql.Timestamp(1.toLong))) } test("columnar shuffle on map [decimal]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest( 50, Seq(new java.math.BigDecimal(0.toLong), new java.math.BigDecimal(1.toLong))) } test("columnar shuffle on map [string]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toString, 1.toString)) } test("columnar shuffle on map [binary]") { - // https://github.com/apache/datafusion-comet/issues/1941 - assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toString.getBytes(), 1.toString.getBytes())) } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 22983119bb..0690cd1103 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -2000,8 +2000,6 @@ class CometExecSuite extends CometTestBase { } test("SparkToColumnar eliminate redundant in AQE") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { @@ -2090,8 +2088,6 @@ class CometExecSuite extends CometTestBase { } test("SparkToColumnar override node name for row input") { - // TODO fix for Spark 4.0.0 - assume(!isSpark40Plus) withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { From 000582188904af3167d724c7f6f023b20c2b61d4 Mon Sep 17 00:00:00 2001 From: Kazuyuki Tanimura Date: Tue, 21 Apr 2026 16:59:36 -0700 Subject: [PATCH 2/3] test: run more Spark 4 tests --- .../apache/comet/CometExpressionSuite.scala | 6 +++++ .../exec/CometColumnarShuffleSuite.scala | 24 +++++++++++++++++++ .../apache/comet/exec/CometExecSuite.scala | 4 ++++ 3 files changed, 34 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 9091b871c5..c3d66340c4 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -2271,6 +2271,8 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("to_json") { + // TODO fix for Spark 4.0.0 + assume(!isSpark40Plus) withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[StructsToJson]) -> "true") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable( @@ -2296,6 +2298,8 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("to_json escaping of field names and string values") { + // TODO fix for Spark 4.0.0 + assume(!isSpark40Plus) withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[StructsToJson]) -> "true") { val gen = new DataGenerator(new Random(42)) val chars = "\\'\"abc\t\r\n\f\b" @@ -2325,6 +2329,8 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("to_json unicode") { + // TODO fix for Spark 4.0.0 + assume(!isSpark40Plus) withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[StructsToJson]) -> "true") { Seq(true, false).foreach { dictionaryEnabled => withParquetTable( diff --git a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala index 71104a3a80..7d865829b6 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala @@ -344,54 +344,78 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar } test("columnar shuffle on map [bool]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(true, false)) } test("columnar shuffle on map [byte]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toByte, 1.toByte)) } test("columnar shuffle on map [short]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toShort, 1.toShort)) } test("columnar shuffle on map [int]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0, 1)) } test("columnar shuffle on map [long]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toLong, 1.toLong)) } test("columnar shuffle on map [float]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toFloat, 1.toFloat)) } test("columnar shuffle on map [double]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toDouble, 1.toDouble)) } test("columnar shuffle on map [date]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(new java.sql.Date(0.toLong), new java.sql.Date(1.toLong))) } test("columnar shuffle on map [timestamp]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest( 50, Seq(new java.sql.Timestamp(0.toLong), new java.sql.Timestamp(1.toLong))) } test("columnar shuffle on map [decimal]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest( 50, Seq(new java.math.BigDecimal(0.toLong), new java.math.BigDecimal(1.toLong))) } test("columnar shuffle on map [string]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toString, 1.toString)) } test("columnar shuffle on map [binary]") { + // https://github.com/apache/datafusion-comet/issues/1941 + assume(!isSpark40Plus) columnarShuffleOnMapTest(50, Seq(0.toString.getBytes(), 1.toString.getBytes())) } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 1bb2f4afa8..9790c87147 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -2388,6 +2388,8 @@ class CometExecSuite extends CometTestBase { } test("SparkToColumnar eliminate redundant in AQE") { + // TODO fix for Spark 4.0.0 + assume(!isSpark40Plus) withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { @@ -2476,6 +2478,8 @@ class CometExecSuite extends CometTestBase { } test("SparkToColumnar override node name for row input") { + // TODO fix for Spark 4.0.0 + assume(!isSpark40Plus) withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", CometConf.COMET_SHUFFLE_MODE.key -> "jvm") { From e5ee82e832b8571048d2335c4c21551fdb98a34f Mon Sep 17 00:00:00 2001 From: Kazuyuki Tanimura Date: Tue, 21 Apr 2026 19:38:46 -0700 Subject: [PATCH 3/3] test: run more Spark 4 tests --- .../test/scala/org/apache/comet/CometJsonExpressionSuite.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala index c2c1ce3663..64c330dbdd 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala @@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.{JsonToStructs, StructsToJson} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions._ +import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.CometStructsToJson import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} @@ -47,6 +48,7 @@ class CometJsonExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe } test("to_json - all supported types") { + assume(!isSpark40Plus) withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") val filename = path.toString