From 48df2530fbf7e3c7dc4c71cb71d1e86ee601195f Mon Sep 17 00:00:00 2001 From: Kazuyuki Tanimura Date: Wed, 22 Apr 2026 01:54:04 -0700 Subject: [PATCH 1/3] test: run more Spark 4 tests --- .../test/scala/org/apache/comet/CometExpressionSuite.scala | 6 ------ 1 file changed, 6 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index c3d66340c4..d028d4467e 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -2401,8 +2401,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("get_struct_field - select primitive fields") { - val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() - assume(!(scanImpl == CometConf.SCAN_AUTO && CometSparkSessionExtensions.isSpark40Plus)) withTempPath { dir => // create input file with Comet disabled withSQLConf(CometConf.COMET_ENABLED.key -> "false") { @@ -2420,8 +2418,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("get_struct_field - select subset of struct") { - val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() - assume(!(scanImpl == CometConf.SCAN_AUTO && CometSparkSessionExtensions.isSpark40Plus)) withTempPath { dir => // create input file with Comet disabled withSQLConf(CometConf.COMET_ENABLED.key -> "false") { @@ -2449,8 +2445,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } test("get_struct_field - read entire struct") { - val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get() - assume(!(scanImpl == CometConf.SCAN_AUTO && CometSparkSessionExtensions.isSpark40Plus)) withTempPath { dir => // create input file with Comet disabled withSQLConf(CometConf.COMET_ENABLED.key -> "false") { From d448f4c89168767bb6253176715091ff46f9ebdf Mon Sep 17 00:00:00 2001 From: Kazuyuki Tanimura Date: Wed, 22 Apr 2026 02:18:51 -0700 Subject: [PATCH 2/3] test: run more JVM tests --- .../org/apache/comet/CometExpressionSuite.scala | 3 +-- .../org/apache/comet/CometMapExpressionSuite.scala | 3 +-- .../org/apache/comet/CometMathExpressionSuite.scala | 12 ++++-------- .../org/apache/comet/exec/CometAggregateSuite.scala | 3 +-- .../apache/comet/exec/CometGenerateExecSuite.scala | 9 +++------ .../org/apache/comet/parquet/ParquetReadSuite.scala | 3 +-- .../org/apache/comet/rules/CometExecRuleSuite.scala | 4 +--- 7 files changed, 12 insertions(+), 25 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index d028d4467e..eef37517a0 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -283,8 +283,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - // TODO: https://github.com/apache/datafusion-comet/issues/2539 - ignore("date_add with scalar overflow") { + test("date_add with scalar overflow") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index 03db26e566..c392463864 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -140,8 +140,7 @@ class CometMapExpressionSuite extends CometTestBase { } } - // fails with "map is not supported" - ignore("size with map input") { + test("size with map input") { withTempDir { dir => withTempView("t1") { val path = new Path(dir.toURI.toString, "test.parquet") diff --git a/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala index 8698964ce6..9d27f2d25f 100644 --- a/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala @@ -92,8 +92,7 @@ class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe DataGenOptions(generateNegativeZero = generateNegativeZero)) } - // https://github.com/apache/datafusion-comet/issues/3561 - ignore("width_bucket") { + test("width_bucket") { assume(isSpark35Plus, "width_bucket was added in Spark 3.5") withSQLConf("spark.comet.exec.localTableScan.enabled" -> "true") { spark @@ -106,8 +105,7 @@ class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe } } - // https://github.com/apache/datafusion-comet/issues/3561 - ignore("width_bucket - edge cases") { + test("width_bucket - edge cases") { assume(isSpark35Plus, "width_bucket was added in Spark 3.5") withSQLConf("spark.comet.exec.localTableScan.enabled" -> "true") { spark @@ -124,8 +122,7 @@ class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe } } - // https://github.com/apache/datafusion-comet/issues/3561 - ignore("width_bucket - NaN values") { + test("width_bucket - NaN values") { assume(isSpark35Plus, "width_bucket was added in Spark 3.5") withSQLConf("spark.comet.exec.localTableScan.enabled" -> "true") { spark @@ -137,8 +134,7 @@ class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelpe } } - // https://github.com/apache/datafusion-comet/issues/3561 - ignore("width_bucket - with range data") { + test("width_bucket - with range data") { assume(isSpark35Plus, "width_bucket was added in Spark 3.5") withSQLConf("spark.comet.exec.localTableScan.enabled" -> "true") { spark diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index 95f3774e01..d4c54d00e4 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -1108,8 +1108,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - // TODO enable once https://github.com/apache/datafusion-comet/issues/1267 is implemented - ignore("distinct") { + test("distinct") { withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { Seq("native", "jvm").foreach { cometShuffleMode => withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> cometShuffleMode) { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala index a9ac3deb34..f117ff9f44 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala @@ -74,8 +74,7 @@ class CometGenerateExecSuite extends CometTestBase { } } - // https://github.com/apache/datafusion-comet/issues/2838 - ignore("explode_outer with empty array") { + test("explode_outer with empty array") { withSQLConf( CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_EXPLODE_ENABLED.key -> "true") { @@ -169,8 +168,7 @@ class CometGenerateExecSuite extends CometTestBase { } } - // https://github.com/apache/datafusion-comet/issues/2838 - ignore("explode_outer with nullable projected column") { + test("explode_outer with nullable projected column") { withSQLConf( CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_EXPLODE_ENABLED.key -> "true") { @@ -199,8 +197,7 @@ class CometGenerateExecSuite extends CometTestBase { } } - // https://github.com/apache/datafusion-comet/issues/2838 - ignore("explode_outer with mixed null, empty, and non-empty arrays") { + test("explode_outer with mixed null, empty, and non-empty arrays") { withSQLConf( CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_EXPLODE_ENABLED.key -> "true") { diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 75ac889228..47f833830b 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -712,8 +712,7 @@ abstract class ParquetReadSuite extends CometTestBase { } } - // TODO: This test fails for both native_datafusion and native_iceberg_compat - ignore(" Missing optional struct field with nested required field") { + test(" Missing optional struct field with nested required field") { Seq(true, false).foreach { dictionaryEnabled => def makeRawParquetFile(path: Path): Unit = { val schemaStr = diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index cf6f8918f4..84b3783d28 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -131,9 +131,7 @@ class CometExecRuleSuite extends CometTestBase { } } - // TODO this test exposes the bug described in - // https://github.com/apache/datafusion-comet/issues/1389 - ignore("CometExecRule should not allow Comet partial and Spark final hash aggregate") { + test("CometExecRule should not allow Comet partial and Spark final hash aggregate") { withTempView("test_data") { createTestDataFrame.createOrReplaceTempView("test_data") From 8be3a03769c3fec022930087ec3599166fa65c16 Mon Sep 17 00:00:00 2001 From: Kazuyuki Tanimura Date: Wed, 22 Apr 2026 02:42:16 -0700 Subject: [PATCH 3/3] test: run more JVM tests --- .../scala/org/apache/comet/CometExpressionSuite.scala | 3 ++- .../scala/org/apache/comet/CometMapExpressionSuite.scala | 3 ++- .../org/apache/comet/exec/CometAggregateSuite.scala | 3 ++- .../org/apache/comet/exec/CometGenerateExecSuite.scala | 9 ++++++--- .../org/apache/comet/parquet/ParquetReadSuite.scala | 3 ++- .../org/apache/comet/rules/CometExecRuleSuite.scala | 4 +++- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index eef37517a0..d028d4467e 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -283,7 +283,8 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("date_add with scalar overflow") { + // TODO: https://github.com/apache/datafusion-comet/issues/2539 + ignore("date_add with scalar overflow") { Seq(true, false).foreach { dictionaryEnabled => withTempDir { dir => val path = new Path(dir.toURI.toString, "test.parquet") diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index c392463864..03db26e566 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -140,7 +140,8 @@ class CometMapExpressionSuite extends CometTestBase { } } - test("size with map input") { + // fails with "map is not supported" + ignore("size with map input") { withTempDir { dir => withTempView("t1") { val path = new Path(dir.toURI.toString, "test.parquet") diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index d4c54d00e4..95f3774e01 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -1108,7 +1108,8 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("distinct") { + // TODO enable once https://github.com/apache/datafusion-comet/issues/1267 is implemented + ignore("distinct") { withSQLConf(CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true") { Seq("native", "jvm").foreach { cometShuffleMode => withSQLConf(CometConf.COMET_SHUFFLE_MODE.key -> cometShuffleMode) { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala index f117ff9f44..a9ac3deb34 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala @@ -74,7 +74,8 @@ class CometGenerateExecSuite extends CometTestBase { } } - test("explode_outer with empty array") { + // https://github.com/apache/datafusion-comet/issues/2838 + ignore("explode_outer with empty array") { withSQLConf( CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_EXPLODE_ENABLED.key -> "true") { @@ -168,7 +169,8 @@ class CometGenerateExecSuite extends CometTestBase { } } - test("explode_outer with nullable projected column") { + // https://github.com/apache/datafusion-comet/issues/2838 + ignore("explode_outer with nullable projected column") { withSQLConf( CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_EXPLODE_ENABLED.key -> "true") { @@ -197,7 +199,8 @@ class CometGenerateExecSuite extends CometTestBase { } } - test("explode_outer with mixed null, empty, and non-empty arrays") { + // https://github.com/apache/datafusion-comet/issues/2838 + ignore("explode_outer with mixed null, empty, and non-empty arrays") { withSQLConf( CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true", CometConf.COMET_EXEC_EXPLODE_ENABLED.key -> "true") { diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 47f833830b..75ac889228 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -712,7 +712,8 @@ abstract class ParquetReadSuite extends CometTestBase { } } - test(" Missing optional struct field with nested required field") { + // TODO: This test fails for both native_datafusion and native_iceberg_compat + ignore(" Missing optional struct field with nested required field") { Seq(true, false).foreach { dictionaryEnabled => def makeRawParquetFile(path: Path): Unit = { val schemaStr = diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index 84b3783d28..cf6f8918f4 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -131,7 +131,9 @@ class CometExecRuleSuite extends CometTestBase { } } - test("CometExecRule should not allow Comet partial and Spark final hash aggregate") { + // TODO this test exposes the bug described in + // https://github.com/apache/datafusion-comet/issues/1389 + ignore("CometExecRule should not allow Comet partial and Spark final hash aggregate") { withTempView("test_data") { createTestDataFrame.createOrReplaceTempView("test_data")