diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 6143d3c2fb3f..078c76e8e764 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -2378,7 +2378,15 @@ class AstBuilder extends DataTypeAstBuilder // exclude null values by default val filtered = if (ctx.nullOperator == null || ctx.nullOperator.EXCLUDE() != null) { - Filter(IsNotNull(Coalesce(valueColumnNames.map(UnresolvedAttribute(_)))), unpivot) + val valueColumns = valueColumnNames.map(UnresolvedAttribute(_)) + val condition = if (valueColumns.length == 1) { + // Keep the single-value plan stable; unary Coalesce does not require type coercion. + IsNotNull(Coalesce(valueColumns)) + } else { + // Multi-value columns can have unrelated types, so test each for null independently. + valueColumns.map(IsNotNull).reduceLeft(Or) + } + Filter(condition, unpivot) } else { unpivot } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/IdentifierClauseParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/IdentifierClauseParserSuite.scala index a08e2daf4d49..fd0f88cf32e1 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/IdentifierClauseParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/IdentifierClauseParserSuite.scala @@ -62,7 +62,7 @@ class IdentifierClauseParserSuite extends AnalysisTest { "col", Seq("v1", "v2"), table("unpivot_test")) - .where(coalesce($"v1", $"v2").isNotNull) + .where($"v1".isNotNull || $"v2".isNotNull) .select(star()) ) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/UnpivotParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/UnpivotParserSuite.scala index 3f59f8de9542..7aef18f0502d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/UnpivotParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/UnpivotParserSuite.scala @@ -81,7 +81,7 @@ class UnpivotParserSuite extends AnalysisTest { "col", Seq("val1", "val2"), table("t")) - .where(coalesce($"val1", $"val2").isNotNull) + .where($"val1".isNotNull || $"val2".isNotNull) .select(star()) ) } @@ -101,13 +101,23 @@ class UnpivotParserSuite extends AnalysisTest { "col", Seq("val1", "val2"), table("t")) - .where(coalesce($"val1", $"val2").isNotNull) + .where($"val1".isNotNull || $"val2".isNotNull) .select(star()) ) } } } + test("unpivot - multiple values with different types") { + assertAnalysisSuccess(parsePlan( + """SELECT * FROM VALUES + | (DATE '2026-01-01', 1, DATE '2026-01-02', 2) + | AS t(date1, int1, date2, int2) + |UNPIVOT EXCLUDE NULLS ( + | (date_value, int_value) FOR kind IN ((date1, int1), (date2, int2)) + |)""".stripMargin)) + } + test("unpivot - multiple values with inner alias") { Seq( "SELECT * FROM t UNPIVOT ((val1, val2) FOR col in ((a A, b), (c, d)))", diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/identifier-clause.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/identifier-clause.sql.out index 31df29577793..5e760c01b59f 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/identifier-clause.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/identifier-clause.sql.out @@ -2737,7 +2737,7 @@ SELECT * FROM unpivot_test UNPIVOT ((v1, v2) FOR col IN ((a, b) AS IDENTIFIER('c -- !query analysis Sort [id#x ASC NULLS FIRST, col#x ASC NULLS FIRST, v1#x ASC NULLS FIRST, v2#x ASC NULLS FIRST], true +- Project [id#x, col#x, v1#x, v2#x] - +- Filter isnotnull(coalesce(v1#x, v2#x)) + +- Filter (isnotnull(v1#x) OR isnotnull(v2#x)) +- Expand [[id#x, cols_ab, a#x, b#x], [id#x, cols_bc, b#x, c#x]], [id#x, col#x, v1#x, v2#x] +- SubqueryAlias spark_catalog.identifier_clause_test_schema.unpivot_test +- Relation spark_catalog.identifier_clause_test_schema.unpivot_test[id#x,a#x,b#x,c#x] csv diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/unpivot.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/unpivot.sql.out index 7f4d1a5b7d46..230cffec9a42 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/unpivot.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/unpivot.sql.out @@ -86,7 +86,7 @@ UNPIVOT ( ) -- !query analysis Project [course#x, year#x, earnings#x, sales#x] -+- Filter isnotnull(coalesce(earnings#x, sales#x)) ++- Filter (isnotnull(earnings#x) OR isnotnull(sales#x)) +- Expand [[course#x, earnings2012_sales2012, earnings2012#x, sales2012#x], [course#x, earnings2013_sales2013, earnings2013#x, sales2013#x], [course#x, earnings2014_sales2014, earnings2014#x, sales2014#x]], [course#x, year#x, earnings#x, sales#x] +- SubqueryAlias courseearningsandsales +- View (`courseEarningsAndSales`, [course#x, earnings2012#x, sales2012#x, earnings2013#x, sales2013#x, earnings2014#x, sales2014#x]) @@ -119,7 +119,7 @@ UNPIVOT ( ) -- !query analysis Project [course#x, year#x, earnings#x, sales#x] -+- Filter isnotnull(coalesce(earnings#x, sales#x)) ++- Filter (isnotnull(earnings#x) OR isnotnull(sales#x)) +- Expand [[course#x, 2012, earnings2012#x, sales2012#x], [course#x, 2013, earnings2013#x, sales2013#x], [course#x, 2014, earnings2014#x, sales2014#x]], [course#x, year#x, earnings#x, sales#x] +- SubqueryAlias courseearningsandsales +- View (`courseEarningsAndSales`, [course#x, earnings2012#x, sales2012#x, earnings2013#x, sales2013#x, earnings2014#x, sales2014#x]) @@ -127,3 +127,62 @@ Project [course#x, year#x, earnings#x, sales#x] +- Project [course#x, earnings2012#x, sales2012#x, earnings2013#x, sales2013#x, earnings2014#x, sales2014#x] +- SubqueryAlias courseEarningsAndSales +- LocalRelation [course#x, earnings2012#x, sales2012#x, earnings2013#x, sales2013#x, earnings2014#x, sales2014#x] + + +-- !query +create temporary view mixedTypeValues as select + 1 as id, + cast('2026-01-01' as date) as date1, cast(null as int) as int1, + cast(null as date) as date2, 2 as int2, + cast(null as date) as date3, cast(null as int) as int3 +-- !query analysis +CreateViewCommand `mixedTypeValues`, select + 1 as id, + cast('2026-01-01' as date) as date1, cast(null as int) as int1, + cast(null as date) as date2, 2 as int2, + cast(null as date) as date3, cast(null as int) as int3, false, false, LocalTempView, UNSUPPORTED, true + +- Project [1 AS id#x, cast(2026-01-01 as date) AS date1#x, cast(null as int) AS int1#x, cast(null as date) AS date2#x, 2 AS int2#x, cast(null as date) AS date3#x, cast(null as int) AS int3#x] + +- OneRowRelation + + +-- !query +SELECT * FROM mixedTypeValues +UNPIVOT EXCLUDE NULLS ( + (date_value, int_value) FOR kind IN ( + (date1, int1) as date_only, + (date2, int2) as int_only, + (date3, int3) as all_null + ) +) +ORDER BY kind +-- !query analysis +Sort [kind#x ASC NULLS FIRST], true ++- Project [id#x, kind#x, date_value#x, int_value#x] + +- Filter (isnotnull(date_value#x) OR isnotnull(int_value#x)) + +- Expand [[id#x, date_only, date1#x, int1#x], [id#x, int_only, date2#x, int2#x], [id#x, all_null, date3#x, int3#x]], [id#x, kind#x, date_value#x, int_value#x] + +- SubqueryAlias mixedtypevalues + +- View (`mixedTypeValues`, [id#x, date1#x, int1#x, date2#x, int2#x, date3#x, int3#x]) + +- Project [cast(id#x as int) AS id#x, cast(date1#x as date) AS date1#x, cast(int1#x as int) AS int1#x, cast(date2#x as date) AS date2#x, cast(int2#x as int) AS int2#x, cast(date3#x as date) AS date3#x, cast(int3#x as int) AS int3#x] + +- Project [1 AS id#x, cast(2026-01-01 as date) AS date1#x, cast(null as int) AS int1#x, cast(null as date) AS date2#x, 2 AS int2#x, cast(null as date) AS date3#x, cast(null as int) AS int3#x] + +- OneRowRelation + + +-- !query +SELECT * FROM mixedTypeValues +UNPIVOT INCLUDE NULLS ( + (date_value, int_value) FOR kind IN ( + (date1, int1) as date_only, + (date2, int2) as int_only, + (date3, int3) as all_null + ) +) +ORDER BY kind +-- !query analysis +Sort [kind#x ASC NULLS FIRST], true ++- Project [id#x, kind#x, date_value#x, int_value#x] + +- Expand [[id#x, date_only, date1#x, int1#x], [id#x, int_only, date2#x, int2#x], [id#x, all_null, date3#x, int3#x]], [id#x, kind#x, date_value#x, int_value#x] + +- SubqueryAlias mixedtypevalues + +- View (`mixedTypeValues`, [id#x, date1#x, int1#x, date2#x, int2#x, date3#x, int3#x]) + +- Project [cast(id#x as int) AS id#x, cast(date1#x as date) AS date1#x, cast(int1#x as int) AS int1#x, cast(date2#x as date) AS date2#x, cast(int2#x as int) AS int2#x, cast(date3#x as date) AS date3#x, cast(int3#x as int) AS int3#x] + +- Project [1 AS id#x, cast(2026-01-01 as date) AS date1#x, cast(null as int) AS int1#x, cast(null as date) AS date2#x, 2 AS int2#x, cast(null as date) AS date3#x, cast(null as int) AS int3#x] + +- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/unpivot.sql b/sql/core/src/test/resources/sql-tests/inputs/unpivot.sql index 08a46a64d165..43092080ec9f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/unpivot.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/unpivot.sql @@ -42,3 +42,32 @@ SELECT * FROM courseEarningsAndSales UNPIVOT ( (earnings, sales) FOR year IN ((earnings2012, sales2012) as `2012`, (earnings2013, sales2013) as `2013`, (earnings2014, sales2014) as `2014`) ); + + +create temporary view mixedTypeValues as select + 1 as id, + cast('2026-01-01' as date) as date1, cast(null as int) as int1, + cast(null as date) as date2, 2 as int2, + cast(null as date) as date3, cast(null as int) as int3; + +-- EXCLUDE NULLS keeps partially-null tuples and removes all-null tuples across different types +SELECT * FROM mixedTypeValues +UNPIVOT EXCLUDE NULLS ( + (date_value, int_value) FOR kind IN ( + (date1, int1) as date_only, + (date2, int2) as int_only, + (date3, int3) as all_null + ) +) +ORDER BY kind; + +-- INCLUDE NULLS retains the all-null tuple that EXCLUDE NULLS drops above +SELECT * FROM mixedTypeValues +UNPIVOT INCLUDE NULLS ( + (date_value, int_value) FOR kind IN ( + (date1, int1) as date_only, + (date2, int2) as int_only, + (date3, int3) as all_null + ) +) +ORDER BY kind; diff --git a/sql/core/src/test/resources/sql-tests/results/unpivot.sql.out b/sql/core/src/test/resources/sql-tests/results/unpivot.sql.out index fe2db8ca1170..185f747b4d4e 100644 --- a/sql/core/src/test/resources/sql-tests/results/unpivot.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/unpivot.sql.out @@ -111,3 +111,50 @@ Java 2013 30000 2 dotNET 2012 15000 NULL dotNET 2013 48000 1 dotNET 2014 22500 1 + + +-- !query +create temporary view mixedTypeValues as select + 1 as id, + cast('2026-01-01' as date) as date1, cast(null as int) as int1, + cast(null as date) as date2, 2 as int2, + cast(null as date) as date3, cast(null as int) as int3 +-- !query schema +struct<> +-- !query output + + + +-- !query +SELECT * FROM mixedTypeValues +UNPIVOT EXCLUDE NULLS ( + (date_value, int_value) FOR kind IN ( + (date1, int1) as date_only, + (date2, int2) as int_only, + (date3, int3) as all_null + ) +) +ORDER BY kind +-- !query schema +struct +-- !query output +1 date_only 2026-01-01 NULL +1 int_only NULL 2 + + +-- !query +SELECT * FROM mixedTypeValues +UNPIVOT INCLUDE NULLS ( + (date_value, int_value) FOR kind IN ( + (date1, int1) as date_only, + (date2, int2) as int_only, + (date3, int3) as all_null + ) +) +ORDER BY kind +-- !query schema +struct +-- !query output +1 all_null NULL NULL +1 date_only 2026-01-01 NULL +1 int_only NULL 2