Skip to content

Apache DataFusion SQL Query Engine

fix: support type coercion for MAP literals with NULL values in VALUES lists (#23521)

Prateek Ganigicf139585f0db
diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rs
index c7a73a7c6c..23ccf7f815 100644
--- a/datafusion/expr-common/src/type_coercion/binary.rs
+++ b/datafusion/expr-common/src/type_coercion/binary.rs
@@ -768,6 +768,7 @@ fn type_union_resolution_coercion(
}
_ => binary_numeric_coercion(lhs_type, rhs_type)
.or_else(|| list_coercion(lhs_type, rhs_type, type_union_resolution_coercion))
+ .or_else(|| map_coercion(lhs_type, rhs_type, type_union_resolution_coercion))
.or_else(|| temporal_coercion_nonstrict_timezone(lhs_type, rhs_type))
.or_else(|| string_coercion(lhs_type, rhs_type))
.or_else(|| null_coercion(lhs_type, rhs_type))
diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs
index 5871f24e7f..cfa3bbe189 100644
--- a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs
+++ b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs
@@ -908,6 +908,60 @@ fn test_type_union_coercion_prefers_finer_timestamp_unit() {
);
}
+/// Tests that `type_union_resolution` unifies Map types by recursing into the
+/// key/value types, so a Map whose value type is Null (e.g. `MAP {'k': NULL}`)
+/// unifies with a concretely-typed Map in a VALUES list.
+/// See <https://github.com/apache/datafusion/issues/23474>.
+#[test]
+fn test_type_union_resolution_map() {
+ fn map_type(value_type: DataType) -> DataType {
+ DataType::Map(
+ Arc::new(Field::new(
+ "entries",
+ DataType::Struct(Fields::from(vec![
+ Field::new("key", DataType::Utf8, false),
+ Field::new("value", value_type, true),
+ ])),
+ false,
+ )),
+ false,
+ )
+ }
+
+ // Null value type unifies with a concrete value type, in both orders
+ assert_eq!(
+ type_union_resolution(&[map_type(DataType::Int64), map_type(DataType::Null)]),
+ Some(map_type(DataType::Int64))
+ );
+ assert_eq!(
+ type_union_resolution(&[map_type(DataType::Null), map_type(DataType::Int64)]),
+ Some(map_type(DataType::Int64))
+ );
+
+ // Numeric value types widen following the scalar rules
+ assert_eq!(
+ type_union_resolution(&[
+ map_type(DataType::Int64),
+ map_type(DataType::Null),
+ map_type(DataType::Float64),
+ ]),
+ Some(map_type(DataType::Float64))
+ );
+
+ // Map cannot unify with a non-Map composite type
+ assert_eq!(
+ type_union_resolution(&[
+ map_type(DataType::Int64),
+ DataType::Struct(Fields::from(vec![Field::new(
+ "key",
+ DataType::Utf8,
+ false
+ )])),
+ ]),
+ None
+ );
+}
+
/// Tests that comparison operators coerce to numeric when comparing
/// numeric and string types.
#[test]
diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.slt
index f8cbb395cb..486a50f960 100644
--- a/datafusion/sqllogictest/test_files/map.slt
+++ b/datafusion/sqllogictest/test_files/map.slt
@@ -920,6 +920,10 @@ SELECT map([column1, column1 * 10], ['x','y']) FROM (VALUES (1), (2), (3)) t;
# tests for DISTINCT / GROUP BY / aggregation on map columns
# https://github.com/apache/datafusion/issues/15428
+# NOTE: the CAST(NULL AS BIGINT) in the VALUES list below predates the fix for
+# https://github.com/apache/datafusion/issues/23474 and is no longer required.
+# It is kept as-is to document the historical workaround; the un-cast form is
+# exercised in the "map NULL value coercion in VALUES" section further below.
statement ok
CREATE TABLE map_distinct_table AS VALUES
(MAP {'k1': 1, 'k2': 2}, 'a', 1),
@@ -1048,3 +1052,144 @@ PUT 25
statement ok
DROP TABLE map_data;
+
+# map NULL value coercion in VALUES
+# https://github.com/apache/datafusion/issues/23474
+# A bare NULL map value used to fail type unification across a VALUES list
+# ("Inconsistent data type across values list") and required an explicit
+# CAST(NULL AS <type>). The map value type now unifies with concrete value
+# types following the same rules as scalar VALUES coercion.
+
+# concrete-typed row first, NULL-valued row second (the issue reproducer)
+statement ok
+CREATE TABLE map_null_concrete_first AS VALUES
+ (MAP {'k1': 1, 'k2': 2}),
+ (MAP {'k1': NULL});
+
+# NULL must round-trip as NULL after coercion, not a default value
+query ? rowsort
+SELECT * FROM map_null_concrete_first;
+----
+{k1: 1, k2: 2}
+{k1: NULL}
+
+# the NULL value type is coerced to the concrete value type (Int64)
+query T
+SELECT arrow_typeof(column1) FROM map_null_concrete_first LIMIT 1;
+----
+Map("entries": non-null Struct("key": non-null Utf8, "value": Int64), unsorted)
+
+statement ok
+DROP TABLE map_null_concrete_first;
+
+# NULL-valued row first, concrete-typed row second (coercion is symmetric)
+statement ok
+CREATE TABLE map_null_first AS VALUES
+ (MAP {'k1': NULL}),
+ (MAP {'k1': 1, 'k2': 2});
+
+query ? rowsort
+SELECT * FROM map_null_first;
+----
+{k1: 1, k2: 2}
+{k1: NULL}
+
+statement ok
+DROP TABLE map_null_first;
+
+# every row has a NULL value: succeeds and the value type stays Null
+statement ok
+CREATE TABLE map_all_null_values AS VALUES
+ (MAP {'k': NULL}),
+ (MAP {'k': NULL});
+
+query ? rowsort
+SELECT * FROM map_all_null_values;
+----
+{k: NULL}
+{k: NULL}
+
+query T
+SELECT arrow_typeof(column1) FROM map_all_null_values LIMIT 1;
+----
+Map("entries": non-null Struct("key": non-null Utf8, "value": Null), unsorted)
+
+statement ok
+DROP TABLE map_all_null_values;
+
+# multiple keys where only one value is NULL
+query ? rowsort
+SELECT * FROM (VALUES
+ (MAP {'a': 1, 'b': NULL}),
+ (MAP {'a': 2, 'b': 3})) t(column1);
+----
+{a: 1, b: NULL}
+{a: 2, b: 3}
+
+# three rows with the NULL-valued row in the middle
+query ? rowsort
+SELECT * FROM (VALUES
+ (MAP {'k': 1}),
+ (MAP {'k': NULL}),
+ (MAP {'k': 2})) t(column1);
+----
+{k: 1}
+{k: 2}
+{k: NULL}
+
+# numeric widening across a NULL-valued row follows the scalar rule
+# (Int64 + Float64 -> Float64)
+statement ok
+CREATE TABLE map_null_widening AS VALUES
+ (MAP {'k': 1}),
+ (MAP {'k': NULL}),
+ (MAP {'k': 1.5});
+
+query ? rowsort
+SELECT * FROM map_null_widening;
+----
+{k: 1.0}
+{k: 1.5}
+{k: NULL}
+
+query T
+SELECT arrow_typeof(column1) FROM map_null_widening LIMIT 1;
+----
+Map("entries": non-null Struct("key": non-null Utf8, "value": Float64), unsorted)
+
+statement ok
+DROP TABLE map_null_widening;
+
+# incompatible concrete value types with a NULL-valued row in between still
+# error; Int64/Utf8 follows the scalar VALUES rule (coerce to the numeric
+# type, then fail to cast the non-numeric string)
+query error Cast error: Cannot cast string 'hello' to value of Int64 type
+SELECT * FROM (VALUES
+ (MAP {'k': 1}),
+ (MAP {'k': NULL}),
+ (MAP {'k': 'hello'})) t(column1);
+
+# NULL value type unification recurses into nested maps
+query ? rowsort
+SELECT * FROM (VALUES
+ (MAP {'outer': MAP {'inner': 1}}),
+ (MAP {'outer': MAP {'inner': NULL}})) t(column1);
+----
+{outer: {inner: 1}}
+{outer: {inner: NULL}}
+
+# INSERT INTO ... VALUES also accepts a NULL map value without a cast
+statement ok
+CREATE TABLE map_null_insert AS VALUES (MAP {'k1': 1, 'k2': 2});
+
+statement ok
+INSERT INTO map_null_insert VALUES (MAP {'k1': NULL});
+
+query ? rowsort
+SELECT * FROM map_null_insert;
+----
+{k1: 1, k2: 2}
+{k1: NULL}
+
+statement ok
+DROP TABLE map_null_insert;