Apache DataFusion SQL Query Engine
fix: align dictionary coercion across typed signatures (#23549)
linfeng5063883381a0
diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rsindex 65a45c0780..37a1c10159 100644--- a/datafusion/expr/src/type_coercion/functions.rs+++ b/datafusion/expr/src/type_coercion/functions.rs@@ -33,7 +33,9 @@ use datafusion_common::utils::{use datafusion_common::{Result, exec_err, internal_err, plan_err, types::NativeType, utils::list_ndims,};-use datafusion_expr_common::signature::{ArrayFunctionArgument, EncodingPreservation};+use datafusion_expr_common::signature::{+ ArrayFunctionArgument, EncodingPreservation, TypeSignatureClass,+};use datafusion_expr_common::type_coercion::binary::type_union_resolution;use datafusion_expr_common::{signature::{ArrayFunctionSignature, FIXED_SIZE_LIST_WILDCARD, TIMEZONE_WILDCARD},@@ -873,31 +875,47 @@ fn get_valid_types(TypeSignature::Coercible(param_types) => {function_length_check(function_name, current_types.len(), param_types.len())?;- fn cast_origin(- current_type: &DataType,- encoding_preservation: EncodingPreservation,- ) -> &DataType {- if encoding_preservation.preserve_dictionary()- && let DataType::Dictionary(_, value_type) = current_type- {- value_type- } else {- current_type+ fn coercion_value_type<'a>(+ current_type: &'a DataType,+ desired_type: &TypeSignatureClass,+ ) -> &'a DataType {+ if matches!(desired_type, TypeSignatureClass::Any) {+ return current_type;+ }++ match current_type {+ DataType::Dictionary(_, value_type) => {+ coercion_value_type(value_type, desired_type)+ }+ _ => current_type,}}fn preserve_encoding(current_type: &DataType,casted_type: DataType,+ desired_type: &TypeSignatureClass,encoding_preservation: EncodingPreservation,) -> DataType {- if encoding_preservation.preserve_dictionary()- && let DataType::Dictionary(key_type, _) = current_type- && !matches!(casted_type, DataType::Dictionary(_, _))- {- DataType::Dictionary(key_type.clone(), Box::new(casted_type))- } else {- casted_type+ if matches!(desired_type, TypeSignatureClass::Any) {+ return casted_type;+ }++ match current_type {+ DataType::Dictionary(key_type, value_type) => {+ let casted_type = preserve_encoding(+ value_type,+ casted_type,+ desired_type,+ encoding_preservation,+ );+ if encoding_preservation.preserve_dictionary() {+ DataType::Dictionary(key_type.clone(), Box::new(casted_type))+ } else {+ casted_type+ }+ }+ _ => casted_type,}}@@ -905,7 +923,8 @@ fn get_valid_types(for (current_type, param) in current_types.iter().zip(param_types.iter()) {let current_native_type: NativeType = current_type.into();let encoding_preservation = param.encoding_preservation();- let cast_origin = cast_origin(current_type, encoding_preservation);+ let coercion_value_type =+ coercion_value_type(current_type, param.desired_type());if param.desired_type()@@ -913,11 +932,12 @@ fn get_valid_types({let casted_type = param.desired_type()- .default_casted_type(¤t_native_type, cast_origin)?;+ .default_casted_type(¤t_native_type, coercion_value_type)?;new_types.push(preserve_encoding(current_type,casted_type,+ param.desired_type(),encoding_preservation,));} else if param@@ -928,10 +948,11 @@ fn get_valid_types(// If the condition is met which means `implicit coercion`` is provided so we can safely unwraplet default_casted_type = param.default_casted_type().unwrap();let casted_type =- default_casted_type.default_cast_for(cast_origin)?;+ default_casted_type.default_cast_for(coercion_value_type)?;new_types.push(preserve_encoding(current_type,casted_type,+ param.desired_type(),encoding_preservation,));} else {@@ -1851,16 +1872,33 @@ mod tests {))?;assert_eq!(vec![DataType::Int64], output);- // Dictionary gets passed through if we use TypeSignatureClass apart from Native- let output = dictionary_input(Coercion::new_exact(TypeSignatureClass::Integer))?;+ // Any always preserves the original physical type+ let output = dictionary_input(Coercion::new_exact(TypeSignatureClass::Any))?;assert_eq!(vec![dictionary.clone()], output);+ let output = dictionary_input(+ Coercion::new_exact(TypeSignatureClass::Any)+ .with_encoding_preservation(EncodingPreservation::dictionary()),+ )?;+ assert_eq!(vec![dictionary.clone()], output);++ // Typed non-Native classes materialize dictionaries by default+ let output = dictionary_input(Coercion::new_exact(TypeSignatureClass::Integer))?;+ assert_eq!(vec![DataType::Int64], output);+let output = dictionary_input(Coercion::new_implicit(TypeSignatureClass::Integer,vec![],NativeType::Int64,))?;- assert_eq!(vec![dictionary.clone()], output);+ assert_eq!(vec![DataType::Int64], output);++ // Typed non-Native classes preserve dictionaries only when requested+ let output = dictionary_input(+ Coercion::new_exact(TypeSignatureClass::Integer)+ .with_encoding_preservation(EncodingPreservation::dictionary()),+ )?;+ assert_eq!(vec![dictionary], output);Ok(())}@@ -1936,7 +1974,7 @@ mod tests {Box::new(DataType::Int64),)]);- // Contrast: without encoding_preservation, non-Native already passes through+ // Without encoding_preservation, non-Native classes materialize dictionariesassert_eq!(dictionary_input(DataType::Int32,@@ -1946,12 +1984,9 @@ mod tests {NativeType::Int64,),)?,- vec![DataType::Dictionary(- Box::new(DataType::Int8),- Box::new(DataType::Int32),- )]+ vec![DataType::Int32]);- // With encoding_preservation, same result — no difference for non-Native+ // With encoding_preservation, non-Native classes preserve dictionariesassert_eq!(dictionary_input(DataType::Int32,@@ -1971,6 +2006,66 @@ mod tests {Ok(())}+ #[test]+ fn test_coercible_nested_dictionary() -> Result<()> {+ let nested_dictionary = DataType::Dictionary(+ Box::new(DataType::Int8),+ Box::new(DataType::Dictionary(+ Box::new(DataType::Int16),+ Box::new(DataType::Int32),+ )),+ );+ let nested_dictionary_input = |coercion| -> Result<Vec<DataType>> {+ fields_with_udf(+ &[Field::new("field", nested_dictionary.clone(), true).into()],+ &MockUdf(Signature::coercible(vec![coercion], Volatility::Immutable)),+ )+ .map(|v| v.into_iter().map(|f| f.data_type().clone()).collect())+ };++ // Without preservation, recursively unwrap dictionaries to the unchanged leaf.+ let output =+ nested_dictionary_input(Coercion::new_exact(TypeSignatureClass::Integer))?;+ assert_eq!(vec![DataType::Int32], output);++ // With preservation, restore the complete dictionary stack around the leaf.+ let output = nested_dictionary_input(+ Coercion::new_exact(TypeSignatureClass::Integer)+ .with_encoding_preservation(EncodingPreservation::dictionary()),+ )?;+ assert_eq!(vec![nested_dictionary.clone()], output);++ let int64_coercion = || {+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_int64()),+ vec![TypeSignatureClass::Integer],+ NativeType::Int64,+ )+ };++ // Without preservation, materialize the coerced leaf type.+ let output = nested_dictionary_input(int64_coercion())?;+ assert_eq!(vec![DataType::Int64], output);++ // With preservation, restore the complete dictionary stack around the coerced leaf.+ let output = nested_dictionary_input(+ int64_coercion()+ .with_encoding_preservation(EncodingPreservation::dictionary()),+ )?;+ assert_eq!(+ vec![DataType::Dictionary(+ Box::new(DataType::Int8),+ Box::new(DataType::Dictionary(+ Box::new(DataType::Int16),+ Box::new(DataType::Int64),+ )),+ )],+ output+ );++ Ok(())+ }+#[test]fn test_coercible_run_end_encoded() -> Result<()> {let run_end_encoded = DataType::RunEndEncoded(diff --git a/datafusion/spark/src/function/bitmap/bitmap_count.rs b/datafusion/spark/src/function/bitmap/bitmap_count.rsindex 89bea101af..18d5848688 100644--- a/datafusion/spark/src/function/bitmap/bitmap_count.rs+++ b/datafusion/spark/src/function/bitmap/bitmap_count.rs@@ -28,8 +28,8 @@ use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64use datafusion_common::utils::take_function_args;use datafusion_common::{Result, internal_err};use datafusion_expr::{- Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,- TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, EncodingPreservation, ScalarFunctionArgs, ScalarUDFImpl,+ Signature, TypeSignatureClass, Volatility,};use datafusion_functions::downcast_arg;use datafusion_functions::utils::make_scalar_function;@@ -49,7 +49,10 @@ impl BitmapCount {pub fn new() -> Self {Self {signature: Signature::coercible(- vec![Coercion::new_exact(TypeSignatureClass::Binary)],+ vec![+ Coercion::new_exact(TypeSignatureClass::Binary)+ .with_encoding_preservation(EncodingPreservation::dictionary()),+ ],Volatility::Immutable,),}diff --git a/datafusion/spark/src/function/math/hex.rs b/datafusion/spark/src/function/math/hex.rsindex 55c9cda63c..c7b82d5373 100644--- a/datafusion/spark/src/function/math/hex.rs+++ b/datafusion/spark/src/function/math/hex.rs@@ -34,8 +34,8 @@ use datafusion_common::{exec_datafusion_err, exec_err,};use datafusion_expr::{- Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,- TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, EncodingPreservation, ScalarFunctionArgs, ScalarUDFImpl,+ Signature, TypeSignature, TypeSignatureClass, Volatility,};/// <https://spark.apache.org/docs/latest/api/sql/index.html#hex>#[derive(Debug, PartialEq, Eq, Hash)]@@ -60,7 +60,8 @@ impl SparkHex {let string = Coercion::new_exact(TypeSignatureClass::Native(logical_string()));- let binary = Coercion::new_exact(TypeSignatureClass::Binary);+ let binary = Coercion::new_exact(TypeSignatureClass::Binary)+ .with_encoding_preservation(EncodingPreservation::dictionary());let variants = vec![// accepts numeric typesdiff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.sltindex 7e15b48a0d..c7c997f330 100644--- a/datafusion/sqllogictest/test_files/expr.slt+++ b/datafusion/sqllogictest/test_files/expr.slt@@ -850,6 +850,15 @@ SELECT to_hex(0)----0+query T+SELECT to_hex(arrow_cast(a, 'Dictionary(Int32, Int64)'))+FROM (VALUES (0), (10), (255), (NULL)) AS t(a)+----+0+a+ff+NULL+# negative values (two's complement encoding)query TSELECT to_hex(-1)diff --git a/datafusion/sqllogictest/test_files/spark/bitmap/bitmap_count.slt b/datafusion/sqllogictest/test_files/spark/bitmap/bitmap_count.sltindex 39dca51222..3ac5337cd7 100644--- a/datafusion/sqllogictest/test_files/spark/bitmap/bitmap_count.slt+++ b/datafusion/sqllogictest/test_files/spark/bitmap/bitmap_count.slt@@ -68,6 +68,21 @@ SELECT bitmap_count(arrow_cast(a, 'Dictionary(Int32, Binary)')) FROM (VALUES (X'16NULL+# The CAST to Dictionary below comes from the explicit arrow_cast. There must+# not be an additional outer CAST(... AS Binary) before bitmap_count.+query TT+EXPLAIN SELECT bitmap_count(arrow_cast(a, 'Dictionary(Int32, Binary)'))+FROM (VALUES (X'1010'), (X'0AB0'), (X'FFFF'), (NULL)) AS t(a);+----+logical_plan+01)Projection: bitmap_count(CAST(t.a AS Dictionary(Int32, Binary))) AS bitmap_count(arrow_cast(t.a,Utf8("Dictionary(Int32, Binary)")))+02)--SubqueryAlias: t+03)----Projection: column1 AS a+04)------Values: (Binary("16,16")), (Binary("10,176")), (Binary("255,255")), (Binary(NULL))+physical_plan+01)ProjectionExec: expr=[bitmap_count(CAST(column1@0 AS Dictionary(Int32, Binary))) as bitmap_count(arrow_cast(t.a,Utf8("Dictionary(Int32, Binary)")))]+02)--DataSourceExec: partitions=1, partition_sizes=[1]+query ISELECT bitmap_count(arrow_cast(a, 'Dictionary(Int8, Binary)')) FROM (VALUES (X'1010'), (X'0AB0'), (X'FFFF'), (NULL)) AS t(a);----diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex f7d2745549..c221104117 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -168,8 +168,10 @@ unchanged.### `Coercion` supports dictionary encoding preservation`datafusion_expr_common::signature::Coercion` now supports optional dictionary-encoding preservation. When enabled for `TypeSignatureClass::Native(...)`-coercions, DataFusion coerces dictionary inputs to+encoding preservation. Typed coercions materialize dictionary inputs by+default, including both `TypeSignatureClass::Native(...)` and broader classes+such as `Integer`, `Numeric`, and `Binary`. When preservation is enabled,+DataFusion instead coerces dictionary inputs to`Dictionary(original_key_type, coerced_value_type)` instead of materializing themto the coerced value type.@@ -186,6 +188,11 @@ derives its return type from that coerced argument type, code that checks exactresult types may need to update its expectations or add an explicit cast tomaterialize the result.+This changes the previous behavior of typed non-native classes such as+`Integer` and `Binary`, which retained the physical dictionary type by default.+UDFs relying on that behavior must now explicitly enable dictionary+preservation. `TypeSignatureClass::Any` is unaffected.+### `GroupsAccumulator::merge_batch` no longer takes `opt_filter`The `opt_filter` argument has been removed from