Apache DataFusion SQL Query Engine
Remove CastColumnExpr and custom_file_casts example; unify on field-aware CastExpr (#21563)
kosiewa13c23df67c2
diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rsindex 6b37db653e..6db0cda565 100644--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs@@ -33,7 +33,7 @@ use datafusion::execution::context::SessionContext;use datafusion::execution::object_store::ObjectStoreUrl;use datafusion::parquet::arrow::ArrowWriter;use datafusion::physical_expr::PhysicalExpr;-use datafusion::physical_expr::expressions::{CastColumnExpr, CastExpr};+use datafusion::physical_expr::expressions::CastExpr;use datafusion::prelude::SessionConfig;use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,@@ -43,9 +43,10 @@ use object_store::path::Path;use object_store::{ObjectStore, ObjectStoreExt, PutPayload};// Example showing how to implement custom casting rules to adapt file schemas.-// This example enforces that casts must be strictly widening: if the file type is Int64 and the table type is Int32, it will error-// before even reading the data.-// Without this custom cast rule DataFusion would happily do the narrowing cast, potentially erroring only if it found a row with data it could not cast.+// This example enforces strictly widening casts: if the file type is Int64 and+// the table type is Int32, it errors before reading the data. Without this+// custom cast rule DataFusion would apply the narrowing cast and might only+// error after reading a row that it could not cast.pub async fn custom_file_casts() -> Result<()> {println!("=== Creating example data ===");@@ -139,7 +140,7 @@ async fn write_data(Ok(())}-/// Factory for creating DefaultValuePhysicalExprAdapter instances+/// Factory for creating custom cast physical expression adapters#[derive(Debug)]struct CustomCastPhysicalExprAdapterFactory {inner: Arc<dyn PhysicalExprAdapterFactory>,@@ -167,8 +168,8 @@ impl PhysicalExprAdapterFactory for CustomCastPhysicalExprAdapterFactory {}}-/// Custom PhysicalExprAdapter that handles missing columns with default values from metadata-/// and wraps DefaultPhysicalExprAdapter for standard schema adaptation+/// Custom `PhysicalExprAdapter` that wraps the default adapter and rejects+/// narrowing file-schema casts.#[derive(Debug, Clone)]struct CustomCastsPhysicalExprAdapter {physical_file_schema: SchemaRef,@@ -177,34 +178,23 @@ struct CustomCastsPhysicalExprAdapter {impl PhysicalExprAdapter for CustomCastsPhysicalExprAdapter {fn rewrite(&self, mut expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {- // First delegate to the inner adapter to handle missing columns and discover any necessary casts+ // First delegate to the inner adapter to handle standard schema adaptation+ // and discover any necessary casts.expr = self.inner.rewrite(expr)?;- // Now we can apply custom casting rules or even swap out all CastExprs for a custom cast kernel / expression- // For example, [DataFusion Comet](https://github.com/apache/datafusion-comet) has a [custom cast kernel](https://github.com/apache/datafusion-comet/blob/b4ac876ab420ed403ac7fc8e1b29f42f1f442566/native/spark-expr/src/conversion_funcs/cast.rs#L133-L138).+ // Now apply custom casting rules or swap CastExprs for a custom cast+ // kernel / expression. For example, DataFusion Comet has a custom cast+ // kernel in its native Spark expression implementation.expr.transform(|expr| {if let Some(cast) = expr.as_any().downcast_ref::<CastExpr>() {let input_data_type =cast.expr().data_type(&self.physical_file_schema)?;- let output_data_type = cast.data_type(&self.physical_file_schema)?;+ let output_data_type = cast.target_field().data_type();if !cast.is_bigger_cast(&input_data_type) {return not_impl_err!("Unsupported CAST from {input_data_type} to {output_data_type}");}}- if let Some(cast) = expr.as_any().downcast_ref::<CastColumnExpr>() {- let input_data_type =- cast.expr().data_type(&self.physical_file_schema)?;- let output_data_type = cast.data_type(&self.physical_file_schema)?;- if !CastExpr::check_bigger_cast(- cast.target_field().data_type(),- &input_data_type,- ) {- return not_impl_err!(- "Unsupported CAST from {input_data_type} to {output_data_type}"- );- }- }Ok(Transformed::no(expr))}).data()diff --git a/datafusion-examples/examples/custom_data_source/main.rs b/datafusion-examples/examples/custom_data_source/main.rsindex 0d21a62591..40409d3690 100644--- a/datafusion-examples/examples/custom_data_source/main.rs+++ b/datafusion-examples/examples/custom_data_source/main.rs@@ -21,7 +21,7 @@//!//! ## Usage//! ```bash-//! cargo run --example custom_data_source -- [all|csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider]+//! cargo run --example custom_data_source -- [all|adapter_serialization|csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider]//! ```//!//! Each subcommand runs a corresponding example:diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rsindex f055c8298b..5805f933a9 100644--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs@@ -644,7 +644,7 @@ mod tests {.expect("Expected CastExpr")}- fn assert_cast_column(cast_expr: &CastExpr, name: &str, index: usize) {+ fn assert_cast_input_column(cast_expr: &CastExpr, name: &str, index: usize) {let inner_col = cast_expr.expr().as_any()@@ -771,7 +771,7 @@ mod tests {let left_cast = assert_cast_expr(left.left());assert_eq!(left_cast.target_field().data_type(), &DataType::Int64);- assert_cast_column(left_cast, "a", 0);+ assert_cast_input_column(left_cast, "a", 0);let right = outer.right()@@ -1672,7 +1672,7 @@ mod tests {let cast_expr = assert_cast_expr(&result);// Verify the inner column points to the correct physical index (1)- assert_cast_column(cast_expr, "a", 1);+ assert_cast_input_column(cast_expr, "a", 1);// Verify cast typesassert_eq!(@@ -1692,7 +1692,7 @@ mod tests {// Regression: this must still resolve against physical field `a` by name.let rewritten = adapter.rewrite(Arc::new(Column::new("a", 0))).unwrap();let cast_expr = assert_cast_expr(&rewritten);- assert_cast_column(cast_expr, "a", 1);+ assert_cast_input_column(cast_expr, "a", 1);assert_eq!(cast_expr.target_field().data_type(), &DataType::Int64);}}diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rsindex 7c4ac60419..8d4167050b 100644--- a/datafusion/physical-expr/src/expressions/cast.rs+++ b/datafusion/physical-expr/src/expressions/cast.rs@@ -386,16 +386,46 @@ mod tests {use arrow::{array::{- Array, Decimal128Array, Float32Array, Float64Array, Int8Array, Int16Array,- Int32Array, Int64Array, StringArray, Time64NanosecondArray,- TimestampNanosecondArray, UInt32Array,+ Array, ArrayRef, Decimal128Array, Float32Array, Float64Array, Int8Array,+ Int16Array, Int32Array, Int64Array, StringArray, StructArray,+ Time64NanosecondArray, TimestampNanosecondArray, UInt32Array,},datatypes::*,};+ use datafusion_common::ScalarValue;+ use datafusion_common::cast::{+ as_boolean_array, as_int64_array, as_string_array, as_struct_array,+ as_uint8_array,+ };use datafusion_physical_expr_common::physical_expr::fmt_sql;use insta::assert_snapshot;use std::collections::HashMap;+ fn make_struct_array(fields: Fields, arrays: Vec<ArrayRef>) -> StructArray {+ StructArray::new(fields, arrays, None)+ }++ fn cast_struct_array(+ column: &str,+ input_field: Field,+ target_field: Field,+ input_array: StructArray,+ ) -> Result<StructArray> {+ let schema = Arc::new(Schema::new(vec![input_field]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(input_array) as ArrayRef],+ )?;+ let expr = CastExpr::new_with_target_field(+ col(column, schema.as_ref())?,+ Arc::new(target_field),+ None,+ );++ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ Ok(as_struct_array(result.as_ref())?.clone())+ }+// runs an end-to-end test of physical type cast// 1. construct a record batch with a column "a" of type A// 2. construct a physical expression of CAST(a AS B)@@ -903,43 +933,30 @@ mod tests {}#[test]- fn field_aware_cast_return_field_preserves_target_metadata() -> Result<()> {- let schema = Schema::new(vec![Field::new("a", Int32, false)]);- let expr = CastExpr::new_with_target_field(- col("a", &schema)?,- Arc::new(Field::new("cast_target", Int64, true).with_metadata(- HashMap::from([("target_meta".to_string(), "1".to_string())]),- )),- None,- );-- let field = expr.return_field(&schema)?;-- assert_eq!(field.name(), "cast_target");- assert_eq!(field.data_type(), &Int64);- assert!(field.is_nullable());- assert_eq!(- field.metadata().get("target_meta").map(String::as_str),- Some("1")- );-- Ok(())- }-- #[test]- fn field_aware_cast_nullable_prefers_child_nullability() -> Result<()> {- // When the child expression is nullable the cast must be treated as- // nullable even if the explicitly supplied target field is marked- // non-nullable. return_field() still reflects the target metadata.- let schema = Schema::new(vec![Field::new("a", Int32, true)]);- let expr = CastExpr::new_with_target_field(- col("a", &schema)?,- Arc::new(Field::new("cast_target", Int64, false)),- None,- );+ fn field_aware_cast_preserves_target_field_semantics() -> Result<()> {+ let metadata = HashMap::from([("target_meta".to_string(), "1".to_string())]);- assert!(expr.nullable(&schema)?);- assert!(!expr.return_field(&schema)?.is_nullable());+ for (child_nullable, target_nullable) in [(true, false), (false, true)] {+ let schema = Schema::new(vec![Field::new("a", Int32, child_nullable)]);+ let expr = CastExpr::new_with_target_field(+ col("a", &schema)?,+ Arc::new(+ Field::new("cast_target", Int64, target_nullable)+ .with_metadata(metadata.clone()),+ ),+ None,+ );++ let field = expr.return_field(&schema)?;+ assert_eq!(field.name(), "cast_target");+ assert_eq!(field.data_type(), &Int64);+ assert_eq!(field.is_nullable(), target_nullable);+ assert_eq!(+ field.metadata().get("target_meta").map(String::as_str),+ Some("1")+ );+ assert_eq!(expr.nullable(&schema)?, child_nullable || target_nullable);+ }Ok(())}@@ -959,23 +976,6 @@ mod tests {Ok(())}- #[test]- fn field_aware_cast_nullable_child_nonnullable_targets_nullable() -> Result<()> {- // child is non-nullable but the target field is marked nullable; the- // nullable() result should still be true because the field allows nulls.- let schema = Schema::new(vec![Field::new("a", Int32, false)]);- let expr = CastExpr::new_with_target_field(- col("a", &schema)?,- Arc::new(Field::new("cast_target", Int64, true)),- None,- );-- assert!(expr.nullable(&schema)?);- assert!(expr.return_field(&schema)?.is_nullable());-- Ok(())- }-#[test]fn struct_cast_validation_uses_nested_target_fields() -> Result<()> {let source_type = Struct(Fields::from(vec![@@ -1002,6 +1002,130 @@ mod tests {Ok(())}+ #[test]+ fn field_aware_cast_struct_array_missing_child() -> Result<()> {+ let source_a = Field::new("a", Int32, true);+ let source_b = Field::new("b", Utf8, true);+ let target_field = Field::new(+ "s",+ Struct(+ vec![+ Arc::new(Field::new("a", Int64, true)),+ Arc::new(Field::new("c", Utf8, true)),+ ]+ .into(),+ ),+ true,+ );++ let struct_array = cast_struct_array(+ "s",+ Field::new(+ "s",+ Struct(+ vec![Arc::new(source_a.clone()), Arc::new(source_b.clone())].into(),+ ),+ true,+ ),+ target_field,+ make_struct_array(+ vec![Arc::new(source_a), Arc::new(source_b)].into(),+ vec![+ Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef,+ Arc::new(StringArray::from(vec![Some("alpha"), Some("beta")]))+ as ArrayRef,+ ],+ ),+ )?;+ let cast_a = as_int64_array(struct_array.column_by_name("a").unwrap().as_ref())?;+ assert_eq!(cast_a.value(0), 1);+ assert!(cast_a.is_null(1));++ let cast_c = as_string_array(struct_array.column_by_name("c").unwrap().as_ref())?;+ assert!(cast_c.is_null(0));+ assert!(cast_c.is_null(1));+ Ok(())+ }++ #[test]+ fn field_aware_cast_nested_struct_array() -> Result<()> {+ let inner_source = Field::new(+ "inner",+ Struct(vec![Arc::new(Field::new("x", Int32, true))].into()),+ true,+ );+ let inner_target = Field::new(+ "inner",+ Struct(+ vec![+ Arc::new(Field::new("x", Int64, true)),+ Arc::new(Field::new("y", Boolean, true)),+ ]+ .into(),+ ),+ true,+ );+ let target_field =+ Field::new("root", Struct(vec![Arc::new(inner_target)].into()), true);++ let inner_struct = make_struct_array(+ vec![Arc::new(Field::new("x", Int32, true))].into(),+ vec![Arc::new(Int32Array::from(vec![Some(7), None])) as ArrayRef],+ );+ let outer_struct = make_struct_array(+ vec![Arc::new(inner_source.clone())].into(),+ vec![Arc::new(inner_struct) as ArrayRef],+ );+ let struct_array = cast_struct_array(+ "root",+ Field::new("root", Struct(vec![Arc::new(inner_source)].into()), true),+ target_field,+ outer_struct,+ )?;+ let inner =+ as_struct_array(struct_array.column_by_name("inner").unwrap().as_ref())?;+ let x = as_int64_array(inner.column_by_name("x").unwrap().as_ref())?;+ assert_eq!(x.value(0), 7);+ assert!(x.is_null(1));+ let y = as_boolean_array(inner.column_by_name("y").unwrap().as_ref())?;+ assert!(y.is_null(0));+ assert!(y.is_null(1));+ Ok(())+ }++ #[test]+ fn field_aware_cast_struct_scalar() -> Result<()> {+ let source_field = Field::new("a", Int32, true);+ let target_field = Field::new(+ "s",+ Struct(vec![Arc::new(Field::new("a", UInt8, true))].into()),+ true,+ );++ let schema = Arc::new(Schema::new(vec![Field::new(+ "s",+ Struct(vec![Arc::new(source_field.clone())].into()),+ true,+ )]));+ let scalar_struct = make_struct_array(+ vec![Arc::new(source_field)].into(),+ vec![Arc::new(Int32Array::from(vec![Some(9)])) as ArrayRef],+ );+ let literal = Arc::new(crate::expressions::Literal::new(ScalarValue::Struct(+ Arc::new(scalar_struct),+ )));+ let expr = CastExpr::new_with_target_field(literal, Arc::new(target_field), None);++ let batch = RecordBatch::new_empty(schema);+ let result = expr.evaluate(&batch)?;+ let ColumnarValue::Scalar(ScalarValue::Struct(array)) = result else {+ panic!("expected struct scalar");+ };+ let casted = as_uint8_array(array.column_by_name("a").unwrap().as_ref())?;+ assert_eq!(casted.value(0), 9);+ Ok(())+ }+#[test]#[ignore] // TODO: https://github.com/apache/datafusion/issues/5396fn test_cast_decimal() -> Result<()> {diff --git a/datafusion/physical-expr/src/expressions/cast_column.rs b/datafusion/physical-expr/src/expressions/cast_column.rsdeleted file mode 100644index 158d467c08..0000000000--- a/datafusion/physical-expr/src/expressions/cast_column.rs+++ /dev/null@@ -1,420 +0,0 @@-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--//! Physical expression for struct-aware casting of columns.--use super::cast::cast_expr_properties;-use crate::physical_expr::PhysicalExpr;-use arrow::{- compute::CastOptions,- datatypes::{DataType, FieldRef, Schema},- record_batch::RecordBatch,-};-use datafusion_common::{- Result, ScalarValue, format::DEFAULT_CAST_OPTIONS, nested_struct::cast_column,-};-use datafusion_expr_common::columnar_value::ColumnarValue;-use datafusion_expr_common::sort_properties::ExprProperties;-use std::{- any::Any,- fmt::{self, Display},- hash::Hash,- sync::Arc,-};-/// A physical expression that applies [`cast_column`] to its input.-///-/// [`CastColumnExpr`] extends the regular [`CastExpr`](super::CastExpr) by-/// retaining schema metadata for both the input and output fields. This allows-/// the evaluator to perform struct-aware casts that honour nested field-/// ordering, preserve nullability, and fill missing fields with null values.-///-/// This expression is intended for schema rewriting scenarios where the-/// planner already resolved the input column but needs to adapt its physical-/// representation to a new [`arrow::datatypes::Field`]. It mirrors the behaviour of the-/// [`datafusion_common::nested_struct::cast_column`] helper while integrating-/// with the `PhysicalExpr` trait so it can participate in the execution plan-/// like any other column expression.-#[derive(Debug, Clone, Eq)]-pub struct CastColumnExpr {- /// The physical expression producing the value to cast.- expr: Arc<dyn PhysicalExpr>,- /// The logical field of the input column.- input_field: FieldRef,- /// The field metadata describing the desired output column.- target_field: FieldRef,- /// Options forwarded to [`cast_column`].- cast_options: CastOptions<'static>,-}--// Manually derive `PartialEq`/`Hash` as `Arc<dyn PhysicalExpr>` does not-// implement these traits by default for the trait object.-impl PartialEq for CastColumnExpr {- fn eq(&self, other: &Self) -> bool {- self.expr.eq(&other.expr)- && self.input_field.eq(&other.input_field)- && self.target_field.eq(&other.target_field)- && self.cast_options.eq(&other.cast_options)- }-}--impl Hash for CastColumnExpr {- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {- self.expr.hash(state);- self.input_field.hash(state);- self.target_field.hash(state);- self.cast_options.hash(state);- }-}--impl CastColumnExpr {- /// Create a new [`CastColumnExpr`].- pub fn new(- expr: Arc<dyn PhysicalExpr>,- input_field: FieldRef,- target_field: FieldRef,- cast_options: Option<CastOptions<'static>>,- ) -> Self {- Self {- expr,- input_field,- target_field,- cast_options: cast_options.unwrap_or(DEFAULT_CAST_OPTIONS),- }- }-- /// The expression that produces the value to be cast.- pub fn expr(&self) -> &Arc<dyn PhysicalExpr> {- &self.expr- }-- /// Field metadata describing the resolved input column.- pub fn input_field(&self) -> &FieldRef {- &self.input_field- }-- /// Field metadata describing the output column after casting.- pub fn target_field(&self) -> &FieldRef {- &self.target_field- }-}--impl Display for CastColumnExpr {- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- write!(- f,- "CAST_COLUMN({} AS {})",- self.expr,- self.target_field.data_type()- )- }-}--impl PhysicalExpr for CastColumnExpr {- fn as_any(&self) -> &dyn Any {- self- }-- fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {- Ok(self.target_field.data_type().clone())- }-- fn nullable(&self, _input_schema: &Schema) -> Result<bool> {- Ok(self.target_field.is_nullable())- }-- fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {- let value = self.expr.evaluate(batch)?;- match value {- ColumnarValue::Array(array) => {- let casted = cast_column(- &array,- self.target_field.data_type(),- &self.cast_options,- )?;- Ok(ColumnarValue::Array(casted))- }- ColumnarValue::Scalar(scalar) => {- let as_array = scalar.to_array_of_size(1)?;- let casted = cast_column(- &as_array,- self.target_field.data_type(),- &self.cast_options,- )?;- let result = ScalarValue::try_from_array(casted.as_ref(), 0)?;- Ok(ColumnarValue::Scalar(result))- }- }- }-- fn return_field(&self, _input_schema: &Schema) -> Result<FieldRef> {- Ok(Arc::clone(&self.target_field))- }-- fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {- vec![&self.expr]- }-- fn with_new_children(- self: Arc<Self>,- mut children: Vec<Arc<dyn PhysicalExpr>>,- ) -> Result<Arc<dyn PhysicalExpr>> {- assert_eq!(children.len(), 1);- let child = children.pop().expect("CastColumnExpr child");- Ok(Arc::new(Self::new(- child,- Arc::clone(&self.input_field),- Arc::clone(&self.target_field),- Some(self.cast_options.clone()),- )))- }-- /// A [`CastColumnExpr`] preserves the ordering of its child if the cast is done- /// under the same datatype family.- fn get_properties(&self, children: &[ExprProperties]) -> Result<ExprProperties> {- cast_expr_properties(&children[0], self.target_field.data_type())- }-- fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- Display::fmt(self, f)- }-}--#[cfg(test)]-mod tests {- use super::*;-- use crate::expressions::{Column, Literal};- use arrow::{- array::{Array, ArrayRef, BooleanArray, Int32Array, StringArray, StructArray},- datatypes::{DataType, Field, Fields, SchemaRef},- };- use datafusion_common::{- Result as DFResult, ScalarValue,- cast::{as_int64_array, as_string_array, as_struct_array, as_uint8_array},- };-- fn make_schema(field: &Field) -> SchemaRef {- Arc::new(Schema::new(vec![field.clone()]))- }-- fn make_struct_array(fields: Fields, arrays: Vec<ArrayRef>) -> StructArray {- StructArray::new(fields, arrays, None)- }-- #[test]- fn cast_primitive_array() -> DFResult<()> {- let input_field = Field::new("a", DataType::Int32, true);- let target_field = Field::new("a", DataType::Int64, true);- let schema = make_schema(&input_field);-- let values = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));- let batch = RecordBatch::try_new(Arc::clone(&schema), vec![values])?;-- let column = Arc::new(Column::new_with_schema("a", schema.as_ref())?);- let expr = CastColumnExpr::new(- column,- Arc::new(input_field.clone()),- Arc::new(target_field.clone()),- None,- );-- let result = expr.evaluate(&batch)?;- let ColumnarValue::Array(array) = result else {- panic!("expected array");- };- let casted = as_int64_array(array.as_ref())?;- assert_eq!(casted.value(0), 1);- assert!(casted.is_null(1));- assert_eq!(casted.value(2), 3);- Ok(())- }-- #[test]- fn cast_struct_array_missing_child() -> DFResult<()> {- let source_a = Field::new("a", DataType::Int32, true);- let source_b = Field::new("b", DataType::Utf8, true);- let input_field = Field::new(- "s",- DataType::Struct(- vec![Arc::new(source_a.clone()), Arc::new(source_b.clone())].into(),- ),- true,- );- let target_a = Field::new("a", DataType::Int64, true);- let target_c = Field::new("c", DataType::Utf8, true);- let target_field = Field::new(- "s",- DataType::Struct(- vec![Arc::new(target_a.clone()), Arc::new(target_c.clone())].into(),- ),- true,- );-- let schema = make_schema(&input_field);- let struct_array = make_struct_array(- vec![Arc::new(source_a.clone()), Arc::new(source_b.clone())].into(),- vec![- Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef,- Arc::new(StringArray::from(vec![Some("alpha"), Some("beta")]))- as ArrayRef,- ],- );- let batch = RecordBatch::try_new(- Arc::clone(&schema),- vec![Arc::new(struct_array) as Arc<_>],- )?;-- let column = Arc::new(Column::new_with_schema("s", schema.as_ref())?);- let expr = CastColumnExpr::new(- column,- Arc::new(input_field.clone()),- Arc::new(target_field.clone()),- None,- );-- let result = expr.evaluate(&batch)?;- let ColumnarValue::Array(array) = result else {- panic!("expected array");- };- let struct_array = as_struct_array(array.as_ref())?;- let cast_a = as_int64_array(struct_array.column_by_name("a").unwrap().as_ref())?;- assert_eq!(cast_a.value(0), 1);- assert!(cast_a.is_null(1));-- let cast_c = as_string_array(struct_array.column_by_name("c").unwrap().as_ref())?;- assert!(cast_c.is_null(0));- assert!(cast_c.is_null(1));- Ok(())- }-- #[test]- fn cast_nested_struct_array() -> DFResult<()> {- let inner_source = Field::new(- "inner",- DataType::Struct(- vec![Arc::new(Field::new("x", DataType::Int32, true))].into(),- ),- true,- );- let outer_field = Field::new(- "root",- DataType::Struct(vec![Arc::new(inner_source.clone())].into()),- true,- );-- let inner_target = Field::new(- "inner",- DataType::Struct(- vec![- Arc::new(Field::new("x", DataType::Int64, true)),- Arc::new(Field::new("y", DataType::Boolean, true)),- ]- .into(),- ),- true,- );- let target_field = Field::new(- "root",- DataType::Struct(vec![Arc::new(inner_target.clone())].into()),- true,- );-- let schema = make_schema(&outer_field);-- let inner_struct = make_struct_array(- vec![Arc::new(Field::new("x", DataType::Int32, true))].into(),- vec![Arc::new(Int32Array::from(vec![Some(7), None])) as ArrayRef],- );- let outer_struct = make_struct_array(- vec![Arc::new(inner_source.clone())].into(),- vec![Arc::new(inner_struct) as ArrayRef],- );- let batch = RecordBatch::try_new(- Arc::clone(&schema),- vec![Arc::new(outer_struct) as ArrayRef],- )?;-- let column = Arc::new(Column::new_with_schema("root", schema.as_ref())?);- let expr = CastColumnExpr::new(- column,- Arc::new(outer_field.clone()),- Arc::new(target_field.clone()),- None,- );-- let result = expr.evaluate(&batch)?;- let ColumnarValue::Array(array) = result else {- panic!("expected array");- };- let struct_array = as_struct_array(array.as_ref())?;- let inner =- as_struct_array(struct_array.column_by_name("inner").unwrap().as_ref())?;- let x = as_int64_array(inner.column_by_name("x").unwrap().as_ref())?;- assert_eq!(x.value(0), 7);- assert!(x.is_null(1));- let y = inner.column_by_name("y").unwrap();- let y = y- .as_any()- .downcast_ref::<BooleanArray>()- .expect("boolean array");- assert!(y.is_null(0));- assert!(y.is_null(1));- Ok(())- }-- #[test]- fn cast_struct_scalar() -> DFResult<()> {- let source_field = Field::new("a", DataType::Int32, true);- let input_field = Field::new(- "s",- DataType::Struct(vec![Arc::new(source_field.clone())].into()),- true,- );- let target_field = Field::new(- "s",- DataType::Struct(- vec![Arc::new(Field::new("a", DataType::UInt8, true))].into(),- ),- true,- );-- let schema = make_schema(&input_field);- let scalar_struct = StructArray::new(- vec![Arc::new(source_field.clone())].into(),- vec![Arc::new(Int32Array::from(vec![Some(9)])) as ArrayRef],- None,- );- let literal =- Arc::new(Literal::new(ScalarValue::Struct(Arc::new(scalar_struct))));- let expr = CastColumnExpr::new(- literal,- Arc::new(input_field.clone()),- Arc::new(target_field.clone()),- None,- );-- let batch = RecordBatch::new_empty(Arc::clone(&schema));- let result = expr.evaluate(&batch)?;- let ColumnarValue::Scalar(ScalarValue::Struct(array)) = result else {- panic!("expected struct scalar");- };- let casted = array.column_by_name("a").unwrap();- let casted = as_uint8_array(casted.as_ref())?;- assert_eq!(casted.value(0), 9);- Ok(())- }-}diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rsindex 8a27009280..1e08235548 100644--- a/datafusion/physical-expr/src/expressions/mod.rs+++ b/datafusion/physical-expr/src/expressions/mod.rs@@ -21,7 +21,6 @@mod binary;mod case;mod cast;-mod cast_column;mod column;mod dynamic_filters;mod in_list;@@ -42,7 +41,6 @@ pub use crate::aggregate::stats::StatsType;pub use binary::{BinaryExpr, binary, similar_to};pub use case::{CaseExpr, case};pub use cast::{CastExpr, cast};-pub use cast_column::CastColumnExpr;pub use column::{Column, col, with_new_schema};pub use datafusion_expr::utils::format_state_name;pub use dynamic_filters::DynamicFilterPhysicalExpr;diff --git a/docs/source/library-user-guide/upgrading/54.0.0.md b/docs/source/library-user-guide/upgrading/54.0.0.mdindex c34c08ec19..16606a0f40 100644--- a/docs/source/library-user-guide/upgrading/54.0.0.md+++ b/docs/source/library-user-guide/upgrading/54.0.0.md@@ -76,6 +76,18 @@ adjustment:See [#15161](https://github.com/apache/datafusion/issues/15161) and[PR #20426](https://github.com/apache/datafusion/pull/20426) for details.+### `CastColumnExpr` removed in favor of field-aware `CastExpr`++`datafusion_physical_expr::expressions::CastColumnExpr` has been removed; use+the field-aware `datafusion_physical_expr::expressions::CastExpr` instead.++If your code downcasted to `CastColumnExpr`, downcast to `CastExpr` instead and+use `CastExpr::target_field()` for the output field metadata and+`CastExpr::expr()` for the input expression. To construct casts with explicit+field semantics, use `CastExpr::new_with_target_field(...)`. The type-only+`CastExpr::new(...)` and `cast(...)` helpers remain available for callers that+only have a `DataType`.+### `comparison_coercion_numeric` removed, replaced by `comparison_coercion`The `comparison_coercion_numeric` function has been removed. Its behavior