Skip to content

Apache DataFusion SQL Query Engine

Remove CastColumnExpr and custom_file_casts example; unify on field-aware CastExpr (#21563)

kosiewa13c23df67c2
diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
index 6b37db653e..6db0cda565 100644
--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
@@ -33,7 +33,7 @@ use datafusion::execution::context::SessionContext;
use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::parquet::arrow::ArrowWriter;
use datafusion::physical_expr::PhysicalExpr;
-use datafusion::physical_expr::expressions::{CastColumnExpr, CastExpr};
+use datafusion::physical_expr::expressions::CastExpr;
use datafusion::prelude::SessionConfig;
use datafusion_physical_expr_adapter::{
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
@@ -43,9 +43,10 @@ use object_store::path::Path;
use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
// Example showing how to implement custom casting rules to adapt file schemas.
-// This example enforces that casts must be strictly widening: if the file type is Int64 and the table type is Int32, it will error
-// before even reading the data.
-// Without this custom cast rule DataFusion would happily do the narrowing cast, potentially erroring only if it found a row with data it could not cast.
+// This example enforces strictly widening casts: if the file type is Int64 and
+// the table type is Int32, it errors before reading the data. Without this
+// custom cast rule DataFusion would apply the narrowing cast and might only
+// error after reading a row that it could not cast.
pub async fn custom_file_casts() -> Result<()> {
println!("=== Creating example data ===");
@@ -139,7 +140,7 @@ async fn write_data(
Ok(())
}
-/// Factory for creating DefaultValuePhysicalExprAdapter instances
+/// Factory for creating custom cast physical expression adapters
#[derive(Debug)]
struct CustomCastPhysicalExprAdapterFactory {
inner: Arc<dyn PhysicalExprAdapterFactory>,
@@ -167,8 +168,8 @@ impl PhysicalExprAdapterFactory for CustomCastPhysicalExprAdapterFactory {
}
}
-/// Custom PhysicalExprAdapter that handles missing columns with default values from metadata
-/// and wraps DefaultPhysicalExprAdapter for standard schema adaptation
+/// Custom `PhysicalExprAdapter` that wraps the default adapter and rejects
+/// narrowing file-schema casts.
#[derive(Debug, Clone)]
struct CustomCastsPhysicalExprAdapter {
physical_file_schema: SchemaRef,
@@ -177,34 +178,23 @@ struct CustomCastsPhysicalExprAdapter {
impl PhysicalExprAdapter for CustomCastsPhysicalExprAdapter {
fn rewrite(&self, mut expr: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {
- // First delegate to the inner adapter to handle missing columns and discover any necessary casts
+ // First delegate to the inner adapter to handle standard schema adaptation
+ // and discover any necessary casts.
expr = self.inner.rewrite(expr)?;
- // Now we can apply custom casting rules or even swap out all CastExprs for a custom cast kernel / expression
- // For example, [DataFusion Comet](https://github.com/apache/datafusion-comet) has a [custom cast kernel](https://github.com/apache/datafusion-comet/blob/b4ac876ab420ed403ac7fc8e1b29f42f1f442566/native/spark-expr/src/conversion_funcs/cast.rs#L133-L138).
+ // Now apply custom casting rules or swap CastExprs for a custom cast
+ // kernel / expression. For example, DataFusion Comet has a custom cast
+ // kernel in its native Spark expression implementation.
expr.transform(|expr| {
if let Some(cast) = expr.as_any().downcast_ref::<CastExpr>() {
let input_data_type =
cast.expr().data_type(&self.physical_file_schema)?;
- let output_data_type = cast.data_type(&self.physical_file_schema)?;
+ let output_data_type = cast.target_field().data_type();
if !cast.is_bigger_cast(&input_data_type) {
return not_impl_err!(
"Unsupported CAST from {input_data_type} to {output_data_type}"
);
}
}
- if let Some(cast) = expr.as_any().downcast_ref::<CastColumnExpr>() {
- let input_data_type =
- cast.expr().data_type(&self.physical_file_schema)?;
- let output_data_type = cast.data_type(&self.physical_file_schema)?;
- if !CastExpr::check_bigger_cast(
- cast.target_field().data_type(),
- &input_data_type,
- ) {
- return not_impl_err!(
- "Unsupported CAST from {input_data_type} to {output_data_type}"
- );
- }
- }
Ok(Transformed::no(expr))
})
.data()
diff --git a/datafusion-examples/examples/custom_data_source/main.rs b/datafusion-examples/examples/custom_data_source/main.rs
index 0d21a62591..40409d3690 100644
--- a/datafusion-examples/examples/custom_data_source/main.rs
+++ b/datafusion-examples/examples/custom_data_source/main.rs
@@ -21,7 +21,7 @@
//!
//! ## Usage
//! ```bash
-//! cargo run --example custom_data_source -- [all|csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider]
+//! cargo run --example custom_data_source -- [all|adapter_serialization|csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider]
//! ```
//!
//! Each subcommand runs a corresponding example:
diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
index f055c8298b..5805f933a9 100644
--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs
+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
@@ -644,7 +644,7 @@ mod tests {
.expect("Expected CastExpr")
}
- fn assert_cast_column(cast_expr: &CastExpr, name: &str, index: usize) {
+ fn assert_cast_input_column(cast_expr: &CastExpr, name: &str, index: usize) {
let inner_col = cast_expr
.expr()
.as_any()
@@ -771,7 +771,7 @@ mod tests {
let left_cast = assert_cast_expr(left.left());
assert_eq!(left_cast.target_field().data_type(), &DataType::Int64);
- assert_cast_column(left_cast, "a", 0);
+ assert_cast_input_column(left_cast, "a", 0);
let right = outer
.right()
@@ -1672,7 +1672,7 @@ mod tests {
let cast_expr = assert_cast_expr(&result);
// Verify the inner column points to the correct physical index (1)
- assert_cast_column(cast_expr, "a", 1);
+ assert_cast_input_column(cast_expr, "a", 1);
// Verify cast types
assert_eq!(
@@ -1692,7 +1692,7 @@ mod tests {
// Regression: this must still resolve against physical field `a` by name.
let rewritten = adapter.rewrite(Arc::new(Column::new("a", 0))).unwrap();
let cast_expr = assert_cast_expr(&rewritten);
- assert_cast_column(cast_expr, "a", 1);
+ assert_cast_input_column(cast_expr, "a", 1);
assert_eq!(cast_expr.target_field().data_type(), &DataType::Int64);
}
}
diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs
index 7c4ac60419..8d4167050b 100644
--- a/datafusion/physical-expr/src/expressions/cast.rs
+++ b/datafusion/physical-expr/src/expressions/cast.rs
@@ -386,16 +386,46 @@ mod tests {
use arrow::{
array::{
- Array, Decimal128Array, Float32Array, Float64Array, Int8Array, Int16Array,
- Int32Array, Int64Array, StringArray, Time64NanosecondArray,
- TimestampNanosecondArray, UInt32Array,
+ Array, ArrayRef, Decimal128Array, Float32Array, Float64Array, Int8Array,
+ Int16Array, Int32Array, Int64Array, StringArray, StructArray,
+ Time64NanosecondArray, TimestampNanosecondArray, UInt32Array,
},
datatypes::*,
};
+ use datafusion_common::ScalarValue;
+ use datafusion_common::cast::{
+ as_boolean_array, as_int64_array, as_string_array, as_struct_array,
+ as_uint8_array,
+ };
use datafusion_physical_expr_common::physical_expr::fmt_sql;
use insta::assert_snapshot;
use std::collections::HashMap;
+ fn make_struct_array(fields: Fields, arrays: Vec<ArrayRef>) -> StructArray {
+ StructArray::new(fields, arrays, None)
+ }
+
+ fn cast_struct_array(
+ column: &str,
+ input_field: Field,
+ target_field: Field,
+ input_array: StructArray,
+ ) -> Result<StructArray> {
+ let schema = Arc::new(Schema::new(vec![input_field]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(input_array) as ArrayRef],
+ )?;
+ let expr = CastExpr::new_with_target_field(
+ col(column, schema.as_ref())?,
+ Arc::new(target_field),
+ None,
+ );
+
+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;
+ Ok(as_struct_array(result.as_ref())?.clone())
+ }
+
// runs an end-to-end test of physical type cast
// 1. construct a record batch with a column "a" of type A
// 2. construct a physical expression of CAST(a AS B)
@@ -903,43 +933,30 @@ mod tests {
}
#[test]
- fn field_aware_cast_return_field_preserves_target_metadata() -> Result<()> {
- let schema = Schema::new(vec![Field::new("a", Int32, false)]);
- let expr = CastExpr::new_with_target_field(
- col("a", &schema)?,
- Arc::new(Field::new("cast_target", Int64, true).with_metadata(
- HashMap::from([("target_meta".to_string(), "1".to_string())]),
- )),
- None,
- );
-
- let field = expr.return_field(&schema)?;
-
- assert_eq!(field.name(), "cast_target");
- assert_eq!(field.data_type(), &Int64);
- assert!(field.is_nullable());
- assert_eq!(
- field.metadata().get("target_meta").map(String::as_str),
- Some("1")
- );
-
- Ok(())
- }
-
- #[test]
- fn field_aware_cast_nullable_prefers_child_nullability() -> Result<()> {
- // When the child expression is nullable the cast must be treated as
- // nullable even if the explicitly supplied target field is marked
- // non-nullable. return_field() still reflects the target metadata.
- let schema = Schema::new(vec![Field::new("a", Int32, true)]);
- let expr = CastExpr::new_with_target_field(
- col("a", &schema)?,
- Arc::new(Field::new("cast_target", Int64, false)),
- None,
- );
+ fn field_aware_cast_preserves_target_field_semantics() -> Result<()> {
+ let metadata = HashMap::from([("target_meta".to_string(), "1".to_string())]);
- assert!(expr.nullable(&schema)?);
- assert!(!expr.return_field(&schema)?.is_nullable());
+ for (child_nullable, target_nullable) in [(true, false), (false, true)] {
+ let schema = Schema::new(vec![Field::new("a", Int32, child_nullable)]);
+ let expr = CastExpr::new_with_target_field(
+ col("a", &schema)?,
+ Arc::new(
+ Field::new("cast_target", Int64, target_nullable)
+ .with_metadata(metadata.clone()),
+ ),
+ None,
+ );
+
+ let field = expr.return_field(&schema)?;
+ assert_eq!(field.name(), "cast_target");
+ assert_eq!(field.data_type(), &Int64);
+ assert_eq!(field.is_nullable(), target_nullable);
+ assert_eq!(
+ field.metadata().get("target_meta").map(String::as_str),
+ Some("1")
+ );
+ assert_eq!(expr.nullable(&schema)?, child_nullable || target_nullable);
+ }
Ok(())
}
@@ -959,23 +976,6 @@ mod tests {
Ok(())
}
- #[test]
- fn field_aware_cast_nullable_child_nonnullable_targets_nullable() -> Result<()> {
- // child is non-nullable but the target field is marked nullable; the
- // nullable() result should still be true because the field allows nulls.
- let schema = Schema::new(vec![Field::new("a", Int32, false)]);
- let expr = CastExpr::new_with_target_field(
- col("a", &schema)?,
- Arc::new(Field::new("cast_target", Int64, true)),
- None,
- );
-
- assert!(expr.nullable(&schema)?);
- assert!(expr.return_field(&schema)?.is_nullable());
-
- Ok(())
- }
-
#[test]
fn struct_cast_validation_uses_nested_target_fields() -> Result<()> {
let source_type = Struct(Fields::from(vec![
@@ -1002,6 +1002,130 @@ mod tests {
Ok(())
}
+ #[test]
+ fn field_aware_cast_struct_array_missing_child() -> Result<()> {
+ let source_a = Field::new("a", Int32, true);
+ let source_b = Field::new("b", Utf8, true);
+ let target_field = Field::new(
+ "s",
+ Struct(
+ vec![
+ Arc::new(Field::new("a", Int64, true)),
+ Arc::new(Field::new("c", Utf8, true)),
+ ]
+ .into(),
+ ),
+ true,
+ );
+
+ let struct_array = cast_struct_array(
+ "s",
+ Field::new(
+ "s",
+ Struct(
+ vec![Arc::new(source_a.clone()), Arc::new(source_b.clone())].into(),
+ ),
+ true,
+ ),
+ target_field,
+ make_struct_array(
+ vec![Arc::new(source_a), Arc::new(source_b)].into(),
+ vec![
+ Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef,
+ Arc::new(StringArray::from(vec![Some("alpha"), Some("beta")]))
+ as ArrayRef,
+ ],
+ ),
+ )?;
+ let cast_a = as_int64_array(struct_array.column_by_name("a").unwrap().as_ref())?;
+ assert_eq!(cast_a.value(0), 1);
+ assert!(cast_a.is_null(1));
+
+ let cast_c = as_string_array(struct_array.column_by_name("c").unwrap().as_ref())?;
+ assert!(cast_c.is_null(0));
+ assert!(cast_c.is_null(1));
+ Ok(())
+ }
+
+ #[test]
+ fn field_aware_cast_nested_struct_array() -> Result<()> {
+ let inner_source = Field::new(
+ "inner",
+ Struct(vec![Arc::new(Field::new("x", Int32, true))].into()),
+ true,
+ );
+ let inner_target = Field::new(
+ "inner",
+ Struct(
+ vec![
+ Arc::new(Field::new("x", Int64, true)),
+ Arc::new(Field::new("y", Boolean, true)),
+ ]
+ .into(),
+ ),
+ true,
+ );
+ let target_field =
+ Field::new("root", Struct(vec![Arc::new(inner_target)].into()), true);
+
+ let inner_struct = make_struct_array(
+ vec![Arc::new(Field::new("x", Int32, true))].into(),
+ vec![Arc::new(Int32Array::from(vec![Some(7), None])) as ArrayRef],
+ );
+ let outer_struct = make_struct_array(
+ vec![Arc::new(inner_source.clone())].into(),
+ vec![Arc::new(inner_struct) as ArrayRef],
+ );
+ let struct_array = cast_struct_array(
+ "root",
+ Field::new("root", Struct(vec![Arc::new(inner_source)].into()), true),
+ target_field,
+ outer_struct,
+ )?;
+ let inner =
+ as_struct_array(struct_array.column_by_name("inner").unwrap().as_ref())?;
+ let x = as_int64_array(inner.column_by_name("x").unwrap().as_ref())?;
+ assert_eq!(x.value(0), 7);
+ assert!(x.is_null(1));
+ let y = as_boolean_array(inner.column_by_name("y").unwrap().as_ref())?;
+ assert!(y.is_null(0));
+ assert!(y.is_null(1));
+ Ok(())
+ }
+
+ #[test]
+ fn field_aware_cast_struct_scalar() -> Result<()> {
+ let source_field = Field::new("a", Int32, true);
+ let target_field = Field::new(
+ "s",
+ Struct(vec![Arc::new(Field::new("a", UInt8, true))].into()),
+ true,
+ );
+
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "s",
+ Struct(vec![Arc::new(source_field.clone())].into()),
+ true,
+ )]));
+ let scalar_struct = make_struct_array(
+ vec![Arc::new(source_field)].into(),
+ vec![Arc::new(Int32Array::from(vec![Some(9)])) as ArrayRef],
+ );
+ let literal = Arc::new(crate::expressions::Literal::new(ScalarValue::Struct(
+ Arc::new(scalar_struct),
+ )));
+ let expr = CastExpr::new_with_target_field(literal, Arc::new(target_field), None);
+
+ let batch = RecordBatch::new_empty(schema);
+ let result = expr.evaluate(&batch)?;
+ let ColumnarValue::Scalar(ScalarValue::Struct(array)) = result else {
+ panic!("expected struct scalar");
+ };
+ let casted = as_uint8_array(array.column_by_name("a").unwrap().as_ref())?;
+ assert_eq!(casted.value(0), 9);
+ Ok(())
+ }
+
#[test]
#[ignore] // TODO: https://github.com/apache/datafusion/issues/5396
fn test_cast_decimal() -> Result<()> {
diff --git a/datafusion/physical-expr/src/expressions/cast_column.rs b/datafusion/physical-expr/src/expressions/cast_column.rs
deleted file mode 100644
index 158d467c08..0000000000
--- a/datafusion/physical-expr/src/expressions/cast_column.rs
+++ /dev/null
@@ -1,420 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Physical expression for struct-aware casting of columns.
-
-use super::cast::cast_expr_properties;
-use crate::physical_expr::PhysicalExpr;
-use arrow::{
- compute::CastOptions,
- datatypes::{DataType, FieldRef, Schema},
- record_batch::RecordBatch,
-};
-use datafusion_common::{
- Result, ScalarValue, format::DEFAULT_CAST_OPTIONS, nested_struct::cast_column,
-};
-use datafusion_expr_common::columnar_value::ColumnarValue;
-use datafusion_expr_common::sort_properties::ExprProperties;
-use std::{
- any::Any,
- fmt::{self, Display},
- hash::Hash,
- sync::Arc,
-};
-/// A physical expression that applies [`cast_column`] to its input.
-///
-/// [`CastColumnExpr`] extends the regular [`CastExpr`](super::CastExpr) by
-/// retaining schema metadata for both the input and output fields. This allows
-/// the evaluator to perform struct-aware casts that honour nested field
-/// ordering, preserve nullability, and fill missing fields with null values.
-///
-/// This expression is intended for schema rewriting scenarios where the
-/// planner already resolved the input column but needs to adapt its physical
-/// representation to a new [`arrow::datatypes::Field`]. It mirrors the behaviour of the
-/// [`datafusion_common::nested_struct::cast_column`] helper while integrating
-/// with the `PhysicalExpr` trait so it can participate in the execution plan
-/// like any other column expression.
-#[derive(Debug, Clone, Eq)]
-pub struct CastColumnExpr {
- /// The physical expression producing the value to cast.
- expr: Arc<dyn PhysicalExpr>,
- /// The logical field of the input column.
- input_field: FieldRef,
- /// The field metadata describing the desired output column.
- target_field: FieldRef,
- /// Options forwarded to [`cast_column`].
- cast_options: CastOptions<'static>,
-}
-
-// Manually derive `PartialEq`/`Hash` as `Arc<dyn PhysicalExpr>` does not
-// implement these traits by default for the trait object.
-impl PartialEq for CastColumnExpr {
- fn eq(&self, other: &Self) -> bool {
- self.expr.eq(&other.expr)
- && self.input_field.eq(&other.input_field)
- && self.target_field.eq(&other.target_field)
- && self.cast_options.eq(&other.cast_options)
- }
-}
-
-impl Hash for CastColumnExpr {
- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
- self.expr.hash(state);
- self.input_field.hash(state);
- self.target_field.hash(state);
- self.cast_options.hash(state);
- }
-}
-
-impl CastColumnExpr {
- /// Create a new [`CastColumnExpr`].
- pub fn new(
- expr: Arc<dyn PhysicalExpr>,
- input_field: FieldRef,
- target_field: FieldRef,
- cast_options: Option<CastOptions<'static>>,
- ) -> Self {
- Self {
- expr,
- input_field,
- target_field,
- cast_options: cast_options.unwrap_or(DEFAULT_CAST_OPTIONS),
- }
- }
-
- /// The expression that produces the value to be cast.
- pub fn expr(&self) -> &Arc<dyn PhysicalExpr> {
- &self.expr
- }
-
- /// Field metadata describing the resolved input column.
- pub fn input_field(&self) -> &FieldRef {
- &self.input_field
- }
-
- /// Field metadata describing the output column after casting.
- pub fn target_field(&self) -> &FieldRef {
- &self.target_field
- }
-}
-
-impl Display for CastColumnExpr {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- write!(
- f,
- "CAST_COLUMN({} AS {})",
- self.expr,
- self.target_field.data_type()
- )
- }
-}
-
-impl PhysicalExpr for CastColumnExpr {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
- Ok(self.target_field.data_type().clone())
- }
-
- fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
- Ok(self.target_field.is_nullable())
- }
-
- fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
- let value = self.expr.evaluate(batch)?;
- match value {
- ColumnarValue::Array(array) => {
- let casted = cast_column(
- &array,
- self.target_field.data_type(),
- &self.cast_options,
- )?;
- Ok(ColumnarValue::Array(casted))
- }
- ColumnarValue::Scalar(scalar) => {
- let as_array = scalar.to_array_of_size(1)?;
- let casted = cast_column(
- &as_array,
- self.target_field.data_type(),
- &self.cast_options,
- )?;
- let result = ScalarValue::try_from_array(casted.as_ref(), 0)?;
- Ok(ColumnarValue::Scalar(result))
- }
- }
- }
-
- fn return_field(&self, _input_schema: &Schema) -> Result<FieldRef> {
- Ok(Arc::clone(&self.target_field))
- }
-
- fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
- vec![&self.expr]
- }
-
- fn with_new_children(
- self: Arc<Self>,
- mut children: Vec<Arc<dyn PhysicalExpr>>,
- ) -> Result<Arc<dyn PhysicalExpr>> {
- assert_eq!(children.len(), 1);
- let child = children.pop().expect("CastColumnExpr child");
- Ok(Arc::new(Self::new(
- child,
- Arc::clone(&self.input_field),
- Arc::clone(&self.target_field),
- Some(self.cast_options.clone()),
- )))
- }
-
- /// A [`CastColumnExpr`] preserves the ordering of its child if the cast is done
- /// under the same datatype family.
- fn get_properties(&self, children: &[ExprProperties]) -> Result<ExprProperties> {
- cast_expr_properties(&children[0], self.target_field.data_type())
- }
-
- fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- Display::fmt(self, f)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use crate::expressions::{Column, Literal};
- use arrow::{
- array::{Array, ArrayRef, BooleanArray, Int32Array, StringArray, StructArray},
- datatypes::{DataType, Field, Fields, SchemaRef},
- };
- use datafusion_common::{
- Result as DFResult, ScalarValue,
- cast::{as_int64_array, as_string_array, as_struct_array, as_uint8_array},
- };
-
- fn make_schema(field: &Field) -> SchemaRef {
- Arc::new(Schema::new(vec![field.clone()]))
- }
-
- fn make_struct_array(fields: Fields, arrays: Vec<ArrayRef>) -> StructArray {
- StructArray::new(fields, arrays, None)
- }
-
- #[test]
- fn cast_primitive_array() -> DFResult<()> {
- let input_field = Field::new("a", DataType::Int32, true);
- let target_field = Field::new("a", DataType::Int64, true);
- let schema = make_schema(&input_field);
-
- let values = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));
- let batch = RecordBatch::try_new(Arc::clone(&schema), vec![values])?;
-
- let column = Arc::new(Column::new_with_schema("a", schema.as_ref())?);
- let expr = CastColumnExpr::new(
- column,
- Arc::new(input_field.clone()),
- Arc::new(target_field.clone()),
- None,
- );
-
- let result = expr.evaluate(&batch)?;
- let ColumnarValue::Array(array) = result else {
- panic!("expected array");
- };
- let casted = as_int64_array(array.as_ref())?;
- assert_eq!(casted.value(0), 1);
- assert!(casted.is_null(1));
- assert_eq!(casted.value(2), 3);
- Ok(())
- }
-
- #[test]
- fn cast_struct_array_missing_child() -> DFResult<()> {
- let source_a = Field::new("a", DataType::Int32, true);
- let source_b = Field::new("b", DataType::Utf8, true);
- let input_field = Field::new(
- "s",
- DataType::Struct(
- vec![Arc::new(source_a.clone()), Arc::new(source_b.clone())].into(),
- ),
- true,
- );
- let target_a = Field::new("a", DataType::Int64, true);
- let target_c = Field::new("c", DataType::Utf8, true);
- let target_field = Field::new(
- "s",
- DataType::Struct(
- vec![Arc::new(target_a.clone()), Arc::new(target_c.clone())].into(),
- ),
- true,
- );
-
- let schema = make_schema(&input_field);
- let struct_array = make_struct_array(
- vec![Arc::new(source_a.clone()), Arc::new(source_b.clone())].into(),
- vec![
- Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef,
- Arc::new(StringArray::from(vec![Some("alpha"), Some("beta")]))
- as ArrayRef,
- ],
- );
- let batch = RecordBatch::try_new(
- Arc::clone(&schema),
- vec![Arc::new(struct_array) as Arc<_>],
- )?;
-
- let column = Arc::new(Column::new_with_schema("s", schema.as_ref())?);
- let expr = CastColumnExpr::new(
- column,
- Arc::new(input_field.clone()),
- Arc::new(target_field.clone()),
- None,
- );
-
- let result = expr.evaluate(&batch)?;
- let ColumnarValue::Array(array) = result else {
- panic!("expected array");
- };
- let struct_array = as_struct_array(array.as_ref())?;
- let cast_a = as_int64_array(struct_array.column_by_name("a").unwrap().as_ref())?;
- assert_eq!(cast_a.value(0), 1);
- assert!(cast_a.is_null(1));
-
- let cast_c = as_string_array(struct_array.column_by_name("c").unwrap().as_ref())?;
- assert!(cast_c.is_null(0));
- assert!(cast_c.is_null(1));
- Ok(())
- }
-
- #[test]
- fn cast_nested_struct_array() -> DFResult<()> {
- let inner_source = Field::new(
- "inner",
- DataType::Struct(
- vec![Arc::new(Field::new("x", DataType::Int32, true))].into(),
- ),
- true,
- );
- let outer_field = Field::new(
- "root",
- DataType::Struct(vec![Arc::new(inner_source.clone())].into()),
- true,
- );
-
- let inner_target = Field::new(
- "inner",
- DataType::Struct(
- vec![
- Arc::new(Field::new("x", DataType::Int64, true)),
- Arc::new(Field::new("y", DataType::Boolean, true)),
- ]
- .into(),
- ),
- true,
- );
- let target_field = Field::new(
- "root",
- DataType::Struct(vec![Arc::new(inner_target.clone())].into()),
- true,
- );
-
- let schema = make_schema(&outer_field);
-
- let inner_struct = make_struct_array(
- vec![Arc::new(Field::new("x", DataType::Int32, true))].into(),
- vec![Arc::new(Int32Array::from(vec![Some(7), None])) as ArrayRef],
- );
- let outer_struct = make_struct_array(
- vec![Arc::new(inner_source.clone())].into(),
- vec![Arc::new(inner_struct) as ArrayRef],
- );
- let batch = RecordBatch::try_new(
- Arc::clone(&schema),
- vec![Arc::new(outer_struct) as ArrayRef],
- )?;
-
- let column = Arc::new(Column::new_with_schema("root", schema.as_ref())?);
- let expr = CastColumnExpr::new(
- column,
- Arc::new(outer_field.clone()),
- Arc::new(target_field.clone()),
- None,
- );
-
- let result = expr.evaluate(&batch)?;
- let ColumnarValue::Array(array) = result else {
- panic!("expected array");
- };
- let struct_array = as_struct_array(array.as_ref())?;
- let inner =
- as_struct_array(struct_array.column_by_name("inner").unwrap().as_ref())?;
- let x = as_int64_array(inner.column_by_name("x").unwrap().as_ref())?;
- assert_eq!(x.value(0), 7);
- assert!(x.is_null(1));
- let y = inner.column_by_name("y").unwrap();
- let y = y
- .as_any()
- .downcast_ref::<BooleanArray>()
- .expect("boolean array");
- assert!(y.is_null(0));
- assert!(y.is_null(1));
- Ok(())
- }
-
- #[test]
- fn cast_struct_scalar() -> DFResult<()> {
- let source_field = Field::new("a", DataType::Int32, true);
- let input_field = Field::new(
- "s",
- DataType::Struct(vec![Arc::new(source_field.clone())].into()),
- true,
- );
- let target_field = Field::new(
- "s",
- DataType::Struct(
- vec![Arc::new(Field::new("a", DataType::UInt8, true))].into(),
- ),
- true,
- );
-
- let schema = make_schema(&input_field);
- let scalar_struct = StructArray::new(
- vec![Arc::new(source_field.clone())].into(),
- vec![Arc::new(Int32Array::from(vec![Some(9)])) as ArrayRef],
- None,
- );
- let literal =
- Arc::new(Literal::new(ScalarValue::Struct(Arc::new(scalar_struct))));
- let expr = CastColumnExpr::new(
- literal,
- Arc::new(input_field.clone()),
- Arc::new(target_field.clone()),
- None,
- );
-
- let batch = RecordBatch::new_empty(Arc::clone(&schema));
- let result = expr.evaluate(&batch)?;
- let ColumnarValue::Scalar(ScalarValue::Struct(array)) = result else {
- panic!("expected struct scalar");
- };
- let casted = array.column_by_name("a").unwrap();
- let casted = as_uint8_array(casted.as_ref())?;
- assert_eq!(casted.value(0), 9);
- Ok(())
- }
-}
diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs
index 8a27009280..1e08235548 100644
--- a/datafusion/physical-expr/src/expressions/mod.rs
+++ b/datafusion/physical-expr/src/expressions/mod.rs
@@ -21,7 +21,6 @@
mod binary;
mod case;
mod cast;
-mod cast_column;
mod column;
mod dynamic_filters;
mod in_list;
@@ -42,7 +41,6 @@ pub use crate::aggregate::stats::StatsType;
pub use binary::{BinaryExpr, binary, similar_to};
pub use case::{CaseExpr, case};
pub use cast::{CastExpr, cast};
-pub use cast_column::CastColumnExpr;
pub use column::{Column, col, with_new_schema};
pub use datafusion_expr::utils::format_state_name;
pub use dynamic_filters::DynamicFilterPhysicalExpr;
diff --git a/docs/source/library-user-guide/upgrading/54.0.0.md b/docs/source/library-user-guide/upgrading/54.0.0.md
index c34c08ec19..16606a0f40 100644
--- a/docs/source/library-user-guide/upgrading/54.0.0.md
+++ b/docs/source/library-user-guide/upgrading/54.0.0.md
@@ -76,6 +76,18 @@ adjustment:
See [#15161](https://github.com/apache/datafusion/issues/15161) and
[PR #20426](https://github.com/apache/datafusion/pull/20426) for details.
+### `CastColumnExpr` removed in favor of field-aware `CastExpr`
+
+`datafusion_physical_expr::expressions::CastColumnExpr` has been removed; use
+the field-aware `datafusion_physical_expr::expressions::CastExpr` instead.
+
+If your code downcasted to `CastColumnExpr`, downcast to `CastExpr` instead and
+use `CastExpr::target_field()` for the output field metadata and
+`CastExpr::expr()` for the input expression. To construct casts with explicit
+field semantics, use `CastExpr::new_with_target_field(...)`. The type-only
+`CastExpr::new(...)` and `cast(...)` helpers remain available for callers that
+only have a `DataType`.
+
### `comparison_coercion_numeric` removed, replaced by `comparison_coercion`
The `comparison_coercion_numeric` function has been removed. Its behavior