Skip to content

Apache DataFusion SQL Query Engine

feat: Integrate CastColumnExpr into PhysicalExprAdapter (#20269)

Kumar Ujjawal1eb520675fb4
diff --git a/datafusion/core/tests/parquet/expr_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rs
index efd492ed27..f412cdf9bd 100644
--- a/datafusion/core/tests/parquet/expr_adapter.rs
+++ b/datafusion/core/tests/parquet/expr_adapter.rs
@@ -17,8 +17,11 @@
use std::sync::Arc;
-use arrow::array::{RecordBatch, record_batch};
-use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use arrow::array::{
+ Array, ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatch, StringArray,
+ StructArray, record_batch,
+};
+use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef};
use bytes::{BufMut, BytesMut};
use datafusion::assert_batches_eq;
use datafusion::common::Result;
@@ -320,6 +323,145 @@ async fn test_physical_expr_adapter_with_non_null_defaults() {
assert_batches_eq!(expected, &batches);
}
+#[tokio::test]
+async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> {
+ use std::collections::HashMap;
+
+ // Physical struct: {id: Int32, name: Utf8}
+ let physical_struct_fields: Fields = vec![
+ Arc::new(Field::new("id", DataType::Int32, false)),
+ Arc::new(Field::new("name", DataType::Utf8, true)),
+ ]
+ .into();
+
+ let struct_array = StructArray::new(
+ physical_struct_fields.clone(),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef,
+ Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef,
+ ],
+ None,
+ );
+
+ let physical_schema = Arc::new(Schema::new(vec![Field::new(
+ "s",
+ DataType::Struct(physical_struct_fields),
+ true,
+ )]));
+
+ let batch =
+ RecordBatch::try_new(Arc::clone(&physical_schema), vec![Arc::new(struct_array)])?;
+
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ let store_url = ObjectStoreUrl::parse("memory://").unwrap();
+ write_parquet(batch, store.clone(), "struct_evolution.parquet").await;
+
+ // Logical struct: {id: Int64?, name: Utf8?, extra: Boolean?} + metadata
+ let logical_struct_fields: Fields = vec![
+ Arc::new(Field::new("id", DataType::Int64, true)),
+ Arc::new(Field::new("name", DataType::Utf8, true)),
+ Arc::new(Field::new("extra", DataType::Boolean, true).with_metadata(
+ HashMap::from([("nested_meta".to_string(), "1".to_string())]),
+ )),
+ ]
+ .into();
+
+ let table_schema = Arc::new(Schema::new(vec![
+ Field::new("s", DataType::Struct(logical_struct_fields), false)
+ .with_metadata(HashMap::from([("top_meta".to_string(), "1".to_string())])),
+ ]));
+
+ let mut cfg = SessionConfig::new()
+ .with_collect_statistics(false)
+ .with_parquet_pruning(false)
+ .with_parquet_page_index_pruning(false);
+ cfg.options_mut().execution.parquet.pushdown_filters = true;
+
+ let ctx = SessionContext::new_with_config(cfg);
+ ctx.register_object_store(store_url.as_ref(), Arc::clone(&store));
+
+ let listing_table_config =
+ ListingTableConfig::new(ListingTableUrl::parse("memory:///").unwrap())
+ .infer_options(&ctx.state())
+ .await
+ .unwrap()
+ .with_schema(table_schema.clone())
+ .with_expr_adapter_factory(Arc::new(DefaultPhysicalExprAdapterFactory));
+
+ let table = ListingTable::try_new(listing_table_config).unwrap();
+ ctx.register_table("t", Arc::new(table)).unwrap();
+
+ let batches = ctx
+ .sql("SELECT s FROM t")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ assert_eq!(batches.len(), 1);
+
+ // Verify top-level metadata propagation
+ let output_schema = batches[0].schema();
+ let s_field = output_schema.field_with_name("s").unwrap();
+ assert_eq!(
+ s_field.metadata().get("top_meta").map(String::as_str),
+ Some("1")
+ );
+
+ // Verify nested struct type/field propagation + values
+ let s_array = batches[0]
+ .column(0)
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .expect("expected struct array");
+
+ let id_array = s_array
+ .column_by_name("id")
+ .expect("id column")
+ .as_any()
+ .downcast_ref::<Int64Array>()
+ .expect("id should be cast to Int64");
+ assert_eq!(id_array.values(), &[1, 2, 3]);
+
+ let extra_array = s_array.column_by_name("extra").expect("extra column");
+ assert_eq!(extra_array.null_count(), 3);
+
+ // Verify nested field metadata propagation
+ let extra_field = match s_field.data_type() {
+ DataType::Struct(fields) => fields
+ .iter()
+ .find(|f| f.name() == "extra")
+ .expect("extra field"),
+ other => panic!("expected struct type for s, got {other:?}"),
+ };
+ assert_eq!(
+ extra_field
+ .metadata()
+ .get("nested_meta")
+ .map(String::as_str),
+ Some("1")
+ );
+
+ // Smoke test: filtering on a missing nested field evaluates correctly
+ let filtered = ctx
+ .sql("SELECT get_field(s, 'extra') AS extra FROM t WHERE get_field(s, 'extra') IS NULL")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ assert_eq!(filtered.len(), 1);
+ assert_eq!(filtered[0].num_rows(), 3);
+ let extra = filtered[0]
+ .column(0)
+ .as_any()
+ .downcast_ref::<BooleanArray>()
+ .expect("extra should be a boolean array");
+ assert_eq!(extra.null_count(), 3);
+
+ Ok(())
+}
+
/// Test demonstrating that a single PhysicalExprAdapterFactory instance can be
/// reused across multiple ListingTable instances.
///
diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
index ec5f9139ed..e1f45bb0d1 100644
--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs
+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
@@ -29,6 +29,7 @@ use arrow::compute::can_cast_types;
use arrow::datatypes::{DataType, Field, SchemaRef};
use datafusion_common::{
Result, ScalarValue, exec_err,
+ metadata::FieldMetadata,
nested_struct::validate_struct_compatibility,
tree_node::{Transformed, TransformedResult, TreeNode},
};
@@ -368,7 +369,10 @@ impl DefaultPhysicalExprAdapterRewriter {
};
let null_value = ScalarValue::Null.cast_to(logical_struct_field.data_type())?;
- Ok(Some(expressions::lit(null_value)))
+ Ok(Some(Arc::new(expressions::Literal::new_with_metadata(
+ null_value,
+ Some(FieldMetadata::from(logical_struct_field.as_ref())),
+ ))))
}
fn rewrite_column(
@@ -416,24 +420,33 @@ impl DefaultPhysicalExprAdapterRewriter {
// If the column is missing from the physical schema fill it in with nulls.
// For a different behavior, provide a custom `PhysicalExprAdapter` implementation.
let null_value = ScalarValue::Null.cast_to(logical_field.data_type())?;
- return Ok(Transformed::yes(expressions::lit(null_value)));
+ return Ok(Transformed::yes(Arc::new(
+ expressions::Literal::new_with_metadata(
+ null_value,
+ Some(FieldMetadata::from(logical_field)),
+ ),
+ )));
}
};
let physical_field = self.physical_file_schema.field(physical_column_index);
- if column.index() == physical_column_index
- && logical_field.data_type() == physical_field.data_type()
- {
+ if column.index() == physical_column_index && logical_field == physical_field {
return Ok(Transformed::no(expr));
}
let column = self.resolve_column(column, physical_column_index)?;
- if logical_field.data_type() == physical_field.data_type() {
- // If the data types match, we can use the column as is
+ if logical_field == physical_field {
+ // If the fields match (including metadata/nullability), we can use the column as is
return Ok(Transformed::yes(Arc::new(column)));
}
+ if logical_field.data_type() == physical_field.data_type() {
+ // The data type matches, but the field metadata / nullability differs.
+ // Emit a CastColumnExpr so downstream schema construction uses the logical field.
+ return self.create_cast_column_expr(column, logical_field);
+ }
+
// We need to cast the column to the logical data type
// TODO: add optimization to move the cast from the column to literal expressions in the case of `col = 123`
// since that's much cheaper to evalaute.
@@ -693,6 +706,43 @@ mod tests {
assert!(result.as_any().downcast_ref::<CastColumnExpr>().is_some());
}
+ #[test]
+ fn test_rewrite_column_with_metadata_or_nullability_mismatch() -> Result<()> {
+ use std::collections::HashMap;
+
+ let physical_schema = Schema::new(vec![Field::new("a", DataType::Int64, true)]);
+ let logical_schema =
+ Schema::new(vec![Field::new("a", DataType::Int64, false).with_metadata(
+ HashMap::from([("logical_meta".to_string(), "1".to_string())]),
+ )]);
+
+ let factory = DefaultPhysicalExprAdapterFactory;
+ let adapter = factory
+ .create(Arc::new(logical_schema), Arc::new(physical_schema.clone()))
+ .unwrap();
+
+ let result = adapter.rewrite(Arc::new(Column::new("a", 0)))?;
+ let cast = result
+ .as_any()
+ .downcast_ref::<CastColumnExpr>()
+ .expect("Expected CastColumnExpr");
+
+ assert_eq!(cast.target_field().data_type(), &DataType::Int64);
+ assert!(!cast.target_field().is_nullable());
+ assert_eq!(
+ cast.target_field()
+ .metadata()
+ .get("logical_meta")
+ .map(String::as_str),
+ Some("1")
+ );
+
+ // Ensure the expression reports the logical nullability regardless of input schema
+ assert!(!result.nullable(physical_schema.as_ref())?);
+
+ Ok(())
+ }
+
#[test]
fn test_rewrite_multi_column_expr_with_type_cast() {
let (physical_schema, logical_schema) = create_test_schema();
@@ -865,6 +915,41 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_rewrite_missing_column_propagates_metadata() -> Result<()> {
+ use std::collections::HashMap;
+
+ let physical_schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let logical_schema = Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Utf8, true).with_metadata(HashMap::from([(
+ "logical_meta".to_string(),
+ "1".to_string(),
+ )])),
+ ]);
+
+ let factory = DefaultPhysicalExprAdapterFactory;
+ let adapter = factory
+ .create(Arc::new(logical_schema), Arc::new(physical_schema.clone()))
+ .unwrap();
+
+ let result = adapter.rewrite(Arc::new(Column::new("b", 1)))?;
+ let literal = result
+ .as_any()
+ .downcast_ref::<Literal>()
+ .expect("Expected literal expression");
+
+ assert_eq!(
+ literal
+ .return_field(physical_schema.as_ref())?
+ .metadata()
+ .get("logical_meta")
+ .map(String::as_str),
+ Some("1")
+ );
+ Ok(())
+ }
+
#[test]
fn test_rewrite_missing_column_non_nullable_error() {
let physical_schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
diff --git a/datafusion/physical-expr/src/equivalence/properties/dependency.rs b/datafusion/physical-expr/src/equivalence/properties/dependency.rs
index edbf7033f4..a42a4abf80 100644
--- a/datafusion/physical-expr/src/equivalence/properties/dependency.rs
+++ b/datafusion/physical-expr/src/equivalence/properties/dependency.rs
@@ -390,7 +390,7 @@ mod tests {
convert_to_sort_reqs, create_test_params, create_test_schema, parse_sort_expr,
};
use crate::equivalence::{ProjectionMapping, convert_to_sort_exprs};
- use crate::expressions::{BinaryExpr, CastExpr, Column, col};
+ use crate::expressions::{BinaryExpr, CastColumnExpr, CastExpr, Column, col};
use crate::projection::tests::output_schema;
use crate::{ConstExpr, EquivalenceProperties, ScalarFunctionExpr};
@@ -1019,6 +1019,44 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_eliminate_redundant_monotonic_sorts_cast_column_expr() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Date32, true),
+ Field::new("b", DataType::Utf8, true),
+ Field::new("c", DataType::Timestamp(TimeUnit::Nanosecond, None), true),
+ ]));
+ let mut properties = EquivalenceProperties::new(Arc::clone(&schema));
+ properties.reorder(
+ ["a", "b", "c"]
+ .into_iter()
+ .map(|c| PhysicalSortExpr::new_default(col(c, schema.as_ref()).unwrap())),
+ )?;
+
+ let col_a = col("a", schema.as_ref())?;
+ let col_b = col("b", schema.as_ref())?;
+ let col_c = col("c", schema.as_ref())?;
+
+ let cast_c = Arc::new(CastColumnExpr::new(
+ Arc::clone(&col_c),
+ Arc::new(Field::new(
+ "c",
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ true,
+ )),
+ Arc::new(Field::new("c", DataType::Date32, true)),
+ None,
+ )) as Arc<dyn PhysicalExpr>;
+
+ properties.add_equal_conditions(cast_c, Arc::clone(&col_a))?;
+ properties.add_constants(std::iter::once(ConstExpr::from(col_b)))?;
+
+ let required = vec![PhysicalSortExpr::new_default(col("c", &schema)?)];
+ assert!(properties.ordering_satisfy(required)?);
+
+ Ok(())
+ }
+
#[test]
fn test_ordering_equivalence_with_lex_monotonic_concat() -> Result<()> {
let schema = Arc::new(Schema::new(vec![
diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs
index a98341b107..e9ed0b226f 100644
--- a/datafusion/physical-expr/src/equivalence/properties/mod.rs
+++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs
@@ -33,7 +33,7 @@ use self::dependency::{
use crate::equivalence::{
AcrossPartitions, EquivalenceGroup, OrderingEquivalenceClass, ProjectionMapping,
};
-use crate::expressions::{CastExpr, Column, Literal, with_new_schema};
+use crate::expressions::{CastColumnExpr, CastExpr, Column, Literal, with_new_schema};
use crate::{
ConstExpr, LexOrdering, LexRequirement, PhysicalExpr, PhysicalSortExpr,
PhysicalSortRequirement,
@@ -858,6 +858,18 @@ impl EquivalenceProperties {
sort_expr.options,
));
}
+ } else if let Some(cast_expr) =
+ r_expr.as_any().downcast_ref::<CastColumnExpr>()
+ {
+ let cast_type = cast_expr.target_field().data_type();
+ if cast_expr.expr().eq(&sort_expr.expr)
+ && CastExpr::check_bigger_cast(cast_type, &expr_type)
+ {
+ result.push(PhysicalSortExpr::new(
+ r_expr,
+ sort_expr.options,
+ ));
+ }
}
}
result.push(sort_expr);
diff --git a/datafusion/physical-expr/src/expressions/cast_column.rs b/datafusion/physical-expr/src/expressions/cast_column.rs
index d80b6f4a58..f6c4d080fc 100644
--- a/datafusion/physical-expr/src/expressions/cast_column.rs
+++ b/datafusion/physical-expr/src/expressions/cast_column.rs
@@ -27,6 +27,8 @@ use datafusion_common::{
Result, ScalarValue, format::DEFAULT_CAST_OPTIONS, nested_struct::cast_column,
};
use datafusion_expr_common::columnar_value::ColumnarValue;
+use datafusion_expr_common::interval_arithmetic::Interval;
+use datafusion_expr_common::sort_properties::ExprProperties;
use std::{
any::Any,
fmt::{self, Display},
@@ -177,6 +179,24 @@ impl PhysicalExpr for CastColumnExpr {
)))
}
+ /// A [`CastColumnExpr`] preserves the ordering of its child if the cast is done
+ /// under the same datatype family.
+ fn get_properties(&self, children: &[ExprProperties]) -> Result<ExprProperties> {
+ let source_datatype = children[0].range.data_type();
+ let target_type = self.target_field.data_type();
+
+ let unbounded = Interval::make_unbounded(target_type)?;
+ if (source_datatype.is_numeric() || source_datatype == DataType::Boolean)
+ && target_type.is_numeric()
+ || source_datatype.is_temporal() && target_type.is_temporal()
+ || source_datatype.eq(target_type)
+ {
+ Ok(children[0].clone().with_range(unbounded))
+ } else {
+ Ok(ExprProperties::new_unknown().with_range(unbounded))
+ }
+ }
+
fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
Display::fmt(self, f)
}
diff --git a/datafusion/sqllogictest/test_files/filter_without_sort_exec.slt b/datafusion/sqllogictest/test_files/filter_without_sort_exec.slt
index ec069212f5..633e19f791 100644
--- a/datafusion/sqllogictest/test_files/filter_without_sort_exec.slt
+++ b/datafusion/sqllogictest/test_files/filter_without_sort_exec.slt
@@ -145,3 +145,47 @@ physical_plan
02)--FilterExec: date@0 = 2006-01-02
03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
04)------StreamingTableExec: partition_sizes=1, projection=[date, ticker, time], infinite_source=true, output_ordering=[date@0 ASC NULLS LAST, ticker@1 ASC NULLS LAST, time@2 ASC NULLS LAST]
+
+# schema adaptation cast should preserve ordering (regression test for cast properties)
+statement ok
+COPY (
+ SELECT arrow_cast(column1, 'Int32') AS b
+ FROM (VALUES (1), (2), (3), (4))
+) TO 'test_files/scratch/filter_without_sort_exec/cast_ordering.parquet'
+STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE cast_ordered (
+ b BIGINT
+)
+STORED AS PARQUET
+WITH ORDER (b)
+LOCATION 'test_files/scratch/filter_without_sort_exec/';
+
+statement ok
+CREATE EXTERNAL TABLE cast_physical
+STORED AS PARQUET
+LOCATION 'test_files/scratch/filter_without_sort_exec/';
+
+query T
+SELECT DISTINCT arrow_typeof(b) FROM cast_physical;
+----
+Int32
+
+query T
+SELECT DISTINCT arrow_typeof(b) FROM cast_ordered;
+----
+Int64
+
+query TT
+EXPLAIN SELECT b FROM cast_ordered WHERE b > 1 ORDER BY b;
+----
+logical_plan
+01)Sort: cast_ordered.b ASC NULLS LAST
+02)--Filter: cast_ordered.b > Int64(1)
+03)----TableScan: cast_ordered projection=[b], partial_filters=[cast_ordered.b > Int64(1)]
+physical_plan
+01)SortPreservingMergeExec: [b@0 ASC NULLS LAST]
+02)--FilterExec: b@0 > 1
+03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/filter_without_sort_exec/cast_ordering.parquet]]}, projection=[b], output_ordering=[b@0 ASC NULLS LAST], file_type=parquet, predicate=b@0 > 1, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 1, required_guarantees=[]