Apache DataFusion SQL Query Engine
fix: preserve source field metadata in TryCast expressions (#21390)
Adrian Garcia Badaraccoa32c4b26c2e4
diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rsindex 81d788b1a2..b613db4b6a 100644--- a/datafusion/expr/src/expr_schema.rs+++ b/datafusion/expr/src/expr_schema.rs@@ -65,6 +65,24 @@ pub trait ExprSchemable {-> Result<(DataType, bool)>;}+/// Derives the output field for a cast expression from the source field.+/// For `TryCast`, `force_nullable` is `true` since a failed cast returns NULL.+fn cast_output_field(+ source_field: &FieldRef,+ target_type: &DataType,+ force_nullable: bool,+) -> Arc<Field> {+ let mut f = source_field+ .as_ref()+ .clone()+ .with_data_type(target_type.clone())+ .with_metadata(source_field.metadata().clone());+ if force_nullable {+ f = f.with_nullable(true);+ }+ Arc::new(f)+}+impl ExprSchemable for Expr {/// Returns the [arrow::datatypes::DataType] of the expression/// based on [ExprSchema]@@ -553,33 +571,25 @@ impl ExprSchemable for Expr {func.return_field_from_args(args)}// _ => Ok((self.get_type(schema)?, self.nullable(schema)?)),- Expr::Cast(Cast { expr, field }) => expr- .to_field(schema)- .map(|(_table_ref, destination_field)| {- // This propagates the nullability of the input rather than- // force the nullability of the destination field. This is- // usually the desired behaviour (i.e., specifying a cast- // destination type usually does not force a user to pick- // nullability, and assuming `true` would prevent the non-nullability- // of the parent expression to make the result eligible for- // optimizations that only apply to non-nullable values).- destination_field- .as_ref()- .clone()- .with_data_type(field.data_type().clone())- .with_metadata(destination_field.metadata().clone())+ Expr::Cast(Cast { expr, field }) => {+ expr.to_field(schema).map(|(_table_ref, src)| {+ cast_output_field(&src, field.data_type(), false)})- .map(Arc::new),+ }Expr::Placeholder(Placeholder {id: _,field: Some(field),}) => Ok(Arc::clone(field).renamed(&schema_name)),+ Expr::TryCast(TryCast { expr, field }) => {+ expr.to_field(schema).map(|(_table_ref, src)| {+ cast_output_field(&src, field.data_type(), true)+ })+ }Expr::Like(_)| Expr::SimilarTo(_)| Expr::Not(_)| Expr::Between(_)| Expr::Case(_)- | Expr::TryCast(_)| Expr::InList(_)| Expr::InSubquery(_)| Expr::SetComparison(_)diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.sltindex f3836b23ec..11ed4fc632 100644--- a/datafusion/sqllogictest/test_files/metadata.slt+++ b/datafusion/sqllogictest/test_files/metadata.slt@@ -220,6 +220,34 @@ FROM table_with_metadata;2020-09-082020-09-08+# Regression test: CAST should preserve source field metadata+query DT+SELECT+ CAST(ts AS DATE) as casted,+ arrow_metadata(CAST(ts AS DATE), 'metadata_key')+FROM table_with_metadata;+----+2020-09-08 ts non-nullable field+2020-09-08 ts non-nullable field+2020-09-08 ts non-nullable field++# Regression test: CAST preserves metadata on integer column+query IT+SELECT+ CAST(id AS BIGINT) as casted,+ arrow_metadata(CAST(id AS BIGINT), 'metadata_key')+FROM table_with_metadata;+----+1 the id field+NULL the id field+3 the id field++# Regression test: CAST with single-argument arrow_metadata (returns full map)+query ?+select arrow_metadata(CAST(id AS BIGINT)) from table_with_metadata limit 1;+----+{metadata_key: the id field}+# Regression test: distinct with castquery DSELECT DISTINCT (ts::DATE) AS dist@@ -347,5 +375,44 @@ select arrow_metadata(id) from table_with_metadata limit 1;----{metadata_key: the id field}+# Regression test: TRY_CAST should preserve source field metadata+query DT+SELECT+ TRY_CAST(ts AS DATE) as try_casted,+ arrow_metadata(TRY_CAST(ts AS DATE), 'metadata_key')+FROM table_with_metadata;+----+2020-09-08 ts non-nullable field+2020-09-08 ts non-nullable field+2020-09-08 ts non-nullable field++# Regression test: TRY_CAST preserves metadata on integer column+query IT+SELECT+ TRY_CAST(id AS BIGINT) as try_casted,+ arrow_metadata(TRY_CAST(id AS BIGINT), 'metadata_key')+FROM table_with_metadata;+----+1 the id field+NULL the id field+3 the id field++# Regression test: TRY_CAST preserves metadata even when cast fails (returns NULL)+query IT+SELECT+ TRY_CAST(name AS INT) as try_casted,+ arrow_metadata(TRY_CAST(name AS INT), 'metadata_key')+FROM table_with_metadata;+----+NULL the name field+NULL the name field+NULL the name field++# Regression test: TRY_CAST with single-argument arrow_metadata (returns full map)+query ?+select arrow_metadata(TRY_CAST(id AS BIGINT)) from table_with_metadata limit 1;+----+{metadata_key: the id field}+statement okdrop table table_with_metadata;