Apache DataFusion SQL Query Engine
fix: keep field metadata when the Parquet scan replaces columns with literals (#26856)
Yury Syrovetsky5fabdb192c77
diff --git a/datafusion/core/tests/parquet/schema.rs b/datafusion/core/tests/parquet/schema.rsindex 41925871f7..883f72ba99 100644--- a/datafusion/core/tests/parquet/schema.rs+++ b/datafusion/core/tests/parquet/schema.rs@@ -176,6 +176,40 @@ async fn schema_merge_can_preserve_metadata() {assert_metadata(&actual, &expected_metadata);}+/// Expressions pushed into the parquet scan see the field metadata of the table schema+#[tokio::test]+async fn pushed_down_projection_sees_field_metadata() {+ let tmp_dir = TempDir::new().unwrap();+ let table_dir = tmp_dir.path().join("parquet_test");++ let f1 = Field::new("id", DataType::Int32, true)+ .with_metadata(make_meta("ARROW:extension:name", "example.ext"));+ let f2 = Field::new("name", DataType::Utf8, true);+ write_files(table_dir.as_path(), vec![Schema::new(vec![f1, f2])]);+ let table_path = table_dir.to_str().unwrap().to_string();++ let ctx = SessionContext::new();+ let options = ParquetReadOptions::default().skip_metadata(false);+ ctx.register_parquet("t", &table_path, options)+ .await+ .unwrap();++ let actual = ctx+ .sql("SELECT arrow_metadata(id, 'ARROW:extension:name') AS m FROM t")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();+ assert_snapshot!(batches_to_sort_string(&actual), @r"+ +-------------++ | m |+ +-------------++ | example.ext |+ +-------------++ ");+}+/// A column that only some files carry must be inferred as nullable, because/// reading the files without it yields nulls, even when every file that has/// the column declares it required.diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rsindex 22fd185b78..e188b4543f 100644--- a/datafusion/datasource-parquet/src/opener/mod.rs+++ b/datafusion/datasource-parquet/src/opener/mod.rs@@ -29,8 +29,10 @@ use arrow::array::RecordBatch;use arrow::datatypes::DataType;use datafusion_datasource::morsel::{Morsel, MorselPlan, MorselPlanner, Morselizer};use datafusion_physical_expr::projection::ProjectionExprs;-use datafusion_physical_expr_adapter::replace_columns_with_literals;use datafusion_physical_expr_adapter::rewrite::rewrite_input_file_name_in_projection;+use datafusion_physical_expr_adapter::{+ replace_columns_with_literals, replace_columns_with_literals_keeping_metadata,+};use std::collections::{HashMap, VecDeque};use std::fmt;use std::future::Future;@@ -922,11 +924,23 @@ impl ParquetMorselizer {let mut projection = self.projection.clone();let mut predicate = self.predicate.clone();if !literal_columns.is_empty() {+ // Keep the field metadata of the replaced columns, such as extension types+ let table_schema = self.table_schema.table_schema();projection = projection.try_map_exprs(|expr| {- replace_columns_with_literals(Arc::clone(&expr), &literal_columns)+ replace_columns_with_literals_keeping_metadata(+ Arc::clone(&expr),+ &literal_columns,+ table_schema,+ )})?;predicate = predicate- .map(|p| replace_columns_with_literals(p, &literal_columns))+ .map(|p| {+ replace_columns_with_literals_keeping_metadata(+ p,+ &literal_columns,+ table_schema,+ )+ }).transpose()?;}diff --git a/datafusion/physical-expr-adapter/src/lib.rs b/datafusion/physical-expr-adapter/src/lib.rsindex d6df61ac1a..1c7ed37eae 100644--- a/datafusion/physical-expr-adapter/src/lib.rs+++ b/datafusion/physical-expr-adapter/src/lib.rs@@ -17,5 +17,5 @@ pub mod schema_rewriter;pub use schema_rewriter::{BatchAdapter, BatchAdapterFactory, DefaultPhysicalExprAdapter,DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,- replace_columns_with_literals,+ replace_columns_with_literals, replace_columns_with_literals_keeping_metadata,};diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rsindex 102ff67151..5c8e4aef69 100644--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs@@ -12,7 +12,7 @@ use std::hash::Hash;use std::sync::Arc;use arrow::array::RecordBatch;-use arrow::datatypes::{DataType, FieldRef, Fields, SchemaRef};+use arrow::datatypes::{DataType, FieldRef, Fields, Schema, SchemaRef};use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err,metadata::FieldMetadata,@@ -25,7 +25,7 @@ use datafusion_physical_expr::expressions::Literal;use datafusion_physical_expr::projection::{ProjectionExprs, Projector};use datafusion_physical_expr::{ScalarFunctionExpr,- expressions::{self, CastExpr, Column},+ expressions::{CastExpr, Column},};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use itertools::Itertools;@@ -52,6 +52,38 @@ pub fn replace_columns_with_literals<K, V>(expr: Arc<dyn PhysicalExpr>,replacements: &HashMap<K, V>,) -> Result<Arc<dyn PhysicalExpr>>+where+ K: Borrow<str> + Eq + Hash,+ V: Borrow<ScalarValue>,+{+ replace_columns_with_literals_impl(expr, replacements, None)+}++/// Replace column references in the given physical expression with literal values,+/// like [`replace_columns_with_literals`], keeping the field metadata of the replaced+/// columns.+///+/// The metadata of each literal, such as an Arrow extension type, is taken from the+/// field with the same name in `schema`. Expressions that read the field metadata of+/// their arguments, such as `arrow_metadata`, then see the same metadata as for the+/// column.+pub fn replace_columns_with_literals_keeping_metadata<K, V>(+ expr: Arc<dyn PhysicalExpr>,+ replacements: &HashMap<K, V>,+ schema: &Schema,+) -> Result<Arc<dyn PhysicalExpr>>+where+ K: Borrow<str> + Eq + Hash,+ V: Borrow<ScalarValue>,+{+ replace_columns_with_literals_impl(expr, replacements, Some(schema))+}++fn replace_columns_with_literals_impl<K, V>(+ expr: Arc<dyn PhysicalExpr>,+ replacements: &HashMap<K, V>,+ schema: Option<&Schema>,+) -> Result<Arc<dyn PhysicalExpr>>whereK: Borrow<str> + Eq + Hash,V: Borrow<ScalarValue>,@@ -60,9 +92,14 @@ whereif let Some(column) = expr.downcast_ref::<Column>()&& let Some(replacement_value) = replacements.get(column.name()){- return Ok(Transformed::yes(expressions::lit(+ let metadata = schema+ .and_then(|schema| schema.field_with_name(column.name()).ok())+ .map(FieldMetadata::from)+ .filter(|metadata| !metadata.is_empty());+ return Ok(Transformed::yes(Arc::new(Literal::new_with_metadata(replacement_value.borrow().clone(),- )));+ metadata,+ ))));}Ok(Transformed::no(expr))})@@ -924,7 +961,7 @@ mod tests {use arrow::datatypes::{Field, Fields, Schema};use datafusion_common::assert_contains;use datafusion_expr::Operator;- use datafusion_physical_expr::expressions::{Column, Literal, col};+ use datafusion_physical_expr::expressions::{self, Column, Literal, col};fn assert_cast_expr(expr: &Arc<dyn PhysicalExpr>) -> &CastExpr {expr.downcast_ref::<CastExpr>().expect("Expected CastExpr")@@ -1262,6 +1299,37 @@ mod tests {Ok(())}+ #[test]+ fn test_replace_columns_with_literals_keeping_metadata() -> Result<()> {+ let value = ScalarValue::Int64(Some(10));+ let replacements = HashMap::from([("a", &value), ("b", &value)]);+ let metadata =+ HashMap::from([("ARROW:extension:name".to_string(), "example".to_string())]);+ let schema = Schema::new(vec![+ Field::new("a", DataType::Int64, true).with_metadata(metadata.clone()),+ Field::new("b", DataType::Int64, true),+ ]);++ let literal_metadata = |name: &str, index: usize| -> Result<_> {+ let column_expr = Arc::new(Column::new(name, index)) as Arc<dyn PhysicalExpr>;+ let result = replace_columns_with_literals_keeping_metadata(+ column_expr,+ &replacements,+ &schema,+ )?;+ let literal = result+ .downcast_ref::<Literal>()+ .expect("Expected literal expression");+ assert_eq!(*literal.value(), value);+ Ok(literal.return_field(&schema)?.metadata().clone())+ };++ assert_eq!(literal_metadata("a", 0)?, metadata);+ assert!(literal_metadata("b", 1)?.is_empty());++ Ok(())+ }+#[test]fn test_replace_columns_with_literals_no_match() -> Result<()> {let value = ScalarValue::Utf8(Some("test_value".to_string()));diff --git a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.sltindex c5eebe7834..a3cd8a18a6 100644--- a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt+++ b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt@@ -84,3 +84,31 @@ FROM ();----00010203040506070809000102030506 NULL++# Extension types of Parquet columns are visible to expressions evaluated in the scan,+# also when the scan replaces a constant column with a literal from the statistics+statement ok+COPY (+ SELECT CAST(arrow_cast(X'00010203040506070809000102030506', 'FixedSizeBinary(16)') AS UUID) AS uuid_val+)+TO 'test_files/scratch/cast_extension_type_metadata/uuid.parquet'+STORED AS PARQUET;++statement ok+SET datafusion.execution.parquet.skip_metadata = false;++statement ok+CREATE EXTERNAL TABLE uuid_parquet+STORED AS PARQUET+LOCATION 'test_files/scratch/cast_extension_type_metadata/uuid.parquet';++query ?T+SELECT uuid_val, arrow_metadata(uuid_val, 'ARROW:extension:name') FROM uuid_parquet;+----+00010203040506070809000102030506 arrow.uuid++statement ok+DROP TABLE uuid_parquet;++statement ok+RESET datafusion.execution.parquet.skip_metadata;