Skip to content

Apache DataFusion SQL Query Engine

fix: keep field metadata when the Parquet scan replaces columns with literals (#26856)

Yury Syrovetsky5fabdb192c77
diff --git a/datafusion/core/tests/parquet/schema.rs b/datafusion/core/tests/parquet/schema.rs
index 41925871f7..883f72ba99 100644
--- a/datafusion/core/tests/parquet/schema.rs
+++ b/datafusion/core/tests/parquet/schema.rs
@@ -176,6 +176,40 @@ async fn schema_merge_can_preserve_metadata() {
assert_metadata(&actual, &expected_metadata);
}
+/// Expressions pushed into the parquet scan see the field metadata of the table schema
+#[tokio::test]
+async fn pushed_down_projection_sees_field_metadata() {
+ let tmp_dir = TempDir::new().unwrap();
+ let table_dir = tmp_dir.path().join("parquet_test");
+
+ let f1 = Field::new("id", DataType::Int32, true)
+ .with_metadata(make_meta("ARROW:extension:name", "example.ext"));
+ let f2 = Field::new("name", DataType::Utf8, true);
+ write_files(table_dir.as_path(), vec![Schema::new(vec![f1, f2])]);
+ let table_path = table_dir.to_str().unwrap().to_string();
+
+ let ctx = SessionContext::new();
+ let options = ParquetReadOptions::default().skip_metadata(false);
+ ctx.register_parquet("t", &table_path, options)
+ .await
+ .unwrap();
+
+ let actual = ctx
+ .sql("SELECT arrow_metadata(id, 'ARROW:extension:name') AS m FROM t")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ assert_snapshot!(batches_to_sort_string(&actual), @r"
+ +-------------+
+ | m |
+ +-------------+
+ | example.ext |
+ +-------------+
+ ");
+}
+
/// A column that only some files carry must be inferred as nullable, because
/// reading the files without it yields nulls, even when every file that has
/// the column declares it required.
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs
index 22fd185b78..e188b4543f 100644
--- a/datafusion/datasource-parquet/src/opener/mod.rs
+++ b/datafusion/datasource-parquet/src/opener/mod.rs
@@ -29,8 +29,10 @@ use arrow::array::RecordBatch;
use arrow::datatypes::DataType;
use datafusion_datasource::morsel::{Morsel, MorselPlan, MorselPlanner, Morselizer};
use datafusion_physical_expr::projection::ProjectionExprs;
-use datafusion_physical_expr_adapter::replace_columns_with_literals;
use datafusion_physical_expr_adapter::rewrite::rewrite_input_file_name_in_projection;
+use datafusion_physical_expr_adapter::{
+ replace_columns_with_literals, replace_columns_with_literals_keeping_metadata,
+};
use std::collections::{HashMap, VecDeque};
use std::fmt;
use std::future::Future;
@@ -922,11 +924,23 @@ impl ParquetMorselizer {
let mut projection = self.projection.clone();
let mut predicate = self.predicate.clone();
if !literal_columns.is_empty() {
+ // Keep the field metadata of the replaced columns, such as extension types
+ let table_schema = self.table_schema.table_schema();
projection = projection.try_map_exprs(|expr| {
- replace_columns_with_literals(Arc::clone(&expr), &literal_columns)
+ replace_columns_with_literals_keeping_metadata(
+ Arc::clone(&expr),
+ &literal_columns,
+ table_schema,
+ )
})?;
predicate = predicate
- .map(|p| replace_columns_with_literals(p, &literal_columns))
+ .map(|p| {
+ replace_columns_with_literals_keeping_metadata(
+ p,
+ &literal_columns,
+ table_schema,
+ )
+ })
.transpose()?;
}
diff --git a/datafusion/physical-expr-adapter/src/lib.rs b/datafusion/physical-expr-adapter/src/lib.rs
index d6df61ac1a..1c7ed37eae 100644
--- a/datafusion/physical-expr-adapter/src/lib.rs
+++ b/datafusion/physical-expr-adapter/src/lib.rs
@@ -17,5 +17,5 @@ pub mod schema_rewriter;
pub use schema_rewriter::{
BatchAdapter, BatchAdapterFactory, DefaultPhysicalExprAdapter,
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
- replace_columns_with_literals,
+ replace_columns_with_literals, replace_columns_with_literals_keeping_metadata,
};
diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
index 102ff67151..5c8e4aef69 100644
--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs
+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
@@ -12,7 +12,7 @@ use std::hash::Hash;
use std::sync::Arc;
use arrow::array::RecordBatch;
-use arrow::datatypes::{DataType, FieldRef, Fields, SchemaRef};
+use arrow::datatypes::{DataType, FieldRef, Fields, Schema, SchemaRef};
use datafusion_common::{
DataFusionError, Result, ScalarValue, exec_err,
metadata::FieldMetadata,
@@ -25,7 +25,7 @@ use datafusion_physical_expr::expressions::Literal;
use datafusion_physical_expr::projection::{ProjectionExprs, Projector};
use datafusion_physical_expr::{
ScalarFunctionExpr,
- expressions::{self, CastExpr, Column},
+ expressions::{CastExpr, Column},
};
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use itertools::Itertools;
@@ -52,6 +52,38 @@ pub fn replace_columns_with_literals<K, V>(
expr: Arc<dyn PhysicalExpr>,
replacements: &HashMap<K, V>,
) -> Result<Arc<dyn PhysicalExpr>>
+where
+ K: Borrow<str> + Eq + Hash,
+ V: Borrow<ScalarValue>,
+{
+ replace_columns_with_literals_impl(expr, replacements, None)
+}
+
+/// Replace column references in the given physical expression with literal values,
+/// like [`replace_columns_with_literals`], keeping the field metadata of the replaced
+/// columns.
+///
+/// The metadata of each literal, such as an Arrow extension type, is taken from the
+/// field with the same name in `schema`. Expressions that read the field metadata of
+/// their arguments, such as `arrow_metadata`, then see the same metadata as for the
+/// column.
+pub fn replace_columns_with_literals_keeping_metadata<K, V>(
+ expr: Arc<dyn PhysicalExpr>,
+ replacements: &HashMap<K, V>,
+ schema: &Schema,
+) -> Result<Arc<dyn PhysicalExpr>>
+where
+ K: Borrow<str> + Eq + Hash,
+ V: Borrow<ScalarValue>,
+{
+ replace_columns_with_literals_impl(expr, replacements, Some(schema))
+}
+
+fn replace_columns_with_literals_impl<K, V>(
+ expr: Arc<dyn PhysicalExpr>,
+ replacements: &HashMap<K, V>,
+ schema: Option<&Schema>,
+) -> Result<Arc<dyn PhysicalExpr>>
where
K: Borrow<str> + Eq + Hash,
V: Borrow<ScalarValue>,
@@ -60,9 +92,14 @@ where
if let Some(column) = expr.downcast_ref::<Column>()
&& let Some(replacement_value) = replacements.get(column.name())
{
- return Ok(Transformed::yes(expressions::lit(
+ let metadata = schema
+ .and_then(|schema| schema.field_with_name(column.name()).ok())
+ .map(FieldMetadata::from)
+ .filter(|metadata| !metadata.is_empty());
+ return Ok(Transformed::yes(Arc::new(Literal::new_with_metadata(
replacement_value.borrow().clone(),
- )));
+ metadata,
+ ))));
}
Ok(Transformed::no(expr))
})
@@ -924,7 +961,7 @@ mod tests {
use arrow::datatypes::{Field, Fields, Schema};
use datafusion_common::assert_contains;
use datafusion_expr::Operator;
- use datafusion_physical_expr::expressions::{Column, Literal, col};
+ use datafusion_physical_expr::expressions::{self, Column, Literal, col};
fn assert_cast_expr(expr: &Arc<dyn PhysicalExpr>) -> &CastExpr {
expr.downcast_ref::<CastExpr>().expect("Expected CastExpr")
@@ -1262,6 +1299,37 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_replace_columns_with_literals_keeping_metadata() -> Result<()> {
+ let value = ScalarValue::Int64(Some(10));
+ let replacements = HashMap::from([("a", &value), ("b", &value)]);
+ let metadata =
+ HashMap::from([("ARROW:extension:name".to_string(), "example".to_string())]);
+ let schema = Schema::new(vec![
+ Field::new("a", DataType::Int64, true).with_metadata(metadata.clone()),
+ Field::new("b", DataType::Int64, true),
+ ]);
+
+ let literal_metadata = |name: &str, index: usize| -> Result<_> {
+ let column_expr = Arc::new(Column::new(name, index)) as Arc<dyn PhysicalExpr>;
+ let result = replace_columns_with_literals_keeping_metadata(
+ column_expr,
+ &replacements,
+ &schema,
+ )?;
+ let literal = result
+ .downcast_ref::<Literal>()
+ .expect("Expected literal expression");
+ assert_eq!(*literal.value(), value);
+ Ok(literal.return_field(&schema)?.metadata().clone())
+ };
+
+ assert_eq!(literal_metadata("a", 0)?, metadata);
+ assert!(literal_metadata("b", 1)?.is_empty());
+
+ Ok(())
+ }
+
#[test]
fn test_replace_columns_with_literals_no_match() -> Result<()> {
let value = ScalarValue::Utf8(Some("test_value".to_string()));
diff --git a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt
index c5eebe7834..a3cd8a18a6 100644
--- a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt
+++ b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt
@@ -84,3 +84,31 @@ FROM (
);
----
00010203040506070809000102030506 NULL
+
+# Extension types of Parquet columns are visible to expressions evaluated in the scan,
+# also when the scan replaces a constant column with a literal from the statistics
+statement ok
+COPY (
+ SELECT CAST(arrow_cast(X'00010203040506070809000102030506', 'FixedSizeBinary(16)') AS UUID) AS uuid_val
+)
+TO 'test_files/scratch/cast_extension_type_metadata/uuid.parquet'
+STORED AS PARQUET;
+
+statement ok
+SET datafusion.execution.parquet.skip_metadata = false;
+
+statement ok
+CREATE EXTERNAL TABLE uuid_parquet
+STORED AS PARQUET
+LOCATION 'test_files/scratch/cast_extension_type_metadata/uuid.parquet';
+
+query ?T
+SELECT uuid_val, arrow_metadata(uuid_val, 'ARROW:extension:name') FROM uuid_parquet;
+----
+00010203040506070809000102030506 arrow.uuid
+
+statement ok
+DROP TABLE uuid_parquet;
+
+statement ok
+RESET datafusion.execution.parquet.skip_metadata;