Apache DataFusion SQL Query Engine
Add `file_row_index` UDF to query file-level row indexes from Parquet files (#22604)
Adam Gutglick152d8c47eb6b
diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rsindex f19dbd6c6f..8ce359942c 100644--- a/datafusion/datasource-parquet/src/row_filter.rs+++ b/datafusion/datasource-parquet/src/row_filter.rs@@ -72,6 +72,7 @@ use arrow::array::BooleanArray;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::error::{ArrowError, Result as ArrowResult};use arrow::record_batch::RecordBatch;+use datafusion_functions::core::file_row_index::FileRowIndexFunc;use datafusion_functions::core::getfield::GetFieldFunc;use parquet::arrow::ProjectionMask;use parquet::arrow::arrow_reader::{ArrowPredicate, RowFilter};@@ -260,6 +261,9 @@ struct PushdownChecker<'schema> {non_primitive_columns: bool,/// Does the expression reference any columns not present in the file schema?projected_columns: bool,+ /// Does the expression references a ScalarUDF that requires some rewrite+ /// and therefore can't be pushed down into the row-filter.+ has_unpushable_udfs: bool,/// Indices into the file schema of columns required to evaluate the expression./// Does not include struct columns accessed via `get_field`.required_columns: Vec<usize>,@@ -276,6 +280,7 @@ impl<'schema> PushdownChecker<'schema> {Self {non_primitive_columns: false,projected_columns: false,+ has_unpushable_udfs: false,required_columns: Vec::new(),struct_field_accesses: Vec::new(),allow_list_columns,@@ -372,7 +377,7 @@ impl<'schema> PushdownChecker<'schema> {#[inline]fn prevents_pushdown(&self) -> bool {- self.non_primitive_columns || self.projected_columns+ self.non_primitive_columns || self.projected_columns || self.has_unpushable_udfs}/// Consumes the checker and returns sorted, deduplicated column indices@@ -484,6 +489,13 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {return Ok(recursion);}+ if ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())+ .is_some()+ {+ self.has_unpushable_udfs = true;+ return Ok(TreeNodeRecursion::Jump);+ }+Ok(TreeNodeRecursion::Continue)}}diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 8228cd273e..840b86dcb8 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -26,6 +26,9 @@ use crate::opener::ParquetMorselizer;use crate::opener::build_pruning_predicates;use crate::opener::build_virtual_columns_state;use crate::row_filter::can_expr_be_pushed_down_with_schemas;+use arrow_schema::Fields;+use arrow_schema::extension::ExtensionType;+use arrow_schema::{DataType, Field};use datafusion_common::config::ConfigOptions;#[cfg(feature = "parquet_encryption")]use datafusion_common::config::EncryptionFactoryOptions;@@ -40,9 +43,14 @@ use datafusion_common::config::TableParquetOptions;use datafusion_datasource::TableSchema;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;+use datafusion_functions::core::file_row_index::FileRowIndexFunc;+use datafusion_physical_expr::expressions::Column;use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr::{EquivalenceProperties, conjunction};-use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;+use datafusion_physical_expr_adapter::expr_references_scalar_udf;+use datafusion_physical_expr_adapter::{+ DefaultPhysicalExprAdapterFactory, rewrite_file_row_index_projection,+};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::physical_expr::fmt_sql;use datafusion_physical_plan::DisplayFormatType;@@ -60,6 +68,7 @@ use datafusion_execution::parquet_encryption::EncryptionFactory;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use itertools::Itertools;use object_store::ObjectStore;+use parquet::arrow::RowNumber;#[cfg(feature = "parquet_encryption")]use parquet::encryption::decrypt::FileDecryptionProperties;@@ -669,7 +678,28 @@ impl FileSource for ParquetSource {projection: &ProjectionExprs,) -> datafusion_common::Result<Option<Arc<dyn FileSource>>> {let mut source = self.clone();- source.projection = self.projection.try_merge(projection)?;++ // If there's no reference to `FileRowIndexFunc` in the projection, we can just merge+ // both projections as-is, there's no need to modify the projection first.+ if !projection.iter().any(|projection_expr| {+ expr_references_scalar_udf::<FileRowIndexFunc>(&projection_expr.expr)+ }) {+ source.projection = self.projection.try_merge(projection)?;+ return Ok(Some(Arc::new(source)));+ }++ // If we can find a reference to `FileRowIndexFunc`, we add it as a virtual column+ // or re-use an existing one in the table's schema.+ let (table_schema, row_index_col) =+ table_schema_with_row_index_col(self.table_schema());++ source.table_schema = table_schema;+ source.projection = rewrite_file_row_index_projection(+ &self.projection,+ projection,+ &row_index_col,+ )?;+Ok(Some(Arc::new(source)))}@@ -952,15 +982,12 @@ impl FileSource for ParquetSource {reversed_eq_properties.ordering_satisfy(order.iter().cloned())?;let sort_order = LexOrdering::new(order.iter().cloned());let column_in_file_schema = sort_order.as_ref().is_some_and(|s| {- s.first()- .expr- .downcast_ref::<datafusion_physical_expr::expressions::Column>()- .is_some_and(|col| {- self.table_schema- .file_schema()- .field_with_name(col.name())- .is_ok()- })+ s.first().expr.downcast_ref::<Column>().is_some_and(|col| {+ self.table_schema+ .file_schema()+ .field_with_name(col.name())+ .is_ok()+ })});if !column_in_file_schema && !reversed_satisfies {@@ -989,6 +1016,69 @@ impl FileSource for ParquetSource {}}+/// Returns the a [`TableSchema`] containing a [`RowNumber`] virtual column and a [`Column`] expression referencing its row index column.+/// The expression is then merged into a projection.+///+/// - If the schema already has a virtual column with the [`RowNumber`] type, it returns the schema unchanged.+/// - If the schema doesn't have the appropriate virtual column, it returns a modified schema with the virtual column appended to it.+fn table_schema_with_row_index_col(table_schema: &TableSchema) -> (TableSchema, Column) {+ // If we can find a virtual column with the `RowNumber` type, we just return the schema+ // and create the appropriate `column` we're going to use+ if let Some((idx, field)) =+ table_schema+ .virtual_columns()+ .iter()+ .enumerate()+ .find(|(_, field)| {+ field+ .extension_type_name()+ .is_some_and(|name| name == RowNumber::NAME)+ })+ {+ let virtual_offset = table_schema.file_schema().fields().len()+ + table_schema.table_partition_cols().len();++ return (+ table_schema.clone(),+ Column::new(field.name(), virtual_offset + idx),+ );+ }++ // The hidden field is shared across all files in this scan, but it must+ // have a unique table-schema name because later rewrites resolve it by+ // column name and index.+ let base_row_index_name = "__datafusion_file_row_index";+ let mut row_index_name = base_row_index_name.to_string();+ let mut suffix = 0;+ while table_schema+ .table_schema()+ .field_with_name(&row_index_name)+ .is_ok()+ {+ suffix += 1;+ row_index_name = format!("{base_row_index_name}_{suffix}");+ }++ let row_index_table_idx = table_schema.table_schema().fields().len();+ let row_index_field = Arc::new(+ Field::new(&row_index_name, DataType::Int64, true).with_extension_type(RowNumber),+ );+ (+ TableSchema::builder(Arc::clone(table_schema.file_schema()))+ .with_table_partition_cols(table_schema.table_partition_cols().clone())+ .with_virtual_columns(+ table_schema+ .virtual_columns()+ .iter()+ .cloned()+ .chain([row_index_field])+ .collect::<Fields>(),+ )+ .build(),+ Column::new(&row_index_name, row_index_table_idx),+ )+}+#[cfg(test)]mod tests {use super::*;@@ -1622,7 +1712,9 @@ mod tests {use datafusion_common::config::ConfigOptions;use datafusion_datasource::TableSchema;use datafusion_expr::{col, lit as logical_lit};+ use datafusion_functions::core::expr_fn::file_row_index;use datafusion_physical_expr::planner::logical2physical;+ use datafusion_physical_expr_adapter::rewrite_file_row_index_expr;use datafusion_physical_plan::filter_pushdown::PushedDown;use parquet::arrow::RowNumber;@@ -1652,13 +1744,20 @@ mod tests {.or(col("value").eq(logical_lit(4i64))),full_schema,);+ let (_, row_index_col) = table_schema_with_row_index_col(source.table_schema());+ let row_index = rewrite_file_row_index_expr(+ logical2physical(&file_row_index().gt(logical_lit(2i64)), full_schema),+ row_index_col.name(),+ row_index_col.index(),+ )+ .expect("file_row_index should rewrite to the row_number virtual column");let config = ConfigOptions::default();let prop = source- .try_pushdown_filters(vec![pushable, virtual_only, mixed], &config)+ .try_pushdown_filters(vec![pushable, virtual_only, mixed, row_index], &config).expect("try_pushdown_filters must not error");- assert_eq!(prop.filters.len(), 3);+ assert_eq!(prop.filters.len(), 4);assert!(matches!(prop.filters[0], PushedDown::Yes),"file-column filter should be pushable"@@ -1672,5 +1771,10 @@ mod tests {"filter mixing a virtual column with a file column must not be \pushed down (row filter would silently drop it)");+ assert!(+ matches!(prop.filters[3], PushedDown::No),+ "file_row_index() rewrites to a virtual column and must not be \+ pushed down"+ );}}diff --git a/datafusion/functions/src/core/file_row_index.rs b/datafusion/functions/src/core/file_row_index.rsnew file mode 100644index 0000000000..7b2667a8b8--- /dev/null+++ b/datafusion/functions/src/core/file_row_index.rs@@ -0,0 +1,96 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Implementation of the `file_row_index` scalar function.++use arrow::datatypes::DataType;+use datafusion_common::utils::take_function_args;+use datafusion_common::{Result, exec_err};+use datafusion_doc::Documentation;+use datafusion_expr::{+ ColumnarValue, ExpressionPlacement, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ Volatility,+};+use datafusion_macros::user_doc;++/// Scalar UDF implementation for `file_row_index()`.+///+/// File sources that can expose per-file row indexes rewrite this placeholder+/// function into a source-provided physical expression. Direct evaluation+/// returns an error because there is no file context outside a scan.+#[user_doc(+ doc_section(label = "Other Functions"),+ description = r#"Returns the zero-based row offset within the source file+that produced the current row.++The value is scoped to one file, so rows from different files in the same scan+can have the same row index. This function is intended to be rewritten at+file-scan time. If the input file is not known (for example, if this function+is evaluated outside a file scan, or was not pushed down into one), direct+evaluation returns an error.+"#,+ syntax_example = "file_row_index()",+ sql_example = r#"```sql+SELECT file_row_index() FROM t;+```"#+)]+#[derive(Debug, PartialEq, Eq, Hash)]+pub struct FileRowIndexFunc {+ signature: Signature,+}++impl Default for FileRowIndexFunc {+ fn default() -> Self {+ Self::new()+ }+}++impl FileRowIndexFunc {+ pub fn new() -> Self {+ Self {+ signature: Signature::nullary(Volatility::Volatile),+ }+ }+}++impl ScalarUDFImpl for FileRowIndexFunc {+ fn name(&self) -> &str {+ "file_row_index"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, args: &[DataType]) -> Result<DataType> {+ let [] = take_function_args(self.name(), args)?;+ Ok(DataType::Int64)+ }++ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ let [] = take_function_args(self.name(), args.args)?;+ exec_err!("file_row_index() is source dependent and cannot be evaluated directly")+ }++ fn placement(&self, _args: &[ExpressionPlacement]) -> ExpressionPlacement {+ ExpressionPlacement::MoveTowardsLeafNodes+ }++ fn documentation(&self) -> Option<&Documentation> {+ self.doc()+ }+}diff --git a/datafusion/functions/src/core/mod.rs b/datafusion/functions/src/core/mod.rsindex 5657f9d888..4665eca99e 100644--- a/datafusion/functions/src/core/mod.rs+++ b/datafusion/functions/src/core/mod.rs@@ -28,6 +28,7 @@ pub mod arrowtypeof;pub mod cast_to_type;pub mod coalesce;pub mod expr_ext;+pub mod file_row_index;pub mod getfield;pub mod greatest;mod greatest_least_utils;@@ -67,6 +68,7 @@ make_udf_function!(version::VersionFunc, version);make_udf_function!(arrow_metadata::ArrowMetadataFunc, arrow_metadata);make_udf_function!(with_metadata::WithMetadataFunc, with_metadata);make_udf_function!(arrow_field::ArrowFieldFunc, arrow_field);+make_udf_function!(file_row_index::FileRowIndexFunc, file_row_index);pub mod expr_fn {use datafusion_expr::{Expr, Literal};@@ -143,6 +145,9 @@ pub mod expr_fn {union_tag,"Returns the name of the currently selected field in the union",arg1+ ),(+ file_row_index,+ "Returns the offset of the row within its source file",));#[doc = "Returns the value of the field with the given name from the struct"]@@ -196,5 +201,6 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {union_tag(),version(),r#struct(),+ file_row_index(),]}diff --git a/datafusion/physical-expr-adapter/src/lib.rs b/datafusion/physical-expr-adapter/src/lib.rsindex ea4db19ee1..fa14bc8b4d 100644--- a/datafusion/physical-expr-adapter/src/lib.rs+++ b/datafusion/physical-expr-adapter/src/lib.rs@@ -29,5 +29,6 @@ pub mod schema_rewriter;pub use schema_rewriter::{BatchAdapter, BatchAdapterFactory, DefaultPhysicalExprAdapter,DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,- replace_columns_with_literals,+ expr_references_scalar_udf, replace_columns_with_literals,+ rewrite_file_row_index_expr, rewrite_file_row_index_projection,};diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rsindex 56502ab873..f287caf32e 100644--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs@@ -25,16 +25,19 @@ use std::hash::Hash;use std::sync::Arc;use arrow::array::RecordBatch;-use arrow::datatypes::{DataType, FieldRef, SchemaRef};+use arrow::datatypes::{DataType, Field, FieldRef, SchemaRef};use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err,metadata::FieldMetadata,nested_struct::validate_data_type_compatibility,- tree_node::{Transformed, TransformedResult, TreeNode},+ tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion},+};+use datafusion_expr::ScalarUDFImpl;+use datafusion_functions::core::{+ file_row_index::FileRowIndexFunc, getfield::GetFieldFunc,};-use datafusion_functions::core::getfield::GetFieldFunc;use datafusion_physical_expr::PhysicalExprSimplifier;-use datafusion_physical_expr::projection::{ProjectionExprs, Projector};+use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs, Projector};use datafusion_physical_expr::{ScalarFunctionExpr,expressions::{self, CastExpr, Column},@@ -81,6 +84,114 @@ where.data()}+/// Return true if `expr` references scalar UDF `T`.+///+/// This matches the concrete [`ScalarUDFImpl`] type rather than the function+/// name, so unrelated UDFs with the same name are not treated as matches.+pub fn expr_references_scalar_udf<T: ScalarUDFImpl>(+ expr: &Arc<dyn PhysicalExpr>,+) -> bool {+ let mut found = false;++ expr.apply(|node| {+ if ScalarFunctionExpr::try_downcast_func::<T>(node.as_ref()).is_some() {+ found = true;+ return Ok(TreeNodeRecursion::Stop);+ }+ Ok(TreeNodeRecursion::Continue)+ })+ .expect("Infallible traversal of PhysicalExpr tree failed");++ found+}++/// Rewrite occurrences of scalar UDF `T` in `expr` using `replacement`.+///+/// The rewrite matches the concrete [`ScalarUDFImpl`] type rather than the+/// function name. `replacement` is called with each matching+/// [`ScalarFunctionExpr`] after its children have been rewritten.+fn rewrite_scalar_udf<T, F>(+ expr: Arc<dyn PhysicalExpr>,+ mut replacement: F,+) -> Result<Arc<dyn PhysicalExpr>>+where+ T: ScalarUDFImpl,+ F: FnMut(&ScalarFunctionExpr) -> Result<Arc<dyn PhysicalExpr>>,+{+ expr.transform_up(|node| {+ if let Some(scalar_fn) = ScalarFunctionExpr::try_downcast_func::<T>(node.as_ref())+ {+ Ok(Transformed::yes(replacement(scalar_fn)?))+ } else {+ Ok(Transformed::no(node))+ }+ })+ .map(|transformed| transformed.data)+}++/// Rewrite `file_row_index()` in `expr` to read from a source-provided+/// row-index column.+///+/// `row_index_idx` is the index of `row_index_name` in the schema that the+/// rewritten expression will be evaluated against. The rewrite uses ordinary+/// physical expressions: a [`Column`] that reads the source row-index values+/// wrapped in a [`CastExpr`] that exposes the public `file_row_index: Int64`+/// return field without source-specific extension metadata.+pub fn rewrite_file_row_index_expr(+ expr: Arc<dyn PhysicalExpr>,+ row_index_name: &str,+ row_index_idx: usize,+) -> Result<Arc<dyn PhysicalExpr>> {+ rewrite_scalar_udf::<FileRowIndexFunc, _>(expr, |_| {+ let source = Arc::new(Column::new(row_index_name, row_index_idx));+ let target_field = Arc::new(Field::new("file_row_index", DataType::Int64, true));+ Ok(Arc::new(CastExpr::new_with_target_field(+ source,+ target_field,+ None,+ )))+ })+}++/// Rewrite `file_row_index()` in a pushed projection to read from a+/// source-provided row-index column.+///+///+/// For example if `row_index_column` is `__datafusion_row_idx` this function rewrites all+/// instances of `file_row_index()` to `__datafusion_row_index` column references.+///+/// `base_projection` is the current projection already pushed into a source.+/// The row-index source column is appended to that base projection if it is not+/// already present. `projection` is rewritten to read from the projected+/// row-index column and then merged on top of the extended base projection.+pub fn rewrite_file_row_index_projection(+ base_projection: &ProjectionExprs,+ projection: &ProjectionExprs,+ row_index_col: &Column,+) -> Result<ProjectionExprs> {+ let mut base_exprs = base_projection.as_ref().to_vec();+ let row_index_projection_idx =+ base_projection.projected_column_position(row_index_col);++ // If the column doesn't exist in the projection yet+ if row_index_projection_idx.is_none() {+ base_exprs.push(ProjectionExpr {+ expr: Arc::new(row_index_col.clone()),+ alias: row_index_col.name().to_owned(),+ });+ }++ let rewritten_projection = projection.clone().try_map_exprs(|expr| {+ rewrite_file_row_index_expr(+ expr,+ row_index_col.name(),+ row_index_projection_idx.unwrap_or(base_exprs.len() - 1),+ )+ })?;++ ProjectionExprs::new(base_exprs).try_merge(&rewritten_projection)+}+/// Trait for adapting [`PhysicalExpr`] expressions to match a target schema.////// This is used in file scans to rewrite expressions so that they can be@@ -631,8 +742,8 @@ mod tests {RecordBatchOptions, StringArray, StringViewArray, StructArray,};use arrow::datatypes::{Field, Fields, Schema};- use datafusion_common::{assert_contains, record_batch};- use datafusion_expr::Operator;+ use datafusion_common::{assert_contains, config::ConfigOptions, record_batch};+ use datafusion_expr::{Operator, ScalarUDF};use datafusion_physical_expr::expressions::{Column, Literal, col};fn assert_cast_expr(expr: &Arc<dyn PhysicalExpr>) -> &CastExpr {@@ -648,6 +759,88 @@ mod tests {assert_eq!(inner_col.index(), index);}+ fn file_row_index_expr() -> Arc<dyn PhysicalExpr> {+ Arc::new(ScalarFunctionExpr::new(+ "file_row_index",+ Arc::new(ScalarUDF::from(FileRowIndexFunc::new())),+ vec![],+ Arc::new(Field::new("file_row_index", DataType::Int64, true)),+ Arc::new(ConfigOptions::default()),+ ))+ }++ #[test]+ fn test_rewrite_scalar_udf_replaces_nested_typed_udf() -> Result<()> {+ let expr = Arc::new(expressions::BinaryExpr::new(+ file_row_index_expr(),+ Operator::Plus,+ expressions::lit(ScalarValue::Int64(Some(1))),+ )) as Arc<dyn PhysicalExpr>;++ let rewritten = rewrite_scalar_udf::<FileRowIndexFunc, _>(expr, |_| {+ Ok(expressions::lit(ScalarValue::Int64(Some(7))))+ })?;++ let binary = rewritten+ .downcast_ref::<expressions::BinaryExpr>()+ .expect("rewritten expression should remain binary");+ assert_eq!(binary.op(), &Operator::Plus);++ let left = binary+ .left()+ .downcast_ref::<Literal>()+ .expect("left side should be rewritten to a literal");+ assert_eq!(left.value(), &ScalarValue::Int64(Some(7)));++ let right = binary+ .right()+ .downcast_ref::<Literal>()+ .expect("right side should remain the original literal");+ assert_eq!(right.value(), &ScalarValue::Int64(Some(1)));+ Ok(())+ }++ #[test]+ fn test_rewrite_file_row_index_expr_to_source_column() -> Result<()> {+ let expr = rewrite_file_row_index_expr(+ file_row_index_expr(),+ "__datafusion_file_row_index",+ 2,+ )?;++ let cast_expr = expr+ .downcast_ref::<CastExpr>()+ .expect("file row index expression should be a cast");+ assert_eq!(cast_expr.cast_type(), &DataType::Int64);+ let target_field = cast_expr.target_field();+ assert_eq!(target_field.name(), "file_row_index");+ assert_eq!(target_field.data_type(), &DataType::Int64);+ assert!(target_field.is_nullable());+ assert!(target_field.metadata().is_empty());++ let source = cast_expr+ .expr()+ .downcast_ref::<Column>()+ .expect("source column");+ assert_eq!(source.name(), "__datafusion_file_row_index");+ assert_eq!(source.index(), 2);++ let input_schema = Schema::new(vec![+ Field::new("value", DataType::Int64, true),+ Field::new("__datafusion_file_row_index", DataType::Int64, false)+ .with_metadata(HashMap::from([(+ "source".to_string(),+ "virtual".to_string(),+ )])),+ ]);+ let return_field = expr.return_field(&input_schema)?;+ assert_eq!(return_field.name(), "file_row_index");+ assert_eq!(return_field.data_type(), &DataType::Int64);+ assert!(return_field.is_nullable());+ assert!(return_field.metadata().is_empty());+ Ok(())+ }+fn stale_index_cast_schemas() -> (SchemaRef, SchemaRef) {let physical_schema = Arc::new(Schema::new(vec![Field::new("b", DataType::Binary, true),diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rsindex cee95685e8..1f6a6eb08f 100644--- a/datafusion/physical-expr/src/projection.rs+++ b/datafusion/physical-expr/src/projection.rs@@ -661,7 +661,7 @@ impl ProjectionExprs {for proj_expr in self.exprs.iter() {let expr = &proj_expr.expr;let col_stats = if let Some(col) = expr.downcast_ref::<Column>() {- stats.column_statistics[col.index()].clone()+ column_statistics_at(&stats.column_statistics, col.index())} else if let Some(literal) = expr.downcast_ref::<Literal>() {// Handle literal expressions (constants) by calculating proper statisticslet data_type = expr.data_type(output_schema)?;@@ -725,6 +725,60 @@ impl ProjectionExprs {stats.column_statistics = column_statistics;Ok(stats)}++ /// Returns the output position of `column` if this projection contains it.+ ///+ /// This only matches projection expressions that are exactly [`Column`] expressions.+ /// Computed expressions, even if they reference `column`, do not match. The+ /// comparison uses [`Column`] equality, so both the name and index must match.+ /// If the same column appears more than once, this returns the first matching+ /// position.+ ///+ /// # Example+ ///+ /// ```rust+ /// use datafusion_common::ScalarValue;+ /// use datafusion_physical_expr::expressions::{Column, Literal};+ /// use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};+ /// use std::sync::Arc;+ ///+ /// let projection = ProjectionExprs::new([+ /// ProjectionExpr::new(Arc::new(Column::new("b", 1)), "b"),+ /// ProjectionExpr::new(+ /// Arc::new(Literal::new(ScalarValue::Int32(Some(42)))),+ /// "answer",+ /// ),+ /// ProjectionExpr::new(Arc::new(Column::new("a", 0)), "a"),+ /// ]);+ ///+ /// assert_eq!(+ /// projection.projected_column_position(&Column::new("b", 1)),+ /// Some(0)+ /// );+ /// assert_eq!(+ /// projection.projected_column_position(&Column::new("a", 0)),+ /// Some(2)+ /// );+ ///+ /// // The literal projection is not a Column expression.+ /// assert_eq!(+ /// projection.projected_column_position(&Column::new("answer", 1)),+ /// None+ /// );+ ///+ /// // Columns not present in the projection also return None.+ /// assert_eq!(+ /// projection.projected_column_position(&Column::new("c", 2)),+ /// None+ /// );+ /// ```+ pub fn projected_column_position(&self, column: &Column) -> Option<usize> {+ self.iter().position(|expr| {+ expr.expr+ .downcast_ref::<Column>()+ .is_some_and(|projected| projected == column)+ })+ }}/// Propagate column statistics through CAST projections. Other expressions@@ -736,7 +790,7 @@ fn project_column_statistics_through_expr(column_stats: &[ColumnStatistics],) -> ColumnStatistics {if let Some(col) = expr.downcast_ref::<Column>() {- return column_stats[col.index()].clone();+ return column_statistics_at(column_stats, col.index());}let Some(cast_expr) = expr.downcast_ref::<CastExpr>() else {return ColumnStatistics::new_unknown();@@ -760,6 +814,16 @@ fn project_column_statistics_through_expr(}}+fn column_statistics_at(+ column_stats: &[ColumnStatistics],+ index: usize,+) -> ColumnStatistics {+ column_stats+ .get(index)+ .cloned()+ .unwrap_or_else(ColumnStatistics::new_unknown)+}+impl<'a> IntoIterator for &'a ProjectionExprs {type Item = &'a ProjectionExpr;type IntoIter = std::slice::Iter<'a, ProjectionExpr>;@@ -2202,6 +2266,43 @@ pub(crate) mod tests {Schema::new(vec![field_0, field_1, field_2])}+ #[test]+ fn test_projected_column_position_returns_output_position() {+ let projection = ProjectionExprs::new([+ ProjectionExpr::new(Arc::new(Column::new("col2", 2)), "col2"),+ ProjectionExpr::new(Arc::new(Column::new("col0", 0)), "col0"),+ ]);++ assert_eq!(+ projection.projected_column_position(&Column::new("col2", 2)),+ Some(0)+ );+ assert_eq!(+ projection.projected_column_position(&Column::new("col0", 0)),+ Some(1)+ );+ }++ #[test]+ fn test_projected_column_position_returns_none_for_non_column_or_missing() {+ let projection = ProjectionExprs::new([+ ProjectionExpr::new(+ Arc::new(Literal::new(ScalarValue::Int64(Some(42)))),+ "col1",+ ),+ ProjectionExpr::new(Arc::new(Column::new("col0", 0)), "col0"),+ ]);++ assert_eq!(+ projection.projected_column_position(&Column::new("col1", 1)),+ None+ );+ assert_eq!(+ projection.projected_column_position(&Column::new("col2", 2)),+ None+ );+ }+#[test]fn test_stats_projection_columns_only() {let source = get_stats();@@ -2913,6 +3014,57 @@ pub(crate) mod tests {byte_size: Precision::Absent,});++ Ok(())+ }++ #[test]+ fn test_project_statistics_missing_column_stats_are_unknown() -> Result<()> {+ let mut input_stats = get_stats();+ let input_schema = get_schema();+ input_stats.column_statistics.truncate(2);++ // The schema has col2, but the statistics do not. This can happen for+ // source-provided virtual columns that are available at execution time+ // but not represented in file-level statistics.+ let projection = ProjectionExprs::new(vec![+ ProjectionExpr {+ expr: Arc::new(Column::new("col2", 2)),+ alias: "virtual_col".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(CastExpr::new(+ Arc::new(Column::new("col2", 2)),+ DataType::Float64,+ None,+ )),+ alias: "casted_virtual_col".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("col0", 0)),+ alias: "physical_col".to_string(),+ },+ ]);++ let output_stats = projection.project_statistics(+ input_stats,+ &projection.project_schema(&input_schema)?,+ )?;++ assert_eq!(output_stats.column_statistics.len(), 3);+ assert_eq!(+ output_stats.column_statistics[0],+ ColumnStatistics::new_unknown()+ );+ assert_eq!(+ output_stats.column_statistics[1],+ ColumnStatistics::new_unknown()+ );+ assert_eq!(+ output_stats.column_statistics[2].max_value,+ Precision::Exact(ScalarValue::Int64(Some(21)))+ );+Ok(())}diff --git a/datafusion/sqllogictest/test_files/file_row_index.slt b/datafusion/sqllogictest/test_files/file_row_index.sltnew file mode 100644index 0000000000..38822bebfd--- /dev/null+++ b/datafusion/sqllogictest/test_files/file_row_index.slt@@ -0,0 +1,171 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at++# http://www.apache.org/licenses/LICENSE-2.0++# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++statement ok+COPY (VALUES (10), (20), (30), (40), (50))+TO 'test_files/scratch/file_row_index/parquet_table/data.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE parquet_table(column1 int)+STORED AS PARQUET+LOCATION 'test_files/scratch/file_row_index/parquet_table/';++query TT+EXPLAIN SELECT file_row_index(), column1 FROM parquet_table+----+logical_plan+01)Projection: file_row_index(), parquet_table.column1+02)--TableScan: parquet_table projection=[column1]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/file_row_index/parquet_table/data.parquet]]}, projection=[CAST(__datafusion_file_row_index@1 AS Int64) as file_row_index(), column1], file_type=parquet+++query II+SELECT file_row_index(), column1 FROM parquet_table ORDER BY column1+----+0 10+1 20+2 30+3 40+4 50++query III+SELECT file_row_index(), file_row_index() + 1, column1+FROM parquet_table+ORDER BY column1+----+0 1 10+1 2 20+2 3 30+3 4 40+4 5 50+++query II+SELECT file_row_index(), column1+FROM parquet_table+WHERE file_row_index() > 2+ORDER BY column1+----+3 40+4 50++# Filter on file_row_index without having it in projection++query TT+EXPLAIN SELECT column1 FROM parquet_table WHERE file_row_index() > 2 ORDER BY column1+----+logical_plan+01)Sort: parquet_table.column1 ASC NULLS LAST+02)--Projection: parquet_table.column1+03)----Filter: __datafusion_extracted_1 > Int64(2)+04)------Projection: file_row_index() AS __datafusion_extracted_1, parquet_table.column1+05)--------TableScan: parquet_table projection=[column1]+physical_plan+01)SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--FilterExec: __datafusion_extracted_1@0 > 2, projection=[column1@1]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/file_row_index/parquet_table/data.parquet]]}, projection=[CAST(__datafusion_file_row_index@1 AS Int64) as __datafusion_extracted_1, column1], file_type=parquet++query I+SELECT column1 FROM parquet_table WHERE file_row_index() > 2 ORDER BY column1+----+40+50++# Filter on file_row_index without projecting it, while enabling filter pushdown++statement ok+SET datafusion.execution.parquet.pushdown_filters = true;++query TT+EXPLAIN SELECT column1 FROM parquet_table WHERE file_row_index() > 2 ORDER BY column1+----+logical_plan+01)Sort: parquet_table.column1 ASC NULLS LAST+02)--Projection: parquet_table.column1+03)----Filter: __datafusion_extracted_1 > Int64(2)+04)------Projection: file_row_index() AS __datafusion_extracted_1, parquet_table.column1+05)--------TableScan: parquet_table projection=[column1]+physical_plan+01)SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--FilterExec: __datafusion_extracted_1@0 > 2, projection=[column1@1]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/file_row_index/parquet_table/data.parquet]]}, projection=[CAST(__datafusion_file_row_index@1 AS Int64) as __datafusion_extracted_1, column1], file_type=parquet++query I+SELECT column1 FROM parquet_table WHERE file_row_index() > 2 ORDER BY column1+----+40+50++statement ok+RESET datafusion.execution.parquet.pushdown_filters;++# Without the rewrite in ParquetSource, `file_row_index()` errors because it+# depends on file-source context++query error file_row_index\(\) is source dependent and cannot be evaluated directly+SELECT file_row_index()++# Testing pushdown over a source that doesn't support `file_row_index()`.++statement ok+COPY (VALUES (10), (20), (30), (40), (50))+TO 'test_files/scratch/file_row_index/csv_table/data.csv'+STORED AS CSV;++statement ok+CREATE EXTERNAL TABLE csv_table(column1 int)+STORED AS CSV+LOCATION 'test_files/scratch/file_row_index/csv_table/data.csv';++query error file_row_index\(\) is source dependent and cannot be evaluated directly+SELECT *, file_row_index() FROM csv_table;++# Testing a table with two files.++statement ok+COPY (VALUES (10), (20))+TO 'test_files/scratch/file_row_index/parquet_two_files/part-1.parquet'+STORED AS PARQUET;++statement ok+COPY (VALUES (30), (40))+TO 'test_files/scratch/file_row_index/parquet_two_files/part-2.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE parquet_two_files(column1 int)+STORED AS PARQUET+LOCATION 'test_files/scratch/file_row_index/parquet_two_files/';++query II+SELECT file_row_index(), column1+FROM parquet_two_files+WHERE file_row_index() = 1+ORDER BY column1+----+1 20+1 40++statement ok+DROP TABLE parquet_two_files;++statement ok+DROP TABLE parquet_table;++statement ok+DROP TABLE csv_table;diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.mdindex e5cd6f3d99..83df7b06fd 100644--- a/docs/source/user-guide/sql/scalar_functions.md+++ b/docs/source/user-guide/sql/scalar_functions.md@@ -5702,6 +5702,7 @@ union_tag(union_expression)- [arrow_try_cast](#arrow_try_cast)- [arrow_typeof](#arrow_typeof)- [cast_to_type](#cast_to_type)+- [file_row_index](#file_row_index)- [get_field](#get_field)- [try_cast_to_type](#try_cast_to_type)- [version](#version)@@ -5885,6 +5886,27 @@ cast_to_type(expression, reference)+-----+```+### `file_row_index`++Returns the zero-based row offset within the source file+that produced the current row.++The value is scoped to one file, so rows from different files in the same scan+can have the same row index. This function is intended to be rewritten at+file-scan time. If the input file is not known (for example, if this function+is evaluated outside a file scan, or was not pushed down into one), direct+evaluation returns an error.++```sql+file_row_index()+```++#### Example++```sql+SELECT file_row_index() FROM t;+```+### `get_field`Returns a field within a map or a struct with the given key.