Skip to content

Apache DataFusion SQL Query Engine

Add `file_row_index` UDF to query file-level row indexes from Parquet files (#22604)

Adam Gutglick152d8c47eb6b
diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rs
index f19dbd6c6f..8ce359942c 100644
--- a/datafusion/datasource-parquet/src/row_filter.rs
+++ b/datafusion/datasource-parquet/src/row_filter.rs
@@ -72,6 +72,7 @@ use arrow::array::BooleanArray;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::error::{ArrowError, Result as ArrowResult};
use arrow::record_batch::RecordBatch;
+use datafusion_functions::core::file_row_index::FileRowIndexFunc;
use datafusion_functions::core::getfield::GetFieldFunc;
use parquet::arrow::ProjectionMask;
use parquet::arrow::arrow_reader::{ArrowPredicate, RowFilter};
@@ -260,6 +261,9 @@ struct PushdownChecker<'schema> {
non_primitive_columns: bool,
/// Does the expression reference any columns not present in the file schema?
projected_columns: bool,
+ /// Does the expression references a ScalarUDF that requires some rewrite
+ /// and therefore can't be pushed down into the row-filter.
+ has_unpushable_udfs: bool,
/// Indices into the file schema of columns required to evaluate the expression.
/// Does not include struct columns accessed via `get_field`.
required_columns: Vec<usize>,
@@ -276,6 +280,7 @@ impl<'schema> PushdownChecker<'schema> {
Self {
non_primitive_columns: false,
projected_columns: false,
+ has_unpushable_udfs: false,
required_columns: Vec::new(),
struct_field_accesses: Vec::new(),
allow_list_columns,
@@ -372,7 +377,7 @@ impl<'schema> PushdownChecker<'schema> {
#[inline]
fn prevents_pushdown(&self) -> bool {
- self.non_primitive_columns || self.projected_columns
+ self.non_primitive_columns || self.projected_columns || self.has_unpushable_udfs
}
/// Consumes the checker and returns sorted, deduplicated column indices
@@ -484,6 +489,13 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {
return Ok(recursion);
}
+ if ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())
+ .is_some()
+ {
+ self.has_unpushable_udfs = true;
+ return Ok(TreeNodeRecursion::Jump);
+ }
+
Ok(TreeNodeRecursion::Continue)
}
}
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 8228cd273e..840b86dcb8 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -26,6 +26,9 @@ use crate::opener::ParquetMorselizer;
use crate::opener::build_pruning_predicates;
use crate::opener::build_virtual_columns_state;
use crate::row_filter::can_expr_be_pushed_down_with_schemas;
+use arrow_schema::Fields;
+use arrow_schema::extension::ExtensionType;
+use arrow_schema::{DataType, Field};
use datafusion_common::config::ConfigOptions;
#[cfg(feature = "parquet_encryption")]
use datafusion_common::config::EncryptionFactoryOptions;
@@ -40,9 +43,14 @@ use datafusion_common::config::TableParquetOptions;
use datafusion_datasource::TableSchema;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
+use datafusion_functions::core::file_row_index::FileRowIndexFunc;
+use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_expr::{EquivalenceProperties, conjunction};
-use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;
+use datafusion_physical_expr_adapter::expr_references_scalar_udf;
+use datafusion_physical_expr_adapter::{
+ DefaultPhysicalExprAdapterFactory, rewrite_file_row_index_projection,
+};
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::physical_expr::fmt_sql;
use datafusion_physical_plan::DisplayFormatType;
@@ -60,6 +68,7 @@ use datafusion_execution::parquet_encryption::EncryptionFactory;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use itertools::Itertools;
use object_store::ObjectStore;
+use parquet::arrow::RowNumber;
#[cfg(feature = "parquet_encryption")]
use parquet::encryption::decrypt::FileDecryptionProperties;
@@ -669,7 +678,28 @@ impl FileSource for ParquetSource {
projection: &ProjectionExprs,
) -> datafusion_common::Result<Option<Arc<dyn FileSource>>> {
let mut source = self.clone();
- source.projection = self.projection.try_merge(projection)?;
+
+ // If there's no reference to `FileRowIndexFunc` in the projection, we can just merge
+ // both projections as-is, there's no need to modify the projection first.
+ if !projection.iter().any(|projection_expr| {
+ expr_references_scalar_udf::<FileRowIndexFunc>(&projection_expr.expr)
+ }) {
+ source.projection = self.projection.try_merge(projection)?;
+ return Ok(Some(Arc::new(source)));
+ }
+
+ // If we can find a reference to `FileRowIndexFunc`, we add it as a virtual column
+ // or re-use an existing one in the table's schema.
+ let (table_schema, row_index_col) =
+ table_schema_with_row_index_col(self.table_schema());
+
+ source.table_schema = table_schema;
+ source.projection = rewrite_file_row_index_projection(
+ &self.projection,
+ projection,
+ &row_index_col,
+ )?;
+
Ok(Some(Arc::new(source)))
}
@@ -952,15 +982,12 @@ impl FileSource for ParquetSource {
reversed_eq_properties.ordering_satisfy(order.iter().cloned())?;
let sort_order = LexOrdering::new(order.iter().cloned());
let column_in_file_schema = sort_order.as_ref().is_some_and(|s| {
- s.first()
- .expr
- .downcast_ref::<datafusion_physical_expr::expressions::Column>()
- .is_some_and(|col| {
- self.table_schema
- .file_schema()
- .field_with_name(col.name())
- .is_ok()
- })
+ s.first().expr.downcast_ref::<Column>().is_some_and(|col| {
+ self.table_schema
+ .file_schema()
+ .field_with_name(col.name())
+ .is_ok()
+ })
});
if !column_in_file_schema && !reversed_satisfies {
@@ -989,6 +1016,69 @@ impl FileSource for ParquetSource {
}
}
+/// Returns the a [`TableSchema`] containing a [`RowNumber`] virtual column and a [`Column`] expression referencing its row index column.
+/// The expression is then merged into a projection.
+///
+/// - If the schema already has a virtual column with the [`RowNumber`] type, it returns the schema unchanged.
+/// - If the schema doesn't have the appropriate virtual column, it returns a modified schema with the virtual column appended to it.
+fn table_schema_with_row_index_col(table_schema: &TableSchema) -> (TableSchema, Column) {
+ // If we can find a virtual column with the `RowNumber` type, we just return the schema
+ // and create the appropriate `column` we're going to use
+ if let Some((idx, field)) =
+ table_schema
+ .virtual_columns()
+ .iter()
+ .enumerate()
+ .find(|(_, field)| {
+ field
+ .extension_type_name()
+ .is_some_and(|name| name == RowNumber::NAME)
+ })
+ {
+ let virtual_offset = table_schema.file_schema().fields().len()
+ + table_schema.table_partition_cols().len();
+
+ return (
+ table_schema.clone(),
+ Column::new(field.name(), virtual_offset + idx),
+ );
+ }
+
+ // The hidden field is shared across all files in this scan, but it must
+ // have a unique table-schema name because later rewrites resolve it by
+ // column name and index.
+ let base_row_index_name = "__datafusion_file_row_index";
+ let mut row_index_name = base_row_index_name.to_string();
+ let mut suffix = 0;
+ while table_schema
+ .table_schema()
+ .field_with_name(&row_index_name)
+ .is_ok()
+ {
+ suffix += 1;
+ row_index_name = format!("{base_row_index_name}_{suffix}");
+ }
+
+ let row_index_table_idx = table_schema.table_schema().fields().len();
+ let row_index_field = Arc::new(
+ Field::new(&row_index_name, DataType::Int64, true).with_extension_type(RowNumber),
+ );
+ (
+ TableSchema::builder(Arc::clone(table_schema.file_schema()))
+ .with_table_partition_cols(table_schema.table_partition_cols().clone())
+ .with_virtual_columns(
+ table_schema
+ .virtual_columns()
+ .iter()
+ .cloned()
+ .chain([row_index_field])
+ .collect::<Fields>(),
+ )
+ .build(),
+ Column::new(&row_index_name, row_index_table_idx),
+ )
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -1622,7 +1712,9 @@ mod tests {
use datafusion_common::config::ConfigOptions;
use datafusion_datasource::TableSchema;
use datafusion_expr::{col, lit as logical_lit};
+ use datafusion_functions::core::expr_fn::file_row_index;
use datafusion_physical_expr::planner::logical2physical;
+ use datafusion_physical_expr_adapter::rewrite_file_row_index_expr;
use datafusion_physical_plan::filter_pushdown::PushedDown;
use parquet::arrow::RowNumber;
@@ -1652,13 +1744,20 @@ mod tests {
.or(col("value").eq(logical_lit(4i64))),
full_schema,
);
+ let (_, row_index_col) = table_schema_with_row_index_col(source.table_schema());
+ let row_index = rewrite_file_row_index_expr(
+ logical2physical(&file_row_index().gt(logical_lit(2i64)), full_schema),
+ row_index_col.name(),
+ row_index_col.index(),
+ )
+ .expect("file_row_index should rewrite to the row_number virtual column");
let config = ConfigOptions::default();
let prop = source
- .try_pushdown_filters(vec![pushable, virtual_only, mixed], &config)
+ .try_pushdown_filters(vec![pushable, virtual_only, mixed, row_index], &config)
.expect("try_pushdown_filters must not error");
- assert_eq!(prop.filters.len(), 3);
+ assert_eq!(prop.filters.len(), 4);
assert!(
matches!(prop.filters[0], PushedDown::Yes),
"file-column filter should be pushable"
@@ -1672,5 +1771,10 @@ mod tests {
"filter mixing a virtual column with a file column must not be \
pushed down (row filter would silently drop it)"
);
+ assert!(
+ matches!(prop.filters[3], PushedDown::No),
+ "file_row_index() rewrites to a virtual column and must not be \
+ pushed down"
+ );
}
}
diff --git a/datafusion/functions/src/core/file_row_index.rs b/datafusion/functions/src/core/file_row_index.rs
new file mode 100644
index 0000000000..7b2667a8b8
--- /dev/null
+++ b/datafusion/functions/src/core/file_row_index.rs
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Implementation of the `file_row_index` scalar function.
+
+use arrow::datatypes::DataType;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, exec_err};
+use datafusion_doc::Documentation;
+use datafusion_expr::{
+ ColumnarValue, ExpressionPlacement, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Scalar UDF implementation for `file_row_index()`.
+///
+/// File sources that can expose per-file row indexes rewrite this placeholder
+/// function into a source-provided physical expression. Direct evaluation
+/// returns an error because there is no file context outside a scan.
+#[user_doc(
+ doc_section(label = "Other Functions"),
+ description = r#"Returns the zero-based row offset within the source file
+that produced the current row.
+
+The value is scoped to one file, so rows from different files in the same scan
+can have the same row index. This function is intended to be rewritten at
+file-scan time. If the input file is not known (for example, if this function
+is evaluated outside a file scan, or was not pushed down into one), direct
+evaluation returns an error.
+"#,
+ syntax_example = "file_row_index()",
+ sql_example = r#"```sql
+SELECT file_row_index() FROM t;
+```"#
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct FileRowIndexFunc {
+ signature: Signature,
+}
+
+impl Default for FileRowIndexFunc {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl FileRowIndexFunc {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::nullary(Volatility::Volatile),
+ }
+ }
+}
+
+impl ScalarUDFImpl for FileRowIndexFunc {
+ fn name(&self) -> &str {
+ "file_row_index"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, args: &[DataType]) -> Result<DataType> {
+ let [] = take_function_args(self.name(), args)?;
+ Ok(DataType::Int64)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ let [] = take_function_args(self.name(), args.args)?;
+ exec_err!("file_row_index() is source dependent and cannot be evaluated directly")
+ }
+
+ fn placement(&self, _args: &[ExpressionPlacement]) -> ExpressionPlacement {
+ ExpressionPlacement::MoveTowardsLeafNodes
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
diff --git a/datafusion/functions/src/core/mod.rs b/datafusion/functions/src/core/mod.rs
index 5657f9d888..4665eca99e 100644
--- a/datafusion/functions/src/core/mod.rs
+++ b/datafusion/functions/src/core/mod.rs
@@ -28,6 +28,7 @@ pub mod arrowtypeof;
pub mod cast_to_type;
pub mod coalesce;
pub mod expr_ext;
+pub mod file_row_index;
pub mod getfield;
pub mod greatest;
mod greatest_least_utils;
@@ -67,6 +68,7 @@ make_udf_function!(version::VersionFunc, version);
make_udf_function!(arrow_metadata::ArrowMetadataFunc, arrow_metadata);
make_udf_function!(with_metadata::WithMetadataFunc, with_metadata);
make_udf_function!(arrow_field::ArrowFieldFunc, arrow_field);
+make_udf_function!(file_row_index::FileRowIndexFunc, file_row_index);
pub mod expr_fn {
use datafusion_expr::{Expr, Literal};
@@ -143,6 +145,9 @@ pub mod expr_fn {
union_tag,
"Returns the name of the currently selected field in the union",
arg1
+ ),(
+ file_row_index,
+ "Returns the offset of the row within its source file",
));
#[doc = "Returns the value of the field with the given name from the struct"]
@@ -196,5 +201,6 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
union_tag(),
version(),
r#struct(),
+ file_row_index(),
]
}
diff --git a/datafusion/physical-expr-adapter/src/lib.rs b/datafusion/physical-expr-adapter/src/lib.rs
index ea4db19ee1..fa14bc8b4d 100644
--- a/datafusion/physical-expr-adapter/src/lib.rs
+++ b/datafusion/physical-expr-adapter/src/lib.rs
@@ -29,5 +29,6 @@ pub mod schema_rewriter;
pub use schema_rewriter::{
BatchAdapter, BatchAdapterFactory, DefaultPhysicalExprAdapter,
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
- replace_columns_with_literals,
+ expr_references_scalar_udf, replace_columns_with_literals,
+ rewrite_file_row_index_expr, rewrite_file_row_index_projection,
};
diff --git a/datafusion/physical-expr-adapter/src/schema_rewriter.rs b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
index 56502ab873..f287caf32e 100644
--- a/datafusion/physical-expr-adapter/src/schema_rewriter.rs
+++ b/datafusion/physical-expr-adapter/src/schema_rewriter.rs
@@ -25,16 +25,19 @@ use std::hash::Hash;
use std::sync::Arc;
use arrow::array::RecordBatch;
-use arrow::datatypes::{DataType, FieldRef, SchemaRef};
+use arrow::datatypes::{DataType, Field, FieldRef, SchemaRef};
use datafusion_common::{
DataFusionError, Result, ScalarValue, exec_err,
metadata::FieldMetadata,
nested_struct::validate_data_type_compatibility,
- tree_node::{Transformed, TransformedResult, TreeNode},
+ tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion},
+};
+use datafusion_expr::ScalarUDFImpl;
+use datafusion_functions::core::{
+ file_row_index::FileRowIndexFunc, getfield::GetFieldFunc,
};
-use datafusion_functions::core::getfield::GetFieldFunc;
use datafusion_physical_expr::PhysicalExprSimplifier;
-use datafusion_physical_expr::projection::{ProjectionExprs, Projector};
+use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs, Projector};
use datafusion_physical_expr::{
ScalarFunctionExpr,
expressions::{self, CastExpr, Column},
@@ -81,6 +84,114 @@ where
.data()
}
+/// Return true if `expr` references scalar UDF `T`.
+///
+/// This matches the concrete [`ScalarUDFImpl`] type rather than the function
+/// name, so unrelated UDFs with the same name are not treated as matches.
+pub fn expr_references_scalar_udf<T: ScalarUDFImpl>(
+ expr: &Arc<dyn PhysicalExpr>,
+) -> bool {
+ let mut found = false;
+
+ expr.apply(|node| {
+ if ScalarFunctionExpr::try_downcast_func::<T>(node.as_ref()).is_some() {
+ found = true;
+ return Ok(TreeNodeRecursion::Stop);
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })
+ .expect("Infallible traversal of PhysicalExpr tree failed");
+
+ found
+}
+
+/// Rewrite occurrences of scalar UDF `T` in `expr` using `replacement`.
+///
+/// The rewrite matches the concrete [`ScalarUDFImpl`] type rather than the
+/// function name. `replacement` is called with each matching
+/// [`ScalarFunctionExpr`] after its children have been rewritten.
+fn rewrite_scalar_udf<T, F>(
+ expr: Arc<dyn PhysicalExpr>,
+ mut replacement: F,
+) -> Result<Arc<dyn PhysicalExpr>>
+where
+ T: ScalarUDFImpl,
+ F: FnMut(&ScalarFunctionExpr) -> Result<Arc<dyn PhysicalExpr>>,
+{
+ expr.transform_up(|node| {
+ if let Some(scalar_fn) = ScalarFunctionExpr::try_downcast_func::<T>(node.as_ref())
+ {
+ Ok(Transformed::yes(replacement(scalar_fn)?))
+ } else {
+ Ok(Transformed::no(node))
+ }
+ })
+ .map(|transformed| transformed.data)
+}
+
+/// Rewrite `file_row_index()` in `expr` to read from a source-provided
+/// row-index column.
+///
+/// `row_index_idx` is the index of `row_index_name` in the schema that the
+/// rewritten expression will be evaluated against. The rewrite uses ordinary
+/// physical expressions: a [`Column`] that reads the source row-index values
+/// wrapped in a [`CastExpr`] that exposes the public `file_row_index: Int64`
+/// return field without source-specific extension metadata.
+pub fn rewrite_file_row_index_expr(
+ expr: Arc<dyn PhysicalExpr>,
+ row_index_name: &str,
+ row_index_idx: usize,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ rewrite_scalar_udf::<FileRowIndexFunc, _>(expr, |_| {
+ let source = Arc::new(Column::new(row_index_name, row_index_idx));
+ let target_field = Arc::new(Field::new("file_row_index", DataType::Int64, true));
+ Ok(Arc::new(CastExpr::new_with_target_field(
+ source,
+ target_field,
+ None,
+ )))
+ })
+}
+
+/// Rewrite `file_row_index()` in a pushed projection to read from a
+/// source-provided row-index column.
+///
+///
+/// For example if `row_index_column` is `__datafusion_row_idx` this function rewrites all
+/// instances of `file_row_index()` to `__datafusion_row_index` column references.
+///
+/// `base_projection` is the current projection already pushed into a source.
+/// The row-index source column is appended to that base projection if it is not
+/// already present. `projection` is rewritten to read from the projected
+/// row-index column and then merged on top of the extended base projection.
+pub fn rewrite_file_row_index_projection(
+ base_projection: &ProjectionExprs,
+ projection: &ProjectionExprs,
+ row_index_col: &Column,
+) -> Result<ProjectionExprs> {
+ let mut base_exprs = base_projection.as_ref().to_vec();
+ let row_index_projection_idx =
+ base_projection.projected_column_position(row_index_col);
+
+ // If the column doesn't exist in the projection yet
+ if row_index_projection_idx.is_none() {
+ base_exprs.push(ProjectionExpr {
+ expr: Arc::new(row_index_col.clone()),
+ alias: row_index_col.name().to_owned(),
+ });
+ }
+
+ let rewritten_projection = projection.clone().try_map_exprs(|expr| {
+ rewrite_file_row_index_expr(
+ expr,
+ row_index_col.name(),
+ row_index_projection_idx.unwrap_or(base_exprs.len() - 1),
+ )
+ })?;
+
+ ProjectionExprs::new(base_exprs).try_merge(&rewritten_projection)
+}
+
/// Trait for adapting [`PhysicalExpr`] expressions to match a target schema.
///
/// This is used in file scans to rewrite expressions so that they can be
@@ -631,8 +742,8 @@ mod tests {
RecordBatchOptions, StringArray, StringViewArray, StructArray,
};
use arrow::datatypes::{Field, Fields, Schema};
- use datafusion_common::{assert_contains, record_batch};
- use datafusion_expr::Operator;
+ use datafusion_common::{assert_contains, config::ConfigOptions, record_batch};
+ use datafusion_expr::{Operator, ScalarUDF};
use datafusion_physical_expr::expressions::{Column, Literal, col};
fn assert_cast_expr(expr: &Arc<dyn PhysicalExpr>) -> &CastExpr {
@@ -648,6 +759,88 @@ mod tests {
assert_eq!(inner_col.index(), index);
}
+ fn file_row_index_expr() -> Arc<dyn PhysicalExpr> {
+ Arc::new(ScalarFunctionExpr::new(
+ "file_row_index",
+ Arc::new(ScalarUDF::from(FileRowIndexFunc::new())),
+ vec![],
+ Arc::new(Field::new("file_row_index", DataType::Int64, true)),
+ Arc::new(ConfigOptions::default()),
+ ))
+ }
+
+ #[test]
+ fn test_rewrite_scalar_udf_replaces_nested_typed_udf() -> Result<()> {
+ let expr = Arc::new(expressions::BinaryExpr::new(
+ file_row_index_expr(),
+ Operator::Plus,
+ expressions::lit(ScalarValue::Int64(Some(1))),
+ )) as Arc<dyn PhysicalExpr>;
+
+ let rewritten = rewrite_scalar_udf::<FileRowIndexFunc, _>(expr, |_| {
+ Ok(expressions::lit(ScalarValue::Int64(Some(7))))
+ })?;
+
+ let binary = rewritten
+ .downcast_ref::<expressions::BinaryExpr>()
+ .expect("rewritten expression should remain binary");
+ assert_eq!(binary.op(), &Operator::Plus);
+
+ let left = binary
+ .left()
+ .downcast_ref::<Literal>()
+ .expect("left side should be rewritten to a literal");
+ assert_eq!(left.value(), &ScalarValue::Int64(Some(7)));
+
+ let right = binary
+ .right()
+ .downcast_ref::<Literal>()
+ .expect("right side should remain the original literal");
+ assert_eq!(right.value(), &ScalarValue::Int64(Some(1)));
+ Ok(())
+ }
+
+ #[test]
+ fn test_rewrite_file_row_index_expr_to_source_column() -> Result<()> {
+ let expr = rewrite_file_row_index_expr(
+ file_row_index_expr(),
+ "__datafusion_file_row_index",
+ 2,
+ )?;
+
+ let cast_expr = expr
+ .downcast_ref::<CastExpr>()
+ .expect("file row index expression should be a cast");
+ assert_eq!(cast_expr.cast_type(), &DataType::Int64);
+ let target_field = cast_expr.target_field();
+ assert_eq!(target_field.name(), "file_row_index");
+ assert_eq!(target_field.data_type(), &DataType::Int64);
+ assert!(target_field.is_nullable());
+ assert!(target_field.metadata().is_empty());
+
+ let source = cast_expr
+ .expr()
+ .downcast_ref::<Column>()
+ .expect("source column");
+ assert_eq!(source.name(), "__datafusion_file_row_index");
+ assert_eq!(source.index(), 2);
+
+ let input_schema = Schema::new(vec![
+ Field::new("value", DataType::Int64, true),
+ Field::new("__datafusion_file_row_index", DataType::Int64, false)
+ .with_metadata(HashMap::from([(
+ "source".to_string(),
+ "virtual".to_string(),
+ )])),
+ ]);
+ let return_field = expr.return_field(&input_schema)?;
+ assert_eq!(return_field.name(), "file_row_index");
+ assert_eq!(return_field.data_type(), &DataType::Int64);
+ assert!(return_field.is_nullable());
+ assert!(return_field.metadata().is_empty());
+ Ok(())
+ }
+
fn stale_index_cast_schemas() -> (SchemaRef, SchemaRef) {
let physical_schema = Arc::new(Schema::new(vec![
Field::new("b", DataType::Binary, true),
diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rs
index cee95685e8..1f6a6eb08f 100644
--- a/datafusion/physical-expr/src/projection.rs
+++ b/datafusion/physical-expr/src/projection.rs
@@ -661,7 +661,7 @@ impl ProjectionExprs {
for proj_expr in self.exprs.iter() {
let expr = &proj_expr.expr;
let col_stats = if let Some(col) = expr.downcast_ref::<Column>() {
- stats.column_statistics[col.index()].clone()
+ column_statistics_at(&stats.column_statistics, col.index())
} else if let Some(literal) = expr.downcast_ref::<Literal>() {
// Handle literal expressions (constants) by calculating proper statistics
let data_type = expr.data_type(output_schema)?;
@@ -725,6 +725,60 @@ impl ProjectionExprs {
stats.column_statistics = column_statistics;
Ok(stats)
}
+
+ /// Returns the output position of `column` if this projection contains it.
+ ///
+ /// This only matches projection expressions that are exactly [`Column`] expressions.
+ /// Computed expressions, even if they reference `column`, do not match. The
+ /// comparison uses [`Column`] equality, so both the name and index must match.
+ /// If the same column appears more than once, this returns the first matching
+ /// position.
+ ///
+ /// # Example
+ ///
+ /// ```rust
+ /// use datafusion_common::ScalarValue;
+ /// use datafusion_physical_expr::expressions::{Column, Literal};
+ /// use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};
+ /// use std::sync::Arc;
+ ///
+ /// let projection = ProjectionExprs::new([
+ /// ProjectionExpr::new(Arc::new(Column::new("b", 1)), "b"),
+ /// ProjectionExpr::new(
+ /// Arc::new(Literal::new(ScalarValue::Int32(Some(42)))),
+ /// "answer",
+ /// ),
+ /// ProjectionExpr::new(Arc::new(Column::new("a", 0)), "a"),
+ /// ]);
+ ///
+ /// assert_eq!(
+ /// projection.projected_column_position(&Column::new("b", 1)),
+ /// Some(0)
+ /// );
+ /// assert_eq!(
+ /// projection.projected_column_position(&Column::new("a", 0)),
+ /// Some(2)
+ /// );
+ ///
+ /// // The literal projection is not a Column expression.
+ /// assert_eq!(
+ /// projection.projected_column_position(&Column::new("answer", 1)),
+ /// None
+ /// );
+ ///
+ /// // Columns not present in the projection also return None.
+ /// assert_eq!(
+ /// projection.projected_column_position(&Column::new("c", 2)),
+ /// None
+ /// );
+ /// ```
+ pub fn projected_column_position(&self, column: &Column) -> Option<usize> {
+ self.iter().position(|expr| {
+ expr.expr
+ .downcast_ref::<Column>()
+ .is_some_and(|projected| projected == column)
+ })
+ }
}
/// Propagate column statistics through CAST projections. Other expressions
@@ -736,7 +790,7 @@ fn project_column_statistics_through_expr(
column_stats: &[ColumnStatistics],
) -> ColumnStatistics {
if let Some(col) = expr.downcast_ref::<Column>() {
- return column_stats[col.index()].clone();
+ return column_statistics_at(column_stats, col.index());
}
let Some(cast_expr) = expr.downcast_ref::<CastExpr>() else {
return ColumnStatistics::new_unknown();
@@ -760,6 +814,16 @@ fn project_column_statistics_through_expr(
}
}
+fn column_statistics_at(
+ column_stats: &[ColumnStatistics],
+ index: usize,
+) -> ColumnStatistics {
+ column_stats
+ .get(index)
+ .cloned()
+ .unwrap_or_else(ColumnStatistics::new_unknown)
+}
+
impl<'a> IntoIterator for &'a ProjectionExprs {
type Item = &'a ProjectionExpr;
type IntoIter = std::slice::Iter<'a, ProjectionExpr>;
@@ -2202,6 +2266,43 @@ pub(crate) mod tests {
Schema::new(vec![field_0, field_1, field_2])
}
+ #[test]
+ fn test_projected_column_position_returns_output_position() {
+ let projection = ProjectionExprs::new([
+ ProjectionExpr::new(Arc::new(Column::new("col2", 2)), "col2"),
+ ProjectionExpr::new(Arc::new(Column::new("col0", 0)), "col0"),
+ ]);
+
+ assert_eq!(
+ projection.projected_column_position(&Column::new("col2", 2)),
+ Some(0)
+ );
+ assert_eq!(
+ projection.projected_column_position(&Column::new("col0", 0)),
+ Some(1)
+ );
+ }
+
+ #[test]
+ fn test_projected_column_position_returns_none_for_non_column_or_missing() {
+ let projection = ProjectionExprs::new([
+ ProjectionExpr::new(
+ Arc::new(Literal::new(ScalarValue::Int64(Some(42)))),
+ "col1",
+ ),
+ ProjectionExpr::new(Arc::new(Column::new("col0", 0)), "col0"),
+ ]);
+
+ assert_eq!(
+ projection.projected_column_position(&Column::new("col1", 1)),
+ None
+ );
+ assert_eq!(
+ projection.projected_column_position(&Column::new("col2", 2)),
+ None
+ );
+ }
+
#[test]
fn test_stats_projection_columns_only() {
let source = get_stats();
@@ -2913,6 +3014,57 @@ pub(crate) mod tests {
byte_size: Precision::Absent,
}
);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_project_statistics_missing_column_stats_are_unknown() -> Result<()> {
+ let mut input_stats = get_stats();
+ let input_schema = get_schema();
+ input_stats.column_statistics.truncate(2);
+
+ // The schema has col2, but the statistics do not. This can happen for
+ // source-provided virtual columns that are available at execution time
+ // but not represented in file-level statistics.
+ let projection = ProjectionExprs::new(vec![
+ ProjectionExpr {
+ expr: Arc::new(Column::new("col2", 2)),
+ alias: "virtual_col".to_string(),
+ },
+ ProjectionExpr {
+ expr: Arc::new(CastExpr::new(
+ Arc::new(Column::new("col2", 2)),
+ DataType::Float64,
+ None,
+ )),
+ alias: "casted_virtual_col".to_string(),
+ },
+ ProjectionExpr {
+ expr: Arc::new(Column::new("col0", 0)),
+ alias: "physical_col".to_string(),
+ },
+ ]);
+
+ let output_stats = projection.project_statistics(
+ input_stats,
+ &projection.project_schema(&input_schema)?,
+ )?;
+
+ assert_eq!(output_stats.column_statistics.len(), 3);
+ assert_eq!(
+ output_stats.column_statistics[0],
+ ColumnStatistics::new_unknown()
+ );
+ assert_eq!(
+ output_stats.column_statistics[1],
+ ColumnStatistics::new_unknown()
+ );
+ assert_eq!(
+ output_stats.column_statistics[2].max_value,
+ Precision::Exact(ScalarValue::Int64(Some(21)))
+ );
+
Ok(())
}
diff --git a/datafusion/sqllogictest/test_files/file_row_index.slt b/datafusion/sqllogictest/test_files/file_row_index.slt
new file mode 100644
index 0000000000..38822bebfd
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/file_row_index.slt
@@ -0,0 +1,171 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+statement ok
+COPY (VALUES (10), (20), (30), (40), (50))
+TO 'test_files/scratch/file_row_index/parquet_table/data.parquet'
+STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE parquet_table(column1 int)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/file_row_index/parquet_table/';
+
+query TT
+EXPLAIN SELECT file_row_index(), column1 FROM parquet_table
+----
+logical_plan
+01)Projection: file_row_index(), parquet_table.column1
+02)--TableScan: parquet_table projection=[column1]
+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/file_row_index/parquet_table/data.parquet]]}, projection=[CAST(__datafusion_file_row_index@1 AS Int64) as file_row_index(), column1], file_type=parquet
+
+
+query II
+SELECT file_row_index(), column1 FROM parquet_table ORDER BY column1
+----
+0 10
+1 20
+2 30
+3 40
+4 50
+
+query III
+SELECT file_row_index(), file_row_index() + 1, column1
+FROM parquet_table
+ORDER BY column1
+----
+0 1 10
+1 2 20
+2 3 30
+3 4 40
+4 5 50
+
+
+query II
+SELECT file_row_index(), column1
+FROM parquet_table
+WHERE file_row_index() > 2
+ORDER BY column1
+----
+3 40
+4 50
+
+# Filter on file_row_index without having it in projection
+
+query TT
+EXPLAIN SELECT column1 FROM parquet_table WHERE file_row_index() > 2 ORDER BY column1
+----
+logical_plan
+01)Sort: parquet_table.column1 ASC NULLS LAST
+02)--Projection: parquet_table.column1
+03)----Filter: __datafusion_extracted_1 > Int64(2)
+04)------Projection: file_row_index() AS __datafusion_extracted_1, parquet_table.column1
+05)--------TableScan: parquet_table projection=[column1]
+physical_plan
+01)SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--FilterExec: __datafusion_extracted_1@0 > 2, projection=[column1@1]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/file_row_index/parquet_table/data.parquet]]}, projection=[CAST(__datafusion_file_row_index@1 AS Int64) as __datafusion_extracted_1, column1], file_type=parquet
+
+query I
+SELECT column1 FROM parquet_table WHERE file_row_index() > 2 ORDER BY column1
+----
+40
+50
+
+# Filter on file_row_index without projecting it, while enabling filter pushdown
+
+statement ok
+SET datafusion.execution.parquet.pushdown_filters = true;
+
+query TT
+EXPLAIN SELECT column1 FROM parquet_table WHERE file_row_index() > 2 ORDER BY column1
+----
+logical_plan
+01)Sort: parquet_table.column1 ASC NULLS LAST
+02)--Projection: parquet_table.column1
+03)----Filter: __datafusion_extracted_1 > Int64(2)
+04)------Projection: file_row_index() AS __datafusion_extracted_1, parquet_table.column1
+05)--------TableScan: parquet_table projection=[column1]
+physical_plan
+01)SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--FilterExec: __datafusion_extracted_1@0 > 2, projection=[column1@1]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/file_row_index/parquet_table/data.parquet]]}, projection=[CAST(__datafusion_file_row_index@1 AS Int64) as __datafusion_extracted_1, column1], file_type=parquet
+
+query I
+SELECT column1 FROM parquet_table WHERE file_row_index() > 2 ORDER BY column1
+----
+40
+50
+
+statement ok
+RESET datafusion.execution.parquet.pushdown_filters;
+
+# Without the rewrite in ParquetSource, `file_row_index()` errors because it
+# depends on file-source context
+
+query error file_row_index\(\) is source dependent and cannot be evaluated directly
+SELECT file_row_index()
+
+# Testing pushdown over a source that doesn't support `file_row_index()`.
+
+statement ok
+COPY (VALUES (10), (20), (30), (40), (50))
+TO 'test_files/scratch/file_row_index/csv_table/data.csv'
+STORED AS CSV;
+
+statement ok
+CREATE EXTERNAL TABLE csv_table(column1 int)
+STORED AS CSV
+LOCATION 'test_files/scratch/file_row_index/csv_table/data.csv';
+
+query error file_row_index\(\) is source dependent and cannot be evaluated directly
+SELECT *, file_row_index() FROM csv_table;
+
+# Testing a table with two files.
+
+statement ok
+COPY (VALUES (10), (20))
+TO 'test_files/scratch/file_row_index/parquet_two_files/part-1.parquet'
+STORED AS PARQUET;
+
+statement ok
+COPY (VALUES (30), (40))
+TO 'test_files/scratch/file_row_index/parquet_two_files/part-2.parquet'
+STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE parquet_two_files(column1 int)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/file_row_index/parquet_two_files/';
+
+query II
+SELECT file_row_index(), column1
+FROM parquet_two_files
+WHERE file_row_index() = 1
+ORDER BY column1
+----
+1 20
+1 40
+
+statement ok
+DROP TABLE parquet_two_files;
+
+statement ok
+DROP TABLE parquet_table;
+
+statement ok
+DROP TABLE csv_table;
diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md
index e5cd6f3d99..83df7b06fd 100644
--- a/docs/source/user-guide/sql/scalar_functions.md
+++ b/docs/source/user-guide/sql/scalar_functions.md
@@ -5702,6 +5702,7 @@ union_tag(union_expression)
- [arrow_try_cast](#arrow_try_cast)
- [arrow_typeof](#arrow_typeof)
- [cast_to_type](#cast_to_type)
+- [file_row_index](#file_row_index)
- [get_field](#get_field)
- [try_cast_to_type](#try_cast_to_type)
- [version](#version)
@@ -5885,6 +5886,27 @@ cast_to_type(expression, reference)
+-----+
```
+### `file_row_index`
+
+Returns the zero-based row offset within the source file
+that produced the current row.
+
+The value is scoped to one file, so rows from different files in the same scan
+can have the same row index. This function is intended to be rewritten at
+file-scan time. If the input file is not known (for example, if this function
+is evaluated outside a file scan, or was not pushed down into one), direct
+evaluation returns an error.
+
+```sql
+file_row_index()
+```
+
+#### Example
+
+```sql
+SELECT file_row_index() FROM t;
+```
+
### `get_field`
Returns a field within a map or a struct with the given key.