Skip to content

Apache DataFusion SQL Query Engine

fix: let leaf expressions reach the scan past a RepartitionExec (#27152)

Cheng-Yi Shih4c430b44b4c2
diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs
index 0da2f759e3..c0936eb929 100644
--- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs
@@ -2,15 +2,18 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+use std::collections::HashMap;
use std::sync::Arc;
use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::datasource::listing::PartitionedFile;
use datafusion::datasource::memory::MemorySourceConfig;
-use datafusion::datasource::physical_plan::CsvSource;
+use datafusion::datasource::physical_plan::{CsvSource, ParquetSource};
use datafusion::datasource::source::DataSourceExec;
+use datafusion::prelude::{ParquetReadOptions, SessionContext};
use datafusion_common::config::{ConfigOptions, CsvOptions};
+use datafusion_common::metadata::FieldMetadata;
use datafusion_common::{JoinSide, JoinType, NullEquality, Result, ScalarValue};
use datafusion_datasource::TableSchemaBuilder;
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
@@ -20,6 +23,7 @@ use datafusion_expr::{
Operator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
};
use datafusion_expr_common::columnar_value::ColumnarValue;
+use datafusion_functions::core::expr_fn::get_field;
use datafusion_physical_expr::expressions::{
BinaryExpr, CaseExpr, CastExpr, Column, Literal, NegativeExpr, binary, cast, col,
};
@@ -1905,3 +1909,129 @@ fn test_sort_merge_join_interleaved_projection_embeds() -> Result<()> {
Ok(())
}
+
+/// Splitting a projection rebuilds it, so it has to carry over the metadata the
+/// planner attached; otherwise the rule's schema check fails the whole query.
+/// Mirrors `keeps_metadata_of_constant_input` in `eliminate_constant_join`, but
+/// over Parquet, because only a source that absorbs the leaf projection makes
+/// the split fire at all.
+#[tokio::test]
+async fn split_keeps_field_metadata() -> Result<()> {
+ let dir = tempfile::tempdir()?;
+ let path = dir.path().join("leaf_split_metadata.parquet");
+ let path = path.to_str().expect("temp path is not utf-8");
+
+ let ctx = SessionContext::new();
+ // Below the five rows, so `EnsureRequirements` inserts the repartition that
+ // the split needs.
+ ctx.sql("SET datafusion.execution.batch_size = 2").await?;
+ ctx.sql("SET datafusion.execution.target_partitions = 4")
+ .await?;
+ ctx.sql(&format!(
+ "COPY (SELECT column1 AS id, column2 AS s FROM VALUES \
+ (1, {{value: 100}}), (2, {{value: 200}}), (3, {{value: 150}}), \
+ (4, {{value: 300}}), (5, {{value: 250}})) \
+ TO '{path}' STORED AS PARQUET"
+ ))
+ .await?
+ .collect()
+ .await?;
+ ctx.register_parquet("t", path, ParquetReadOptions::default())
+ .await?;
+
+ let metadata =
+ FieldMetadata::from(HashMap::from([("k".to_string(), "v".to_string())]));
+ let df = ctx.table("t").await?.select(vec![
+ get_field(datafusion_expr::col("s"), "value")
+ .alias_with_metadata("v", Some(metadata)),
+ (datafusion_expr::col("id") + datafusion_expr::lit(1i64)).alias("w"),
+ ])?;
+
+ // The physical plan is what the rule rewrites, so the assertion has to come
+ // from it rather than from the DataFrame's logical schema.
+ let plan = df.create_physical_plan().await?;
+ assert_eq!(
+ plan.schema().field(0).metadata().get("k"),
+ Some(&"v".to_string())
+ );
+
+ Ok(())
+}
+
+/// The split only handles round robin partitioning, because hash and range keys
+/// reference columns by index and would have to be remapped against the
+/// rewritten child schema. No SQL reaches this shape, so the test builds it:
+/// above a join, the leaf is already extracted into its own projection below the
+/// join, and nothing else puts a projection on a hash repartition.
+#[tokio::test]
+async fn split_declines_hash_repartition() -> Result<()> {
+ let schema: SchemaRef = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int64, true),
+ Field::new(
+ "s",
+ DataType::Struct(vec![Field::new("value", DataType::Int64, true)].into()),
+ true,
+ ),
+ ]));
+ let config = FileScanConfigBuilder::new(
+ ObjectStoreUrl::parse("test:///").unwrap(),
+ Arc::new(ParquetSource::new(Arc::clone(&schema))),
+ )
+ .with_file(PartitionedFile::new("x".to_string(), 100))
+ .build();
+ // Parquet absorbs a leaf projection, so the split would commit here if the
+ // partitioning check did not stop it first.
+ let hash = Arc::new(RepartitionExec::try_new(
+ DataSourceExec::from_data_source(config),
+ Partitioning::Hash(vec![col("id", &schema)?], 4),
+ )?);
+
+ let leaf: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
+ "get_field",
+ datafusion_functions::core::get_field(),
+ vec![
+ col("s", &schema)?,
+ Arc::new(Literal::new(ScalarValue::from("value"))),
+ ],
+ Arc::new(Field::new("v", DataType::Int64, true)),
+ Arc::new(ConfigOptions::default()),
+ ));
+ let projection: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(
+ vec![
+ ProjectionExpr {
+ expr: leaf,
+ alias: "v".to_string(),
+ },
+ ProjectionExpr {
+ expr: binary(
+ col("id", &schema)?,
+ Operator::Plus,
+ Arc::new(Literal::new(ScalarValue::Int64(Some(1)))),
+ &schema,
+ )?,
+ alias: "w".to_string(),
+ },
+ ],
+ hash,
+ )?);
+
+ let optimized = ProjectionPushdown::new()
+ .optimize(Arc::clone(&projection), &ConfigOptions::default())?;
+ let actual = displayable(optimized.as_ref())
+ .indent(true)
+ .to_string()
+ .trim()
+ .to_string();
+
+ // Unchanged. Without the check the scan absorbs the leaf and the hash keys
+ // are left indexing the old schema.
+ assert_snapshot!(
+ actual,
+ @r"
+ ProjectionExec: expr=[get_field(s@1, value) as v, id@0 + 1 as w]
+ RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[id, s], file_type=parquet
+ "
+ );
+ Ok(())
+}
diff --git a/datafusion/physical-optimizer/src/projection_pushdown.rs b/datafusion/physical-optimizer/src/projection_pushdown.rs
index 7faa72c1d9..364ec8d75a 100644
--- a/datafusion/physical-optimizer/src/projection_pushdown.rs
+++ b/datafusion/physical-optimizer/src/projection_pushdown.rs
@@ -18,14 +18,19 @@ use datafusion_common::tree_node::{
Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
};
use datafusion_common::{JoinSide, JoinType, Result, internal_datafusion_err};
+use datafusion_expr::ExpressionPlacement;
use datafusion_physical_expr::expressions::{Column, LambdaExpr, LambdaVariable};
+use datafusion_physical_expr::projection::ProjectionExpr;
use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, is_volatile};
-use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::joins::NestedLoopJoinExec;
use datafusion_physical_plan::joins::utils::{ColumnIndex, JoinFilter};
use datafusion_physical_plan::projection::{
ProjectionExec, remove_unnecessary_projections,
};
+use datafusion_physical_plan::repartition::RepartitionExec;
+use datafusion_physical_plan::{
+ ChildrenPropertiesMode, ExecutionPlan, Partitioning, ReplaceChildrenOptions,
+};
/// This rule inspects `ProjectionExec`'s in the given physical plan and tries to
/// remove or swap with its child.
@@ -61,7 +66,20 @@ impl PhysicalOptimizerRule for ProjectionPushdown {
})
.map(|t| t.data)?;
- plan.transform_down(remove_unnecessary_projections).data()
+ // Push what can already descend, then split the projections that are
+ // stuck above a `RepartitionExec`. The split needs no cleanup pass of
+ // its own: the outer projection it leaves either keeps every expression
+ // a plain column, and is then no narrower than the widened child, or it
+ // narrows and holds a computed one. `RepartitionExec` declines both, it
+ // is not removable because it renames, and nothing else moved.
+ let plan = plan.transform_down(remove_unnecessary_projections).data()?;
+ plan.transform_down(|plan| {
+ Ok(match try_split_leaf_projection(&plan)? {
+ Some(new_plan) => Transformed::yes(new_plan),
+ None => Transformed::no(plan),
+ })
+ })
+ .data()
}
fn name(&self) -> &str {
@@ -434,21 +452,236 @@ fn remap_filter_expr(
} else if let Some(variable) = expr.downcast_ref::<LambdaVariable>()
&& old_input_len != new_input_len
{
- let offset = variable.index().checked_sub(old_input_len).ok_or_else(|| {
- internal_datafusion_err!(
- "Lambda parameter {variable} precedes the {old_input_len} input columns"
- )
- })?;
- return Ok(Transformed::yes(Arc::new(LambdaVariable::new(
- new_input_len + offset,
- Arc::clone(variable.field()),
- )) as _));
+ let rebased = rebase_lambda_variable(variable, old_input_len, new_input_len)?;
+ return Ok(Transformed::yes(rebased));
}
Ok(Transformed::no(expr))
})
.data()
}
+/// Shifts a lambda parameter by the change in input width, for the reason
+/// [`remap_filter_expr`] documents.
+fn rebase_lambda_variable(
+ variable: &LambdaVariable,
+ old_input_len: usize,
+ new_input_len: usize,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let offset = variable.index().checked_sub(old_input_len).ok_or_else(|| {
+ internal_datafusion_err!(
+ "Lambda parameter {variable} precedes the {old_input_len} input columns"
+ )
+ })?;
+ Ok(Arc::new(LambdaVariable::new(
+ new_input_len + offset,
+ Arc::clone(variable.field()),
+ )))
+}
+
+/// Prefix for the columns this rule adds when it splits a projection.
+const LEAF_COLUMN_PREFIX: &str = "__datafusion_leaf";
+
+/// Name of the column the split adds for the leaf expression at `index`.
+///
+/// Single source of truth: the lower projection's alias and the reference the
+/// outer projection is rewritten to must agree, or the rewrite silently points
+/// at nothing.
+fn leaf_column_name(index: usize) -> String {
+ format!("{LEAF_COLUMN_PREFIX}_{index}")
+}
+
+/// Splits a `ProjectionExec` that mixes leaf expressions with ones that must
+/// stay in place, moving the leaves into a projection the source absorbs.
+///
+/// `benefits_from_input_partitioning` is true as soon as any expression is
+/// `KeepInPlace`, which both inserts a `RepartitionExec` below the projection
+/// and stops that projection descending past it, so a `MoveTowardsLeafNodes`
+/// expression sharing it never reaches the source. Splitting leaves the
+/// computation above the repartition, still parallelised.
+fn try_split_leaf_projection(
+ plan: &Arc<dyn ExecutionPlan>,
+) -> Result<Option<Arc<dyn ExecutionPlan>>> {
+ let Some(projection) = plan.downcast_ref::<ProjectionExec>() else {
+ return Ok(None);
+ };
+ let Some(repartition) = projection.input().downcast_ref::<RepartitionExec>() else {
+ return Ok(None);
+ };
+ // Hash and Range partitioning reference columns by index, which would have
+ // to be remapped against the rewritten child schema. Round robin carries no
+ // column references, and is what `EnsureRequirements` inserts here.
+ if !matches!(repartition.partitioning(), Partitioning::RoundRobinBatch(_)) {
+ return Ok(None);
+ }
+
+ let leaves = collect_leaf_exprs(projection.expr())?;
+ if leaves.is_empty() {
+ return Ok(None);
+ }
+
+ // Pass through only the child columns the outer expressions still need. A
+ // column used *only* inside an extracted leaf must be dropped, otherwise the
+ // source keeps reading it: for `s['a'] + 1` that would read the whole struct
+ // alongside the extracted field, which is worse than not splitting at all.
+ let child = repartition.input();
+ let child_schema = child.schema();
+ let passthrough_indices = collect_outer_columns(projection.expr(), &leaves)?;
+ let mut lower_exprs = passthrough_indices
+ .iter()
+ .map(|&idx| {
+ let field = child_schema.field(idx);
+ ProjectionExpr {
+ expr: Arc::new(Column::new(field.name(), idx)),
+ alias: field.name().to_string(),
+ }
+ })
+ .collect::<Vec<_>>();
+ for (index, leaf) in leaves.iter().enumerate() {
+ lower_exprs.push(ProjectionExpr {
+ expr: Arc::clone(leaf),
+ alias: leaf_column_name(index),
+ });
+ }
+ let lower_projection = ProjectionExec::try_new(lower_exprs, Arc::clone(child))?;
+
+ // Only commit the split if the child really absorbs the leaf projection.
+ // Otherwise this would just add a projection that helps nobody.
+ let Some(new_child) = child.try_swapping_with_projection(&lower_projection)? else {
+ return Ok(None);
+ };
+
+ let new_exprs = projection
+ .expr()
+ .iter()
+ .map(|proj_expr| {
+ Ok(ProjectionExpr {
+ expr: rewrite_outer_expr(
+ Arc::clone(&proj_expr.expr),
+ &leaves,
+ &passthrough_indices,
+ child_schema.fields().len(),
+ )?,
+ alias: proj_expr.alias.clone(),
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ // The child gained the leaf columns, so the repartition's properties have to
+ // be recomputed rather than carried over.
+ let new_repartition = Arc::clone(projection.input()).replace_children(
+ vec![new_child],
+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
+ )?;
+ // `try_new` would derive the output schema from the expressions alone and
+ // drop the metadata the planner set, which `schema_check` then rejects.
+ Ok(Some(Arc::new(
+ ProjectionExec::try_new_with_schema_metadata(
+ new_exprs,
+ new_repartition,
+ projection.schema().as_ref(),
+ )?,
+ )))
+}
+
+/// Collects the distinct outermost [`ExpressionPlacement::MoveTowardsLeafNodes`]
+/// sub-expressions of `exprs`.
+///
+/// Recursion stops at a leaf, so one nested inside another is not collected too:
+/// that would add a column no expression reads, and read more of the struct than
+/// the outer leaf needs. Plain SQL does not reach it, because the logical
+/// optimizer folds nested field access into a single call.
+fn collect_leaf_exprs(exprs: &[ProjectionExpr]) -> Result<Vec<Arc<dyn PhysicalExpr>>> {
+ let mut leaves: Vec<Arc<dyn PhysicalExpr>> = vec![];
+ for proj_expr in exprs {
+ proj_expr.expr.apply(|node| {
+ if node.placement() == ExpressionPlacement::MoveTowardsLeafNodes {
+ if !leaves.iter().any(|leaf| leaf == node) {
+ leaves.push(Arc::clone(node));
+ }
+ return Ok(TreeNodeRecursion::Jump);
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ }
+ Ok(leaves)
+}
+
+/// Collects the child column indices that `exprs` still reference *outside* the
+/// extracted leaf sub-expressions, sorted and deduplicated.
+///
+/// Recursion stops at a leaf: columns used only inside one are supplied by the
+/// lower projection itself and must not be passed through.
+fn collect_outer_columns(
+ exprs: &[ProjectionExpr],
+ leaves: &[Arc<dyn PhysicalExpr>],
+) -> Result<Vec<usize>> {
+ let mut indices: Vec<usize> = vec![];
+ for proj_expr in exprs {
+ proj_expr.expr.apply(|node| {
+ if leaves.iter().any(|leaf| leaf == node) {
+ return Ok(TreeNodeRecursion::Jump);
+ }
+ if let Some(column) = node.downcast_ref::<Column>() {
+ indices.push(column.index());
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ }
+ indices.sort_unstable();
+ indices.dedup();
+ Ok(indices)
+}
+
+/// Rewrites an outer projection expression against the split child schema:
+/// every leaf sub-expression becomes a reference to the column the split added
+/// for it, and every surviving column or lambda parameter is remapped to its
+/// new index. `old_input_len` is the child's width before the split.
+fn rewrite_outer_expr(
+ expr: Arc<dyn PhysicalExpr>,
+ leaves: &[Arc<dyn PhysicalExpr>],
+ passthrough_indices: &[usize],
+ old_input_len: usize,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ // The leaf columns were appended after the passthrough ones.
+ let first_leaf = passthrough_indices.len();
+ let new_input_len = first_leaf + leaves.len();
+ expr.transform_down(|node| {
+ if let Some(pos) = leaves.iter().position(|leaf| leaf == &node) {
+ let column = Column::new(&leaf_column_name(pos), first_leaf + pos);
+ return Ok(Transformed::new(
+ Arc::new(column) as Arc<dyn PhysicalExpr>,
+ true,
+ TreeNodeRecursion::Jump,
+ ));
+ }
+ if let Some(column) = node.downcast_ref::<Column>() {
+ let new_index = passthrough_indices
+ .iter()
+ .position(|&idx| idx == column.index())
+ .ok_or_else(|| {
+ internal_datafusion_err!(
+ "projection split dropped column {} that is still referenced",
+ column.index()
+ )
+ })?;
+ if new_index != column.index() {
+ let remapped = Column::new(column.name(), new_index);
+ return Ok(Transformed::yes(Arc::new(remapped) as Arc<dyn PhysicalExpr>));
+ }
+ }
+ // The split widens the input, which moves the lambda parameters indexed
+ // after it; see `remap_filter_expr`.
+ if let Some(variable) = node.downcast_ref::<LambdaVariable>()
+ && old_input_len != new_input_len
+ {
+ let rebased = rebase_lambda_variable(variable, old_input_len, new_input_len)?;
+ return Ok(Transformed::yes(rebased));
+ }
+ Ok(Transformed::no(node))
+ })
+ .data()
+}
+
#[cfg(test)]
mod test {
use super::*;
diff --git a/datafusion/sqllogictest/test_files/file_row_index.slt b/datafusion/sqllogictest/test_files/file_row_index.slt
index 935c4d6c72..45ccb38a8f 100644
--- a/datafusion/sqllogictest/test_files/file_row_index.slt
+++ b/datafusion/sqllogictest/test_files/file_row_index.slt
@@ -151,6 +151,169 @@ ORDER BY column1
statement ok
DROP TABLE parquet_two_files;
+# A leaf expression must still reach the scan when its projection also holds a
+# `KeepInPlace` one; see `try_split_leaf_projection` in
+# `datafusion/physical-optimizer/src/projection_pushdown.rs`. These need a
+# repartition, so `batch_size` is below the row count: do not raise it and do
+# not add a bare `LIMIT`, or the queries pass against broken code.
+
+statement ok
+SET datafusion.execution.batch_size = 2;
+
+# The marker wrapped in arithmetic.
+query II
+SELECT file_row_index() + 1, column1 FROM parquet_table ORDER BY column1
+----
+1 10
+2 20
+3 30
+4 40
+5 50
+
+# The marker is BARE here. An unrelated computed column in the same projection
+# is enough, because the flag is a property of the whole projection.
+query II
+SELECT file_row_index(), column1 + 1 FROM parquet_table ORDER BY column1
+----
+0 11
+1 21
+2 31
+3 41
+4 51
+
+# Not specific to arithmetic: any enclosing expression has the same effect.
+query TI
+SELECT arrow_typeof(file_row_index()), column1 FROM parquet_table ORDER BY column1
+----
+Int64 10
+Int64 20
+Int64 30
+Int64 40
+Int64 50
+
+# The sibling marker behaves the same way. Compared against a predicate rather
+# than printed, so the expectation does not depend on the absolute file path.
+query BI
+SELECT length(input_file_name()) > 0, column1 FROM parquet_table ORDER BY column1
+----
+true 10
+true 20
+true 30
+true 40
+true 50
+
+# Both markers in one projection, each inside an enclosing expression.
+query BI
+SELECT input_file_name() IS NOT NULL AND file_row_index() >= 0, column1
+FROM parquet_table ORDER BY column1
+----
+true 10
+true 20
+true 30
+true 40
+true 50
+
+# A bare `input_file_name()` beside a computed column: the same trigger as the
+# `file_row_index()` case above. Asserted through the plan rather than the
+# values, because the value is an absolute path.
+query TT
+EXPLAIN SELECT input_file_name(), column1 + 1 FROM parquet_table
+----
+logical_plan
+01)Projection: input_file_name(), CAST(parquet_table.column1 AS Int64) + Int64(1)
+02)--TableScan: parquet_table projection=[column1]
+physical_plan
+01)ProjectionExec: expr=[__datafusion_leaf_0@1 as input_file_name(), CAST(column1@0 AS Int64) + 1 as parquet_table.column1 + Int64(1)]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/file_row_index/parquet_table/data.parquet]]}, projection=[column1, input_file_name() as __datafusion_leaf_0], file_type=parquet
+
+# The same marker twice. Both calls are one expression, so the split gives them
+# a single column and the marker is evaluated once per row, not twice. Both
+# outputs must still agree with each other.
+query III
+SELECT file_row_index(), file_row_index() + 1, column1
+FROM parquet_table ORDER BY column1
+----
+0 1 10
+1 2 20
+2 3 30
+3 4 40
+4 5 50
+
+# The single column is what the values above cannot show: both outputs would
+# also agree if the marker were read twice.
+query TT
+EXPLAIN SELECT file_row_index(), file_row_index() + 1, column1 FROM parquet_table
+----
+logical_plan
+01)Projection: file_row_index(), file_row_index() + Int64(1), parquet_table.column1
+02)--TableScan: parquet_table projection=[column1]
+physical_plan
+01)ProjectionExec: expr=[__datafusion_leaf_0@1 as file_row_index(), __datafusion_leaf_0@1 + 1 as file_row_index() + Int64(1), column1@0 as column1]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/file_row_index/parquet_table/data.parquet]]}, projection=[column1, CAST(__datafusion_file_row_index@1 AS Int64;N) as __datafusion_leaf_0], file_type=parquet
+
+# `ORDER BY ... LIMIT` is a TopK that sorts on a column the split produced,
+# with `preserve_partitioning=[true]`, so it runs above the repartition on the
+# rewritten schema. No dynamic filter reaches the scan here: that needs the
+# sort key to be the computed sibling rather than the rewritten marker.
+query II
+SELECT file_row_index() AS r, column1 + 1 FROM parquet_table ORDER BY r DESC LIMIT 2
+----
+4 51
+3 41
+
+# Shapes that already worked must keep working: the leaf rules extract from
+# `Filter`, `Sort` and `Aggregate`, so those never depended on this fix.
+query I
+SELECT column1 FROM parquet_table WHERE file_row_index() + 1 = 3
+----
+30
+
+query I
+SELECT column1 FROM parquet_table ORDER BY file_row_index() + 1
+----
+10
+20
+30
+40
+50
+
+query I
+SELECT sum(file_row_index() + 1) FROM parquet_table
+----
+15
+
+# A bare marker on its own is unaffected: no `KeepInPlace` expression shares
+# the projection, so nothing is inserted beneath it.
+query II
+SELECT file_row_index(), column1 FROM parquet_table ORDER BY column1
+----
+0 10
+1 20
+2 30
+3 40
+4 50
+
+# Raising `batch_size` above the row count removes the repartition. This must
+# keep working, and it is also the control showing that `batch_size` is what
+# triggers the repartition.
+statement ok
+SET datafusion.execution.batch_size = 8192;
+
+query II
+SELECT file_row_index() + 1, column1 FROM parquet_table ORDER BY column1
+----
+1 10
+2 20
+3 30
+4 40
+5 50
+
+# `target_partitions = 1` also removes the repartition, but is not covered here:
+# `RESET` restores the global default rather than the harness value, which the
+# harness rejects as leftover configuration.
+
statement ok
DROP TABLE parquet_table;
diff --git a/datafusion/sqllogictest/test_files/input_file_name.slt b/datafusion/sqllogictest/test_files/input_file_name.slt
index 3ab598fb1e..53639c17fa 100644
--- a/datafusion/sqllogictest/test_files/input_file_name.slt
+++ b/datafusion/sqllogictest/test_files/input_file_name.slt
@@ -64,6 +64,27 @@ physical_plan
query error Execution error: input_file_name\(\) is source dependent and cannot be evaluated directly
SELECT input_file_name() FROM (VALUES (1)) v(x);
+# CSV has its own scan implementation, so the leaf split has to reach it too.
+# Needs a `KeepInPlace` sibling and a repartition, hence `column1 + 1` and the
+# `batch_size` below the row count.
+statement ok
+SET datafusion.execution.batch_size = 2;
+
+query BI
+SELECT length(input_file_name()) > 0, column1 + 1
+FROM csv_table
+ORDER BY 2
+----
+true 11
+true 21
+true 31
+true 41
+true 51
+true 61
+
+statement ok
+SET datafusion.execution.batch_size = 8192;
+
statement ok
DROP TABLE csv_table;
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index b5b001026d..eb698f6ef9 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -2675,5 +2675,205 @@ A 2
statement ok
DROP TABLE duplicate_alias;
+# A leaf expression must still reach the scan when its projection also holds a
+# `KeepInPlace` expression. See `try_split_leaf_projection` in
+# `datafusion/physical-optimizer/src/projection_pushdown.rs` for why that used
+# to be impossible.
+#
+# These are plan assertions on purpose. `get_field` stranded above the
+# repartition still returns the right answer, it just reads the whole struct,
+# so a result-only test passes whether or not the bug is present.
+#
+# Needs both a repartition (`target_partitions` > 1) and a row estimate above
+# `batch_size`, hence the two SETs. Restore them with `SET` rather than `RESET`:
+# `RESET` puts back the global default instead of the harness value, and the
+# harness rejects a file that leaves configuration modified.
+
+# `arr` and `tags` are only read by the lambda queries further down. They are
+# appended after `s` so the `get_field(s@1, ...)` indices below do not move.
+statement ok
+COPY (
+ SELECT column1 AS id, column2 AS s, column3 AS arr, column4 AS tags
+ FROM VALUES
+ (1, {value: 100, label: 'alpha'}, [1, 2, 3], ['a', 'b']),
+ (2, {value: 200, label: 'beta'}, [4, 5], ['c']),
+ (3, {value: 150, label: 'gamma'}, [6], ['d', 'e', 'f']),
+ (4, {value: 300, label: 'delta'}, [7, 8, 9], ['g']),
+ (5, {value: 250, label: 'epsilon'}, [10], ['h', 'i'])
+) TO 'test_files/scratch/projection_pushdown/leaf_split.parquet'
+STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE leaf_split STORED AS PARQUET
+LOCATION 'test_files/scratch/projection_pushdown/leaf_split.parquet';
+
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+statement ok
+SET datafusion.execution.batch_size = 2;
+
+# The scan must read only the `value` leaf. A plan holding a bare `s` in the
+# `DataSourceExec` projection means the whole struct is being read.
+query TT
+EXPLAIN SELECT s['value'] + 1 AS v FROM leaf_split;
+----
+logical_plan
+01)Projection: get_field(leaf_split.s, Utf8("value")) + Int64(1) AS v
+02)--TableScan: leaf_split projection=[s]
+physical_plan
+01)ProjectionExec: expr=[__datafusion_leaf_0@0 + 1 as v]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/leaf_split.parquet]]}, projection=[get_field(s@1, value) as __datafusion_leaf_0], file_type=parquet
+
+# The leaf is bare here: the unrelated computed column is what triggers it.
+query TT
+EXPLAIN SELECT s['value'] AS v, id + 1 AS w FROM leaf_split;
+----
+logical_plan
+01)Projection: get_field(leaf_split.s, Utf8("value")) AS v, leaf_split.id + Int64(1) AS w
+02)--TableScan: leaf_split projection=[id, s]
+physical_plan
+01)ProjectionExec: expr=[__datafusion_leaf_0@1 as v, id@0 + 1 as w]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/leaf_split.parquet]]}, projection=[id, get_field(s@1, value) as __datafusion_leaf_0], file_type=parquet
+
+# `s` is genuinely needed by the outer projection here, so it must stay.
+query TT
+EXPLAIN SELECT s['value'] + 1 AS v, s AS whole FROM leaf_split;
+----
+logical_plan
+01)Projection: get_field(leaf_split.s, Utf8("value")) + Int64(1) AS v, leaf_split.s AS whole
+02)--TableScan: leaf_split projection=[s]
+physical_plan
+01)ProjectionExec: expr=[__datafusion_leaf_0@1 + 1 as v, s@0 as whole]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/leaf_split.parquet]]}, projection=[s, get_field(s@1, value) as __datafusion_leaf_0], file_type=parquet
+
+# Two distinct leaves plus a computed sibling: both leaves must reach the scan
+# and the struct must not be read whole.
+query TT
+EXPLAIN SELECT s['value'] AS v, s['label'] AS l, id + 1 AS w FROM leaf_split;
+----
+logical_plan
+01)Projection: get_field(leaf_split.s, Utf8("value")) AS v, get_field(leaf_split.s, Utf8("label")) AS l, leaf_split.id + Int64(1) AS w
+02)--TableScan: leaf_split projection=[id, s]
+physical_plan
+01)ProjectionExec: expr=[__datafusion_leaf_0@1 as v, __datafusion_leaf_1@2 as l, id@0 + 1 as w]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/leaf_split.parquet]]}, projection=[id, get_field(s@1, value) as __datafusion_leaf_0, get_field(s@1, label) as __datafusion_leaf_1], file_type=parquet
+
+query ITI
+SELECT s['value'] AS v, s['label'] AS l, id + 1 AS w FROM leaf_split ORDER BY v;
+----
+100 alpha 2
+150 gamma 4
+200 beta 3
+250 epsilon 6
+300 delta 5
+
+query I
+SELECT s['value'] + 1 AS v FROM leaf_split ORDER BY v;
+----
+101
+151
+201
+251
+301
+
+# A lambda parameter is indexed after the input columns, so the split moves it;
+# see `rewrite_outer_expr`. These are value queries because an index left behind
+# is only sometimes an error, see the third one.
+
+statement ok
+SET datafusion.sql_parser.dialect = databricks;
+
+query ?IT
+SELECT array_transform(arr, x -> x + 1) AS a, s['value'] AS v, s['label'] AS l
+FROM leaf_split ORDER BY v;
+----
+[2, 3, 4] 100 alpha
+[7] 150 gamma
+[5, 6] 200 beta
+[11] 250 epsilon
+[8, 9, 10] 300 delta
+
+# Nested lambdas need two leaves: with one the split is width-neutral, the
+# parameter does not move, and the query passes against broken code.
+query ?IT
+SELECT array_transform(arr, x -> x + array_length(array_transform(arr, y -> y * 2))) AS a,
+ s['value'] AS v, s['label'] AS l
+FROM leaf_split ORDER BY v;
+----
+[4, 5, 6] 100 alpha
+[7] 150 gamma
+[6, 7] 200 beta
+[11] 250 epsilon
+[10, 11, 12] 300 delta
+
+# Naming the parameter after the column a stale index would land on makes
+# `LambdaVariable`'s field check match, so the wrong column is read with no
+# error at all. The name has to track `leaf_column_name` to stay adversarial.
+query ?ITI
+SELECT array_transform(tags, __datafusion_leaf_1 -> concat(__datafusion_leaf_1, '!')) AS a,
+ s['value'] AS v, s['label'] AS l, id
+FROM leaf_split ORDER BY v;
+----
+[a!, b!] 100 alpha 1
+[d!, e!, f!] 150 gamma 3
+[c!] 200 beta 2
+[h!, i!] 250 epsilon 5
+[g!] 300 delta 4
+
+# A leaf expression inside the lambda body, rather than beside it: the split has
+# to find it through `LambdaExpr` before the parameter rebase even applies.
+query ?IT
+SELECT array_transform(arr, x -> x + file_row_index()) AS a, s['value'] AS v, s['label'] AS l
+FROM leaf_split ORDER BY v;
+----
+[1, 2, 3] 100 alpha
+[8] 150 gamma
+[5, 6] 200 beta
+[14] 250 epsilon
+[10, 11, 12] 300 delta
+
+statement ok
+SET datafusion.sql_parser.dialect = generic;
+
+# A source that does not absorb the projection keeps the plan unchanged: the
+# split must not add a projection nobody can use. A memory table refuses
+# because `all_alias_free_columns` rejects the aliased `get_field`, and that is
+# the default for any node that does not override `try_swapping_with_projection`.
+statement ok
+CREATE TABLE leaf_split_mem AS
+SELECT column1 AS id, column2 AS s FROM VALUES
+ (1, {value: 100}), (2, {value: 200}), (3, {value: 150}),
+ (4, {value: 300}), (5, {value: 250});
+
+query TT
+EXPLAIN SELECT s['value'] AS v, id + 1 AS w FROM leaf_split_mem;
+----
+logical_plan
+01)Projection: get_field(leaf_split_mem.s, Utf8("value")) AS v, leaf_split_mem.id + Int64(1) AS w
+02)--TableScan: leaf_split_mem projection=[id, s]
+physical_plan
+01)ProjectionExec: expr=[get_field(s@1, value) as v, id@0 + 1 as w]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+03)----DataSourceExec: partitions=1, partition_sizes=[3]
+
+statement ok
+DROP TABLE leaf_split_mem;
+
+statement ok
+SET datafusion.execution.batch_size = 8192;
+
+# Back to 4, which is what this file leaves configured and what the harness
+# started from.
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+statement ok
+DROP TABLE leaf_split;
+
statement ok
RESET datafusion.execution.parquet.pushdown_filters;