Skip to content

Apache DataFusion SQL Query Engine

Redesign the try_reverse_output to support more cases (#19446)

Qi Zhu10db6b3712af
diff --git a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs
index 1d31374076..caef0fba05 100644
--- a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs
+++ b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs
@@ -24,14 +24,18 @@
//! 4. Early termination is enabled for TopK queries
//! 5. Prefix matching works correctly
+use datafusion_physical_expr::expressions;
+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_optimizer::pushdown_sort::PushdownSort;
+use std::sync::Arc;
use crate::physical_optimizer::test_utils::{
OptimizationTest, coalesce_batches_exec, coalesce_partitions_exec, parquet_exec,
- parquet_exec_with_sort, repartition_exec, schema, sort_exec, sort_exec_with_fetch,
- sort_expr,
+ parquet_exec_with_sort, projection_exec, projection_exec_with_alias,
+ repartition_exec, schema, simple_projection_exec, sort_exec, sort_exec_with_fetch,
+ sort_expr, sort_expr_named, test_scan_with_ordering,
};
#[test]
@@ -619,16 +623,14 @@ fn test_pushdown_through_blocking_node() {
// Middle: Aggregate (blocks pushdown from outer sort)
// GROUP BY a, COUNT(b)
let group_by = PhysicalGroupBy::new_single(vec![(
- Arc::new(datafusion_physical_expr::expressions::Column::new("a", 0)) as _,
+ Arc::new(expressions::Column::new("a", 0)) as _,
"a".to_string(),
)]);
let count_expr = Arc::new(
AggregateExprBuilder::new(
count_udaf(),
- vec![
- Arc::new(datafusion_physical_expr::expressions::Column::new("b", 1)) as _,
- ],
+ vec![Arc::new(expressions::Column::new("b", 1)) as _],
)
.schema(Arc::clone(&schema))
.alias("COUNT(b)")
@@ -670,3 +672,369 @@ fn test_pushdown_through_blocking_node() {
"
);
}
+
+// ============================================================================
+// PROJECTION TESTS
+// ============================================================================
+
+#[test]
+fn test_sort_pushdown_through_simple_projection() {
+ // Sort pushes through projection with simple column references
+ let schema = schema();
+
+ // Source has [a ASC] ordering
+ let a = sort_expr("a", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();
+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);
+
+ // Projection: SELECT a, b (simple column references)
+ let projection = simple_projection_exec(source, vec![0, 1]); // columns a, b
+
+ // Request [a DESC] - should push through projection to source
+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();
+ let plan = sort_exec(desc_ordering, projection);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ output:
+ Ok:
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ "
+ );
+}
+
+#[test]
+fn test_sort_pushdown_through_projection_with_alias() {
+ // Sort pushes through projection with column aliases
+ let schema = schema();
+
+ // Source has [a ASC] ordering
+ let a = sort_expr("a", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();
+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);
+
+ // Projection: SELECT a AS id, b AS value
+ let projection = projection_exec_with_alias(source, vec![(0, "id"), (1, "value")]);
+
+ // Request [id DESC] - should map to [a DESC] and push down
+ let id_expr = sort_expr_named("id", 0);
+ let desc_ordering = LexOrdering::new(vec![id_expr.reverse()]).unwrap();
+ let plan = sort_exec(desc_ordering, projection);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: expr=[id@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as id, b@1 as value]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ output:
+ Ok:
+ - SortExec: expr=[id@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as id, b@1 as value]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ "
+ );
+}
+
+#[test]
+fn test_no_sort_pushdown_through_computed_projection() {
+ use datafusion_expr::Operator;
+
+ // Sort should NOT push through projection with computed columns
+ let schema = schema();
+
+ // Source has [a ASC] ordering
+ let a = sort_expr("a", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();
+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);
+
+ // Projection: SELECT a+b as sum, c
+ let projection = projection_exec(
+ vec![
+ (
+ Arc::new(expressions::BinaryExpr::new(
+ Arc::new(expressions::Column::new("a", 0)),
+ Operator::Plus,
+ Arc::new(expressions::Column::new("b", 1)),
+ )) as Arc<dyn PhysicalExpr>,
+ "sum".to_string(),
+ ),
+ (
+ Arc::new(expressions::Column::new("c", 2)) as Arc<dyn PhysicalExpr>,
+ "c".to_string(),
+ ),
+ ],
+ source,
+ )
+ .unwrap();
+
+ // Request [sum DESC] - should NOT push down (sum is computed)
+ let sum_expr = sort_expr_named("sum", 0);
+ let desc_ordering = LexOrdering::new(vec![sum_expr.reverse()]).unwrap();
+ let plan = sort_exec(desc_ordering, projection);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: expr=[sum@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 + b@1 as sum, c@2 as c]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ output:
+ Ok:
+ - SortExec: expr=[sum@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 + b@1 as sum, c@2 as c]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ "
+ );
+}
+
+#[test]
+fn test_sort_pushdown_projection_reordered_columns() {
+ // Sort pushes through projection that reorders columns
+ let schema = schema();
+
+ // Source has [a ASC] ordering
+ let a = sort_expr("a", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();
+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);
+
+ // Projection: SELECT c, b, a (columns reordered)
+ let projection = simple_projection_exec(source, vec![2, 1, 0]); // c, b, a
+
+ // Request [a DESC] where a is now at index 2 in projection output
+ let a_expr_at_2 = sort_expr_named("a", 2);
+ let desc_ordering = LexOrdering::new(vec![a_expr_at_2.reverse()]).unwrap();
+ let plan = sort_exec(desc_ordering, projection);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: expr=[a@2 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ output:
+ Ok:
+ - SortExec: expr=[a@2 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ "
+ );
+}
+
+#[test]
+fn test_sort_pushdown_projection_with_limit() {
+ // Sort with LIMIT pushes through simple projection
+ let schema = schema();
+
+ // Source has [a ASC] ordering
+ let a = sort_expr("a", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();
+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);
+
+ // Projection: SELECT a, b
+ let projection = simple_projection_exec(source, vec![0, 1]);
+
+ // Request [a DESC] with LIMIT 10
+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();
+ let plan = sort_exec_with_fetch(desc_ordering, Some(10), projection);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ output:
+ Ok:
+ - SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ "
+ );
+}
+
+#[test]
+fn test_sort_pushdown_through_projection_and_coalesce() {
+ // Sort pushes through both projection and coalesce batches
+ let schema = schema();
+
+ // Source has [a ASC] ordering
+ let a = sort_expr("a", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();
+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);
+
+ let coalesce = coalesce_batches_exec(source, 1024);
+
+ // Projection: SELECT a, b
+ let projection = simple_projection_exec(coalesce, vec![0, 1]);
+
+ // Request [a DESC]
+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();
+ let plan = sort_exec(desc_ordering, projection);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
+ - CoalesceBatchesExec: target_batch_size=1024
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ output:
+ Ok:
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as a, b@1 as b]
+ - CoalesceBatchesExec: target_batch_size=1024
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ "
+ );
+}
+
+#[test]
+fn test_sort_pushdown_projection_subset_of_columns() {
+ // Sort pushes through projection that selects subset of columns
+ let schema = schema();
+
+ // Source has [a ASC, b ASC] ordering
+ let a = sort_expr("a", &schema);
+ let b = sort_expr("b", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone(), b.clone()]).unwrap();
+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);
+
+ // Projection: SELECT a (subset of columns)
+ let projection = simple_projection_exec(source, vec![0]);
+
+ // Request [a DESC]
+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();
+ let plan = sort_exec(desc_ordering, projection);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as a]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC, b@1 ASC], file_type=parquet
+ output:
+ Ok:
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - ProjectionExec: expr=[a@0 as a]
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ "
+ );
+}
+
+// ============================================================================
+// TESTSCAN DEMONSTRATION TESTS
+// ============================================================================
+// These tests use TestScan to demonstrate how sort pushdown works more clearly
+// than ParquetExec. TestScan can accept ANY ordering (not just reverse) and
+// displays the requested ordering explicitly in the output.
+
+#[test]
+fn test_sort_pushdown_with_test_scan_basic() {
+ // Demonstrates TestScan showing requested ordering clearly
+ let schema = schema();
+
+ // Source has [a ASC] ordering
+ let a = sort_expr("a", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();
+ let source = test_scan_with_ordering(schema.clone(), source_ordering);
+
+ // Request [a DESC] ordering
+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();
+ let plan = sort_exec(desc_ordering, source);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - TestScan: output_ordering=[a@0 ASC]
+ output:
+ Ok:
+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
+ - TestScan: output_ordering=[a@0 ASC], requested_ordering=[a@0 DESC NULLS LAST]
+ "
+ );
+}
+
+#[test]
+fn test_sort_pushdown_with_test_scan_multi_column() {
+ // Demonstrates TestScan with multi-column ordering
+ let schema = schema();
+
+ // Source has [a ASC, b DESC] ordering
+ let a = sort_expr("a", &schema);
+ let b = sort_expr("b", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone(), b.clone().reverse()]).unwrap();
+ let source = test_scan_with_ordering(schema.clone(), source_ordering);
+
+ // Request [a DESC, b ASC] ordering (reverse of source)
+ let reverse_ordering = LexOrdering::new(vec![a.reverse(), b]).unwrap();
+ let plan = sort_exec(reverse_ordering, source);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: expr=[a@0 DESC NULLS LAST, b@1 ASC], preserve_partitioning=[false]
+ - TestScan: output_ordering=[a@0 ASC, b@1 DESC NULLS LAST]
+ output:
+ Ok:
+ - SortExec: expr=[a@0 DESC NULLS LAST, b@1 ASC], preserve_partitioning=[false]
+ - TestScan: output_ordering=[a@0 ASC, b@1 DESC NULLS LAST], requested_ordering=[a@0 DESC NULLS LAST, b@1 ASC]
+ "
+ );
+}
+
+#[test]
+fn test_sort_pushdown_with_test_scan_arbitrary_ordering() {
+ // Demonstrates that TestScan can accept ANY ordering (not just reverse)
+ // This is different from ParquetExec which only supports reverse scans
+ let schema = schema();
+
+ // Source has [a ASC, b ASC] ordering
+ let a = sort_expr("a", &schema);
+ let b = sort_expr("b", &schema);
+ let source_ordering = LexOrdering::new(vec![a.clone(), b.clone()]).unwrap();
+ let source = test_scan_with_ordering(schema.clone(), source_ordering);
+
+ // Request [a ASC, b DESC] - NOT a simple reverse, but TestScan accepts it
+ let mixed_ordering = LexOrdering::new(vec![a, b.reverse()]).unwrap();
+ let plan = sort_exec(mixed_ordering, source);
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, PushdownSort::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]
+ - TestScan: output_ordering=[a@0 ASC, b@1 ASC]
+ output:
+ Ok:
+ - SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]
+ - TestScan: output_ordering=[a@0 ASC, b@1 ASC], requested_ordering=[a@0 ASC, b@1 DESC NULLS LAST]
+ "
+ );
+}
diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rs
index c81ccb6310..5b50181d7f 100644
--- a/datafusion/core/tests/physical_optimizer/test_utils.rs
+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs
@@ -41,6 +41,7 @@ use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_expr::{WindowFrame, WindowFunctionDefinition};
use datafusion_functions_aggregate::count::count_udaf;
+use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};
use datafusion_physical_expr::expressions::{self, col};
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
@@ -54,6 +55,7 @@ use datafusion_physical_plan::aggregates::{
};
use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
use datafusion_physical_plan::filter::FilterExec;
use datafusion_physical_plan::joins::utils::{JoinFilter, JoinOn};
use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode, SortMergeJoinExec};
@@ -68,7 +70,7 @@ use datafusion_physical_plan::union::UnionExec;
use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_window_expr};
use datafusion_physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, InputOrderMode, Partitioning,
- PlanProperties, displayable,
+ PlanProperties, SortOrderPushdownResult, displayable,
};
/// Create a non sorted parquet exec
@@ -774,3 +776,206 @@ pub fn format_execution_plan(plan: &Arc<dyn ExecutionPlan>) -> Vec<String> {
fn format_lines(s: &str) -> Vec<String> {
s.trim().split('\n').map(|s| s.to_string()).collect()
}
+
+/// Create a simple ProjectionExec with column indices (simplified version)
+pub fn simple_projection_exec(
+ input: Arc<dyn ExecutionPlan>,
+ columns: Vec<usize>,
+) -> Arc<dyn ExecutionPlan> {
+ let schema = input.schema();
+ let exprs: Vec<(Arc<dyn PhysicalExpr>, String)> = columns
+ .iter()
+ .map(|&i| {
+ let field = schema.field(i);
+ (
+ Arc::new(expressions::Column::new(field.name(), i))
+ as Arc<dyn PhysicalExpr>,
+ field.name().to_string(),
+ )
+ })
+ .collect();
+
+ projection_exec(exprs, input).unwrap()
+}
+
+/// Create a ProjectionExec with column aliases
+pub fn projection_exec_with_alias(
+ input: Arc<dyn ExecutionPlan>,
+ columns: Vec<(usize, &str)>,
+) -> Arc<dyn ExecutionPlan> {
+ let schema = input.schema();
+ let exprs: Vec<(Arc<dyn PhysicalExpr>, String)> = columns
+ .iter()
+ .map(|&(i, alias)| {
+ (
+ Arc::new(expressions::Column::new(schema.field(i).name(), i))
+ as Arc<dyn PhysicalExpr>,
+ alias.to_string(),
+ )
+ })
+ .collect();
+
+ projection_exec(exprs, input).unwrap()
+}
+
+/// Create a sort expression with custom name and index
+pub fn sort_expr_named(name: &str, index: usize) -> PhysicalSortExpr {
+ PhysicalSortExpr {
+ expr: Arc::new(expressions::Column::new(name, index)),
+ options: SortOptions::default(),
+ }
+}
+
+/// A test data source that can display any requested ordering
+/// This is useful for testing sort pushdown behavior
+#[derive(Debug, Clone)]
+pub struct TestScan {
+ schema: SchemaRef,
+ output_ordering: Vec<LexOrdering>,
+ plan_properties: PlanProperties,
+ // Store the requested ordering for display
+ requested_ordering: Option<LexOrdering>,
+}
+
+impl TestScan {
+ /// Create a new TestScan with the given schema and output ordering
+ pub fn new(schema: SchemaRef, output_ordering: Vec<LexOrdering>) -> Self {
+ let eq_properties = if !output_ordering.is_empty() {
+ // Convert Vec<LexOrdering> to the format expected by new_with_orderings
+ // We need to extract the inner Vec<PhysicalSortExpr> from each LexOrdering
+ let orderings: Vec<Vec<PhysicalSortExpr>> = output_ordering
+ .iter()
+ .map(|lex_ordering| {
+ // LexOrdering implements IntoIterator, so we can collect it
+ lex_ordering.iter().cloned().collect()
+ })
+ .collect();
+
+ EquivalenceProperties::new_with_orderings(Arc::clone(&schema), orderings)
+ } else {
+ EquivalenceProperties::new(Arc::clone(&schema))
+ };
+
+ let plan_properties = PlanProperties::new(
+ eq_properties,
+ Partitioning::UnknownPartitioning(1),
+ EmissionType::Incremental,
+ Boundedness::Bounded,
+ );
+
+ Self {
+ schema,
+ output_ordering,
+ plan_properties,
+ requested_ordering: None,
+ }
+ }
+
+ /// Create a TestScan with a single output ordering
+ pub fn with_ordering(schema: SchemaRef, ordering: LexOrdering) -> Self {
+ Self::new(schema, vec![ordering])
+ }
+}
+
+impl DisplayAs for TestScan {
+ fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+ match t {
+ DisplayFormatType::Default | DisplayFormatType::Verbose => {
+ write!(f, "TestScan")?;
+ if !self.output_ordering.is_empty() {
+ write!(f, ": output_ordering=[")?;
+ // Format the ordering in a readable way
+ for (i, sort_expr) in self.output_ordering[0].iter().enumerate() {
+ if i > 0 {
+ write!(f, ", ")?;
+ }
+ write!(f, "{sort_expr}")?;
+ }
+ write!(f, "]")?;
+ }
+ // This is the key part - show what ordering was requested
+ if let Some(ref req) = self.requested_ordering {
+ write!(f, ", requested_ordering=[")?;
+ for (i, sort_expr) in req.iter().enumerate() {
+ if i > 0 {
+ write!(f, ", ")?;
+ }
+ write!(f, "{sort_expr}")?;
+ }
+ write!(f, "]")?;
+ }
+ Ok(())
+ }
+ DisplayFormatType::TreeRender => {
+ write!(f, "TestScan")
+ }
+ }
+ }
+}
+
+impl ExecutionPlan for TestScan {
+ fn name(&self) -> &str {
+ "TestScan"
+ }
+
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+
+ fn properties(&self) -> &PlanProperties {
+ &self.plan_properties
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ vec![]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ if children.is_empty() {
+ Ok(self)
+ } else {
+ internal_err!("TestScan should have no children")
+ }
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ internal_err!("TestScan is for testing optimizer only, not for execution")
+ }
+
+ fn partition_statistics(&self, _partition: Option<usize>) -> Result<Statistics> {
+ Ok(Statistics::new_unknown(&self.schema))
+ }
+
+ // This is the key method - implement sort pushdown
+ fn try_pushdown_sort(
+ &self,
+ order: &[PhysicalSortExpr],
+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {
+ // For testing purposes, accept ANY ordering request
+ // and create a new TestScan that shows what was requested
+ let requested_ordering = LexOrdering::new(order.to_vec());
+
+ let mut new_scan = self.clone();
+ new_scan.requested_ordering = requested_ordering;
+
+ // Always return Inexact to keep the Sort node (like Phase 1 behavior)
+ Ok(SortOrderPushdownResult::Inexact {
+ inner: Arc::new(new_scan),
+ })
+ }
+}
+
+/// Helper function to create a TestScan with ordering
+pub fn test_scan_with_ordering(
+ schema: SchemaRef,
+ ordering: LexOrdering,
+) -> Arc<dyn ExecutionPlan> {
+ Arc::new(TestScan::with_ordering(schema, ordering))
+}
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 7d7662a4a9..2e0919b144 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -38,8 +38,8 @@ use datafusion_common::config::TableParquetOptions;
use datafusion_datasource::TableSchema;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
-use datafusion_physical_expr::conjunction;
use datafusion_physical_expr::projection::ProjectionExprs;
+use datafusion_physical_expr::{EquivalenceProperties, conjunction};
use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::physical_expr::fmt_sql;
@@ -54,7 +54,7 @@ use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
#[cfg(feature = "parquet_encryption")]
use datafusion_execution::parquet_encryption::EncryptionFactory;
-use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
use itertools::Itertools;
use object_store::ObjectStore;
#[cfg(feature = "parquet_encryption")]
@@ -288,9 +288,6 @@ pub struct ParquetSource {
pub(crate) projection: ProjectionExprs,
#[cfg(feature = "parquet_encryption")]
pub(crate) encryption_factory: Option<Arc<dyn EncryptionFactory>>,
- /// The ordering of data within the files
- /// This is set by FileScanConfig when it knows the file ordering
- file_ordering: Option<LexOrdering>,
/// If true, read files in reverse order and reverse row groups within files.
/// But it's not guaranteed that rows within row groups are in reverse order,
/// so we still need to sort them after reading, so the reverse scan is inexact.
@@ -320,7 +317,6 @@ impl ParquetSource {
metadata_size_hint: None,
#[cfg(feature = "parquet_encryption")]
encryption_factory: None,
- file_ordering: None,
reverse_row_groups: false,
}
}
@@ -397,12 +393,6 @@ impl ParquetSource {
self
}
- /// If set, indicates the ordering of data within the files being read.
- pub fn with_file_ordering(mut self, ordering: Option<LexOrdering>) -> Self {
- self.file_ordering = ordering;
- self
- }
-
/// Return the value described in [`Self::with_pushdown_filters`]
pub(crate) fn pushdown_filters(&self) -> bool {
self.table_parquet_options.global.pushdown_filters
@@ -769,44 +759,61 @@ impl FileSource for ParquetSource {
fn try_reverse_output(
&self,
order: &[PhysicalSortExpr],
+ eq_properties: &EquivalenceProperties,
) -> datafusion_common::Result<SortOrderPushdownResult<Arc<dyn FileSource>>> {
- // Check if we have file ordering information
- let file_ordering = match &self.file_ordering {
- Some(ordering) => ordering,
- None => return Ok(SortOrderPushdownResult::Unsupported),
- };
-
- // Create a LexOrdering from the requested order to use the is_reverse method
- let Some(requested_ordering) = LexOrdering::new(order.to_vec()) else {
- // Empty ordering requested, cannot optimize
+ if order.is_empty() {
return Ok(SortOrderPushdownResult::Unsupported);
- };
+ }
- // Check if reversing the file ordering would satisfy the requested ordering
- if file_ordering.is_reverse(&requested_ordering) {
- // Phase 1: Enable reverse row group scanning
- let new_source = self.clone().with_reverse_row_groups(true);
+ // Build new equivalence properties with the reversed ordering.
+ // This allows us to check if the reversed ordering satisfies the request
+ // by leveraging:
+ // - Function monotonicity (e.g., extract_year_month preserves ordering)
+ // - Constant columns (from filters)
+ // - Other equivalence relationships
+ //
+ // Example flow:
+ // 1. File ordering: [extract_year_month(ws) DESC, ws DESC]
+ // 2. After reversal: [extract_year_month(ws) ASC, ws ASC]
+ // 3. Requested: [ws ASC]
+ // 4. Through extract_year_month's monotonicity property, the reversed
+ // ordering satisfies [ws ASC] even though it has additional prefix
+ let reversed_eq_properties = {
+ let mut new = eq_properties.clone();
+ new.clear_orderings();
+
+ // Reverse each ordering in the equivalence properties
+ let reversed_orderings = eq_properties
+ .oeq_class()
+ .iter()
+ .map(|ordering| {
+ ordering
+ .iter()
+ .map(|expr| expr.reverse())
+ .collect::<Vec<_>>()
+ })
+ .collect::<Vec<_>>();
+
+ new.add_orderings(reversed_orderings);
+ new
+ };
- // Return Inexact because we're only reversing row group order,
- // not guaranteeing perfect row-level ordering
- return Ok(SortOrderPushdownResult::Inexact {
- inner: Arc::new(new_source) as Arc<dyn FileSource>,
- });
+ // Check if the reversed ordering satisfies the requested ordering
+ if !reversed_eq_properties.ordering_satisfy(order.iter().cloned())? {
+ return Ok(SortOrderPushdownResult::Unsupported);
}
+ // Return Inexact because we're only reversing row group order,
+ // not guaranteeing perfect row-level ordering
+ let new_source = self.clone().with_reverse_row_groups(true);
+ Ok(SortOrderPushdownResult::Inexact {
+ inner: Arc::new(new_source) as Arc<dyn FileSource>,
+ })
+
// TODO Phase 2: Add support for other optimizations:
// - File reordering based on min/max statistics
// - Detection of exact ordering (return Exact to remove Sort operator)
// - Partial sort pushdown for prefix matches
-
- Ok(SortOrderPushdownResult::Unsupported)
- }
-
- fn with_file_ordering_info(
- &self,
- ordering: Option<LexOrdering>,
- ) -> datafusion_common::Result<Arc<dyn FileSource>> {
- Ok(Arc::new(self.clone().with_file_ordering(ordering)))
}
}
diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rs
index 4aac3d46d4..f5380c27ec 100644
--- a/datafusion/datasource/src/file.rs
+++ b/datafusion/datasource/src/file.rs
@@ -30,7 +30,7 @@ use crate::schema_adapter::SchemaAdapterFactory;
use datafusion_common::config::ConfigOptions;
use datafusion_common::{Result, not_impl_err};
use datafusion_physical_expr::projection::ProjectionExprs;
-use datafusion_physical_expr::{LexOrdering, PhysicalExpr};
+use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, PhysicalExpr};
use datafusion_physical_plan::DisplayFormatType;
use datafusion_physical_plan::SortOrderPushdownResult;
use datafusion_physical_plan::filter_pushdown::{FilterPushdownPropagation, PushedDown};
@@ -150,15 +150,50 @@ pub trait FileSource: Send + Sync {
/// Try to create a new FileSource that can produce data in the specified sort order.
///
+ /// This method attempts to optimize data retrieval to match the requested ordering.
+ /// It receives both the requested ordering and equivalence properties that describe
+ /// the output data from this file source.
+ ///
+ /// # Parameters
+ /// * `order` - The requested sort ordering from the query
+ /// * `eq_properties` - Equivalence properties of the data that will be produced by this
+ /// file source. These properties describe the ordering, constant columns, and other
+ /// relationships in the output data, allowing the implementation to determine if
+ /// optimizations like reversed scanning can help satisfy the requested ordering.
+ /// This includes information about:
+ /// - The file's natural ordering (from output_ordering in FileScanConfig)
+ /// - Constant columns (e.g., from filters like `ticker = 'AAPL'`)
+ /// - Monotonic functions (e.g., `extract_year_month(timestamp)`)
+ /// - Other equivalence relationships
+ ///
+ /// # Examples
+ ///
+ /// ## Example 1: Simple reverse
+ /// ```text
+ /// File ordering: [a ASC, b DESC]
+ /// Requested: [a DESC]
+ /// Reversed file: [a DESC, b ASC]
+ /// Result: Satisfies request (prefix match) → Inexact
+ /// ```
+ ///
+ /// ## Example 2: Monotonic function
+ /// ```text
+ /// File ordering: [extract_year_month(ts) ASC, ts ASC]
+ /// Requested: [ts DESC]
+ /// Reversed file: [extract_year_month(ts) DESC, ts DESC]
+ /// Result: Through monotonicity, satisfies [ts DESC] → Inexact
+ /// ```
+ ///
/// # Returns
/// * `Exact` - Created a source that guarantees perfect ordering
- /// * `Inexact` - Created a source optimized for ordering (e.g., reordered files) but not perfectly sorted
+ /// * `Inexact` - Created a source optimized for ordering (e.g., reversed row groups) but not perfectly sorted
/// * `Unsupported` - Cannot optimize for this ordering
///
/// Default implementation returns `Unsupported`.
fn try_reverse_output(
&self,
_order: &[PhysicalSortExpr],
+ _eq_properties: &EquivalenceProperties,
) -> Result<SortOrderPushdownResult<Arc<dyn FileSource>>> {
Ok(SortOrderPushdownResult::Unsupported)
}
@@ -192,24 +227,6 @@ pub trait FileSource: Send + Sync {
Ok(None)
}
- /// Set the file ordering information
- ///
- /// This allows the file source to know how the files are sorted,
- /// enabling it to make informed decisions about sort pushdown.
- ///
- /// # Default Implementation
- ///
- /// Returns `not_impl_err!`. FileSource implementations that support
- /// sort optimization should override this method.
- fn with_file_ordering_info(
- &self,
- _ordering: Option<LexOrdering>,
- ) -> Result<Arc<dyn FileSource>> {
- // Default: clone self without modification
- // ParquetSource will override this
- not_impl_err!("with_file_ordering_info not implemented for this FileSource")
- }
-
/// Deprecated: Set optional schema adapter factory.
///
/// `SchemaAdapterFactory` has been removed. Use `PhysicalExprAdapterFactory` instead.
diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rs
index b17b5fba68..c8636343cc 100644
--- a/datafusion/datasource/src/file_scan_config.rs
+++ b/datafusion/datasource/src/file_scan_config.rs
@@ -829,23 +829,10 @@ impl DataSource for FileScanConfig {
&self,
order: &[PhysicalSortExpr],
) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {
- let file_ordering = self.output_ordering.first().cloned();
-
- if file_ordering.is_none() {
- return Ok(SortOrderPushdownResult::Unsupported);
- }
-
- // Use the trait method instead of downcasting
- // Try to provide file ordering info to the source
- // If not supported (e.g., CsvSource), fall back to original source
- let file_source_with_ordering = self
+ // Delegate to FileSource to check if reverse scanning can satisfy the request.
+ let pushdown_result = self
.file_source
- .with_file_ordering_info(file_ordering)
- .unwrap_or_else(|_| Arc::clone(&self.file_source));
-
- // Try to reverse the datasource with ordering info,
- // and currently only ParquetSource supports it with inexact reverse with row groups.
- let pushdown_result = file_source_with_ordering.try_reverse_output(order)?;
+ .try_reverse_output(order, &self.eq_properties())?;
match pushdown_result {
SortOrderPushdownResult::Exact { inner } => {
diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rs
index d929c78850..a1fad86777 100644
--- a/datafusion/physical-plan/src/coop.rs
+++ b/datafusion/physical-plan/src/coop.rs
@@ -82,7 +82,7 @@ use crate::filter_pushdown::{
use crate::projection::ProjectionExec;
use crate::{
DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, RecordBatchStream,
- SendableRecordBatchStream,
+ SendableRecordBatchStream, SortOrderPushdownResult,
};
use arrow::record_batch::RecordBatch;
use arrow_schema::Schema;
@@ -91,6 +91,7 @@ use datafusion_execution::TaskContext;
use crate::execution_plan::SchedulingType;
use crate::stream::RecordBatchStreamAdapter;
+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
use futures::{Stream, StreamExt};
/// A stream that passes record batches through unchanged while cooperating with the Tokio runtime.
@@ -328,6 +329,27 @@ impl ExecutionPlan for CooperativeExec {
) -> Result<FilterPushdownPropagation<Arc<dyn ExecutionPlan>>> {
Ok(FilterPushdownPropagation::if_all(child_pushdown_result))
}
+
+ fn try_pushdown_sort(
+ &self,
+ order: &[PhysicalSortExpr],
+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {
+ let child = self.input();
+
+ match child.try_pushdown_sort(order)? {
+ SortOrderPushdownResult::Exact { inner } => {
+ let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;
+ Ok(SortOrderPushdownResult::Exact { inner: new_exec })
+ }
+ SortOrderPushdownResult::Inexact { inner } => {
+ let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;
+ Ok(SortOrderPushdownResult::Inexact { inner: new_exec })
+ }
+ SortOrderPushdownResult::Unsupported => {
+ Ok(SortOrderPushdownResult::Unsupported)
+ }
+ }
+ }
}
/// Creates a [`CooperativeStream`] wrapper around the given [`RecordBatchStream`].
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs
index a56e9272f1..135f78677f 100644
--- a/datafusion/physical-plan/src/projection.rs
+++ b/datafusion/physical-plan/src/projection.rs
@@ -24,7 +24,7 @@ use super::expressions::{Column, Literal};
use super::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
use super::{
DisplayAs, ExecutionPlanProperties, PlanProperties, RecordBatchStream,
- SendableRecordBatchStream, Statistics,
+ SendableRecordBatchStream, SortOrderPushdownResult, Statistics,
};
use crate::execution_plan::CardinalityEffect;
use crate::filter_pushdown::{
@@ -51,7 +51,9 @@ use datafusion_physical_expr::equivalence::ProjectionMapping;
use datafusion_physical_expr::projection::Projector;
use datafusion_physical_expr::utils::collect_columns;
use datafusion_physical_expr_common::physical_expr::{PhysicalExprRef, fmt_sql};
-use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement};
+use datafusion_physical_expr_common::sort_expr::{
+ LexOrdering, LexRequirement, PhysicalSortExpr,
+};
// Re-exported from datafusion-physical-expr for backwards compatibility
// We recommend updating your imports to use datafusion-physical-expr directly
pub use datafusion_physical_expr::projection::{
@@ -357,6 +359,72 @@ impl ExecutionPlan for ProjectionExec {
) -> Result<FilterPushdownPropagation<Arc<dyn ExecutionPlan>>> {
Ok(FilterPushdownPropagation::if_all(child_pushdown_result))
}
+
+ fn try_pushdown_sort(
+ &self,
+ order: &[PhysicalSortExpr],
+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {
+ let child = self.input();
+ let mut child_order = Vec::new();
+
+ // Check and transform sort expressions
+ for sort_expr in order {
+ // Recursively transform the expression
+ let mut can_pushdown = true;
+ let transformed = Arc::clone(&sort_expr.expr).transform(|expr| {
+ if let Some(col) = expr.as_any().downcast_ref::<Column>() {
+ // Check if column index is valid.
+ // This should always be true but fail gracefully if it's not.
+ if col.index() >= self.expr().len() {
+ can_pushdown = false;
+ return Ok(Transformed::no(expr));
+ }
+
+ let proj_expr = &self.expr()[col.index()];
+
+ // Check if projection expression is a simple column
+ // We cannot push down order by clauses that depend on
+ // projected computations as they would have nothing to reference.
+ if let Some(child_col) =
+ proj_expr.expr.as_any().downcast_ref::<Column>()
+ {
+ // Replace with the child column
+ Ok(Transformed::yes(Arc::new(child_col.clone()) as _))
+ } else {
+ // Projection involves computation, cannot push down
+ can_pushdown = false;
+ Ok(Transformed::no(expr))
+ }
+ } else {
+ Ok(Transformed::no(expr))
+ }
+ })?;
+
+ if !can_pushdown {
+ return Ok(SortOrderPushdownResult::Unsupported);
+ }
+
+ child_order.push(PhysicalSortExpr {
+ expr: transformed.data,
+ options: sort_expr.options,
+ });
+ }
+
+ // Recursively push down to child node
+ match child.try_pushdown_sort(&child_order)? {
+ SortOrderPushdownResult::Exact { inner } => {
+ let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;
+ Ok(SortOrderPushdownResult::Exact { inner: new_exec })
+ }
+ SortOrderPushdownResult::Inexact { inner } => {
+ let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;
+ Ok(SortOrderPushdownResult::Inexact { inner: new_exec })
+ }
+ SortOrderPushdownResult::Unsupported => {
+ Ok(SortOrderPushdownResult::Unsupported)
+ }
+ }
+ }
}
impl ProjectionStream {
diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
index 3d08cdf751..3e403171e0 100644
--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
@@ -426,329 +426,3 @@ SET datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = true;
statement ok
SET datafusion.optimizer.enable_dynamic_filter_pushdown = true;
-
-# Test 6: Sort Pushdown for ordered Parquet files
-
-# Create a sorted dataset
-statement ok
-CREATE TABLE sorted_data(id INT, value INT, name VARCHAR) AS VALUES
-(1, 100, 'a'),
-(2, 200, 'b'),
-(3, 300, 'c'),
-(4, 400, 'd'),
-(5, 500, 'e'),
-(6, 600, 'f'),
-(7, 700, 'g'),
-(8, 800, 'h'),
-(9, 900, 'i'),
-(10, 1000, 'j');
-
-# Copy to parquet with sorting
-query I
-COPY (SELECT * FROM sorted_data ORDER BY id ASC)
-TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet';
-----
-10
-
-statement ok
-CREATE EXTERNAL TABLE sorted_parquet(id INT, value INT, name VARCHAR)
-STORED AS PARQUET
-LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet'
-WITH ORDER (id ASC);
-
-# Test 6.1: Sort pushdown with DESC (opposite of ASC)
-# Should show reverse_row_groups=true
-query TT
-EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;
-----
-logical_plan
-01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3
-02)--TableScan: sorted_parquet projection=[id, value, name]
-physical_plan
-01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
-
-# Test 6.2: Verify results are correct
-query IIT
-SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;
-----
-10 1000 j
-9 900 i
-8 800 h
-
-# Test 6.3: Should NOT apply for ASC (same direction)
-query TT
-EXPLAIN SELECT * FROM sorted_parquet ORDER BY id ASC LIMIT 3;
-----
-logical_plan
-01)Sort: sorted_parquet.id ASC NULLS LAST, fetch=3
-02)--TableScan: sorted_parquet projection=[id, value, name]
-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet
-
-# Test 6.4: Disable sort pushdown
-statement ok
-SET datafusion.optimizer.enable_sort_pushdown = false;
-
-query TT
-EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;
-----
-logical_plan
-01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3
-02)--TableScan: sorted_parquet projection=[id, value, name]
-physical_plan
-01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]
-
-# Re-enable
-statement ok
-SET datafusion.optimizer.enable_sort_pushdown = true;
-
-# Test 6.5: With OFFSET
-query TT
-EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;
-----
-logical_plan
-01)Limit: skip=2, fetch=3
-02)--Sort: sorted_parquet.id DESC NULLS FIRST, fetch=5
-03)----TableScan: sorted_parquet projection=[id, value, name]
-physical_plan
-01)GlobalLimitExec: skip=2, fetch=3
-02)--SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]
-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
-
-query IIT
-SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;
-----
-8 800 h
-7 700 g
-6 600 f
-
-# Test 6.6: Reverse scan with row selection (page index pruning)
-# This tests that when reverse_row_groups=true, the RowSelection is also properly reversed
-
-# Create a dataset with multiple row groups and enable page index
-statement ok
-CREATE TABLE multi_rg_data(id INT, category VARCHAR, value INT) AS VALUES
-(1, 'alpha', 10),
-(2, 'alpha', 20),
-(3, 'beta', 30),
-(4, 'beta', 40),
-(5, 'gamma', 50),
-(6, 'gamma', 60),
-(7, 'delta', 70),
-(8, 'delta', 80);
-
-# Write with small row groups (2 rows each = 4 row groups)
-statement ok
-SET datafusion.execution.parquet.max_row_group_size = 2;
-
-query I
-COPY (SELECT * FROM multi_rg_data ORDER BY id ASC)
-TO 'test_files/scratch/dynamic_filter_pushdown_config/multi_rg_sorted.parquet';
-----
-8
-
-# Reset row group size
-statement ok
-SET datafusion.execution.parquet.max_row_group_size = 1048576;
-
-statement ok
-CREATE EXTERNAL TABLE multi_rg_sorted(id INT, category VARCHAR, value INT)
-STORED AS PARQUET
-LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/multi_rg_sorted.parquet'
-WITH ORDER (id ASC);
-
-# Enable page index for better pruning
-statement ok
-SET datafusion.execution.parquet.enable_page_index = true;
-
-statement ok
-SET datafusion.execution.parquet.pushdown_filters = true;
-
-# Test with reverse scan and filter that prunes some row groups
-# This will create a RowSelection with partial row group scans
-query TT
-EXPLAIN SELECT * FROM multi_rg_sorted
-WHERE category IN ('alpha', 'gamma')
-ORDER BY id DESC LIMIT 5;
-----
-logical_plan
-01)Sort: multi_rg_sorted.id DESC NULLS FIRST, fetch=5
-02)--Filter: multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")
-03)----TableScan: multi_rg_sorted projection=[id, category, value], partial_filters=[multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")]
-physical_plan
-01)SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/multi_rg_sorted.parquet]]}, projection=[id, category, value], file_type=parquet, predicate=(category@1 = alpha OR category@1 = gamma) AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=category_null_count@2 != row_count@3 AND category_min@0 <= alpha AND alpha <= category_max@1 OR category_null_count@2 != row_count@3 AND category_min@0 <= gamma AND gamma <= category_max@1, required_guarantees=[category in (alpha, gamma)]
-
-# Verify the results are correct despite reverse scanning with row selection
-# Expected: gamma values (6, 5) then alpha values (2, 1), in DESC order by id
-query ITI
-SELECT * FROM multi_rg_sorted
-WHERE category IN ('alpha', 'gamma')
-ORDER BY id DESC LIMIT 5;
-----
-6 gamma 60
-5 gamma 50
-2 alpha 20
-1 alpha 10
-
-# Test with more complex selection pattern
-query ITI
-SELECT * FROM multi_rg_sorted
-WHERE category IN ('beta', 'delta')
-ORDER BY id DESC;
-----
-8 delta 80
-7 delta 70
-4 beta 40
-3 beta 30
-
-# Test forward scan for comparison (should give same logical results in ASC order)
-query ITI
-SELECT * FROM multi_rg_sorted
-WHERE category IN ('alpha', 'gamma')
-ORDER BY id ASC;
-----
-1 alpha 10
-2 alpha 20
-5 gamma 50
-6 gamma 60
-
-# Disable reverse scan and verify it still works
-statement ok
-SET datafusion.optimizer.enable_sort_pushdown = false;
-
-query ITI
-SELECT * FROM multi_rg_sorted
-WHERE category IN ('alpha', 'gamma')
-ORDER BY id DESC LIMIT 5;
-----
-6 gamma 60
-5 gamma 50
-2 alpha 20
-1 alpha 10
-
-# Re-enable
-statement ok
-SET datafusion.optimizer.enable_sort_pushdown = true;
-
-# Test 6.7: Sort pushdown with more than one partition
-# Create multiple parquet files to trigger it
-
-# Split data into multiple files
-statement ok
-CREATE TABLE sorted_data_part1(id INT, value INT, name VARCHAR) AS VALUES
-(1, 100, 'a'),
-(2, 200, 'b'),
-(3, 300, 'c');
-
-statement ok
-CREATE TABLE sorted_data_part2(id INT, value INT, name VARCHAR) AS VALUES
-(4, 400, 'd'),
-(5, 500, 'e'),
-(6, 600, 'f');
-
-statement ok
-CREATE TABLE sorted_data_part3(id INT, value INT, name VARCHAR) AS VALUES
-(7, 700, 'g'),
-(8, 800, 'h'),
-(9, 900, 'i'),
-(10, 1000, 'j');
-
-# Create directory for multi-file parquet
-query I
-COPY (SELECT * FROM sorted_data_part1 ORDER BY id ASC)
-TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part1.parquet';
-----
-3
-
-query I
-COPY (SELECT * FROM sorted_data_part2 ORDER BY id ASC)
-TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part2.parquet';
-----
-3
-
-query I
-COPY (SELECT * FROM sorted_data_part3 ORDER BY id ASC)
-TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part3.parquet';
-----
-4
-
-# Create external table pointing to directory with multiple files
-statement ok
-CREATE EXTERNAL TABLE sorted_parquet_multi(id INT, value INT, name VARCHAR)
-STORED AS PARQUET
-LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/'
-WITH ORDER (id ASC);
-
-# Enable multiple partitions
-statement ok
-SET datafusion.execution.target_partitions = 4;
-
-# Now we should see RepartitionExec because we have 3 input partitions (3 files)
-query TT
-EXPLAIN SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;
-----
-logical_plan
-01)Sort: sorted_parquet_multi.id DESC NULLS FIRST, fetch=3
-02)--TableScan: sorted_parquet_multi projection=[id, value, name]
-physical_plan
-01)SortPreservingMergeExec: [id@0 DESC], fetch=3
-02)--SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part3.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
-
-# Verify correctness with repartitioning and multiple files
-query IIT
-SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;
-----
-10 1000 j
-9 900 i
-8 800 h
-
-# Test ASC order (should not trigger reverse scan)
-query IIT
-SELECT * FROM sorted_parquet_multi ORDER BY id ASC LIMIT 3;
-----
-1 100 a
-2 200 b
-3 300 c
-
-# Cleanup
-statement ok
-DROP TABLE sorted_data_part1;
-
-statement ok
-DROP TABLE sorted_data_part2;
-
-statement ok
-DROP TABLE sorted_data_part3;
-
-statement ok
-DROP TABLE sorted_parquet_multi;
-
-# Reset to default
-statement ok
-SET datafusion.execution.target_partitions = 4;
-
-# Cleanup
-statement ok
-DROP TABLE multi_rg_data;
-
-statement ok
-DROP TABLE multi_rg_sorted;
-
-statement ok
-SET datafusion.execution.parquet.enable_page_index = false;
-
-statement ok
-SET datafusion.execution.parquet.pushdown_filters = false;
-
-# Cleanup
-statement ok
-DROP TABLE sorted_data;
-
-statement ok
-DROP TABLE sorted_parquet;
-
-statement ok
-SET datafusion.optimizer.enable_sort_pushdown = true;
diff --git a/datafusion/sqllogictest/test_files/sort_pushdown.slt b/datafusion/sqllogictest/test_files/sort_pushdown.slt
new file mode 100644
index 0000000000..58d9915a24
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/sort_pushdown.slt
@@ -0,0 +1,886 @@
+#Sort Pushdown for ordered Parquet files
+statement ok
+SET datafusion.execution.parquet.pushdown_filters = true;
+
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = true;
+
+# Test 1: Sort Pushdown for ordered Parquet files
+# Create a sorted dataset
+statement ok
+CREATE TABLE sorted_data(id INT, value INT, name VARCHAR) AS VALUES
+(1, 100, 'a'),
+(2, 200, 'b'),
+(3, 300, 'c'),
+(4, 400, 'd'),
+(5, 500, 'e'),
+(6, 600, 'f'),
+(7, 700, 'g'),
+(8, 800, 'h'),
+(9, 900, 'i'),
+(10, 1000, 'j');
+
+# Copy to parquet with sorting
+query I
+COPY (SELECT * FROM sorted_data ORDER BY id ASC)
+TO 'test_files/scratch/sort_pushdown/sorted_data.parquet';
+----
+10
+
+statement ok
+CREATE EXTERNAL TABLE sorted_parquet(id INT, value INT, name VARCHAR)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/sorted_data.parquet'
+WITH ORDER (id ASC);
+
+# Test 1.1: Sort pushdown with DESC (opposite of ASC)
+# Should show reverse_row_groups=true
+query TT
+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;
+----
+logical_plan
+01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3
+02)--TableScan: sorted_parquet projection=[id, value, name]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+
+# Test 1.2: Verify results are correct
+query IIT
+SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;
+----
+10 1000 j
+9 900 i
+8 800 h
+
+# Test 1.3: Should NOT apply for ASC (same direction)
+query TT
+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id ASC LIMIT 3;
+----
+logical_plan
+01)Sort: sorted_parquet.id ASC NULLS LAST, fetch=3
+02)--TableScan: sorted_parquet projection=[id, value, name]
+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet
+
+# Test 1.4: Disable sort pushdown
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = false;
+
+query TT
+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;
+----
+logical_plan
+01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3
+02)--TableScan: sorted_parquet projection=[id, value, name]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]
+
+# Re-enable
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = true;
+
+# Test 1.5: With OFFSET
+query TT
+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;
+----
+logical_plan
+01)Limit: skip=2, fetch=3
+02)--Sort: sorted_parquet.id DESC NULLS FIRST, fetch=5
+03)----TableScan: sorted_parquet projection=[id, value, name]
+physical_plan
+01)GlobalLimitExec: skip=2, fetch=3
+02)--SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+
+query IIT
+SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;
+----
+8 800 h
+7 700 g
+6 600 f
+
+# Test 1.6: Reverse scan with row selection (page index pruning)
+# This tests that when reverse_row_groups=true, the RowSelection is also properly reversed
+
+# Create a dataset with multiple row groups and enable page index
+statement ok
+CREATE TABLE multi_rg_data(id INT, category VARCHAR, value INT) AS VALUES
+(1, 'alpha', 10),
+(2, 'alpha', 20),
+(3, 'beta', 30),
+(4, 'beta', 40),
+(5, 'gamma', 50),
+(6, 'gamma', 60),
+(7, 'delta', 70),
+(8, 'delta', 80);
+
+# Write with small row groups (2 rows each = 4 row groups)
+statement ok
+SET datafusion.execution.parquet.max_row_group_size = 2;
+
+query I
+COPY (SELECT * FROM multi_rg_data ORDER BY id ASC)
+TO 'test_files/scratch/sort_pushdown/multi_rg_sorted.parquet';
+----
+8
+
+# Reset row group size
+statement ok
+SET datafusion.execution.parquet.max_row_group_size = 1048576;
+
+statement ok
+CREATE EXTERNAL TABLE multi_rg_sorted(id INT, category VARCHAR, value INT)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/multi_rg_sorted.parquet'
+WITH ORDER (id ASC);
+
+# Enable page index for better pruning
+statement ok
+SET datafusion.execution.parquet.enable_page_index = true;
+
+statement ok
+SET datafusion.execution.parquet.pushdown_filters = true;
+
+# Test with reverse scan and filter that prunes some row groups
+# This will create a RowSelection with partial row group scans
+query TT
+EXPLAIN SELECT * FROM multi_rg_sorted
+WHERE category IN ('alpha', 'gamma')
+ORDER BY id DESC LIMIT 5;
+----
+logical_plan
+01)Sort: multi_rg_sorted.id DESC NULLS FIRST, fetch=5
+02)--Filter: multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")
+03)----TableScan: multi_rg_sorted projection=[id, category, value], partial_filters=[multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")]
+physical_plan
+01)SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_rg_sorted.parquet]]}, projection=[id, category, value], file_type=parquet, predicate=(category@1 = alpha OR category@1 = gamma) AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=category_null_count@2 != row_count@3 AND category_min@0 <= alpha AND alpha <= category_max@1 OR category_null_count@2 != row_count@3 AND category_min@0 <= gamma AND gamma <= category_max@1, required_guarantees=[category in (alpha, gamma)]
+
+# Verify the results are correct despite reverse scanning with row selection
+# Expected: gamma values (6, 5) then alpha values (2, 1), in DESC order by id
+query ITI
+SELECT * FROM multi_rg_sorted
+WHERE category IN ('alpha', 'gamma')
+ORDER BY id DESC LIMIT 5;
+----
+6 gamma 60
+5 gamma 50
+2 alpha 20
+1 alpha 10
+
+# Test with more complex selection pattern
+query ITI
+SELECT * FROM multi_rg_sorted
+WHERE category IN ('beta', 'delta')
+ORDER BY id DESC;
+----
+8 delta 80
+7 delta 70
+4 beta 40
+3 beta 30
+
+# Test forward scan for comparison (should give same logical results in ASC order)
+query ITI
+SELECT * FROM multi_rg_sorted
+WHERE category IN ('alpha', 'gamma')
+ORDER BY id ASC;
+----
+1 alpha 10
+2 alpha 20
+5 gamma 50
+6 gamma 60
+
+# Disable reverse scan and verify it still works
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = false;
+
+query ITI
+SELECT * FROM multi_rg_sorted
+WHERE category IN ('alpha', 'gamma')
+ORDER BY id DESC LIMIT 5;
+----
+6 gamma 60
+5 gamma 50
+2 alpha 20
+1 alpha 10
+
+# Re-enable
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = true;
+
+# Test 1.7: Sort pushdown with more than one partition
+# Create multiple parquet files to trigger it
+
+# Split data into multiple files
+statement ok
+CREATE TABLE sorted_data_part1(id INT, value INT, name VARCHAR) AS VALUES
+(1, 100, 'a'),
+(2, 200, 'b'),
+(3, 300, 'c');
+
+statement ok
+CREATE TABLE sorted_data_part2(id INT, value INT, name VARCHAR) AS VALUES
+(4, 400, 'd'),
+(5, 500, 'e'),
+(6, 600, 'f');
+
+statement ok
+CREATE TABLE sorted_data_part3(id INT, value INT, name VARCHAR) AS VALUES
+(7, 700, 'g'),
+(8, 800, 'h'),
+(9, 900, 'i'),
+(10, 1000, 'j');
+
+# Create directory for multi-file parquet
+query I
+COPY (SELECT * FROM sorted_data_part1 ORDER BY id ASC)
+TO 'test_files/scratch/sort_pushdown/sorted_multi/part1.parquet';
+----
+3
+
+query I
+COPY (SELECT * FROM sorted_data_part2 ORDER BY id ASC)
+TO 'test_files/scratch/sort_pushdown/sorted_multi/part2.parquet';
+----
+3
+
+query I
+COPY (SELECT * FROM sorted_data_part3 ORDER BY id ASC)
+TO 'test_files/scratch/sort_pushdown/sorted_multi/part3.parquet';
+----
+4
+
+# Create external table pointing to directory with multiple files
+statement ok
+CREATE EXTERNAL TABLE sorted_parquet_multi(id INT, value INT, name VARCHAR)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/sorted_multi/'
+WITH ORDER (id ASC);
+
+# Enable multiple partitions
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+# Now we should see RepartitionExec because we have 3 input partitions (3 files)
+query TT
+EXPLAIN SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;
+----
+logical_plan
+01)Sort: sorted_parquet_multi.id DESC NULLS FIRST, fetch=3
+02)--TableScan: sorted_parquet_multi projection=[id, value, name]
+physical_plan
+01)SortPreservingMergeExec: [id@0 DESC], fetch=3
+02)--SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[true]
+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part3.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+
+# Verify correctness with repartitioning and multiple files
+query IIT
+SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;
+----
+10 1000 j
+9 900 i
+8 800 h
+
+# Test ASC order (should not trigger reverse scan)
+query IIT
+SELECT * FROM sorted_parquet_multi ORDER BY id ASC LIMIT 3;
+----
+1 100 a
+2 200 b
+3 300 c
+
+# Cleanup
+statement ok
+DROP TABLE sorted_data_part1;
+
+statement ok
+DROP TABLE sorted_data_part2;
+
+statement ok
+DROP TABLE sorted_data_part3;
+
+statement ok
+DROP TABLE sorted_parquet_multi;
+
+# Reset to default
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+# Cleanup
+statement ok
+DROP TABLE multi_rg_data;
+
+statement ok
+DROP TABLE multi_rg_sorted;
+
+statement ok
+SET datafusion.execution.parquet.enable_page_index = false;
+
+statement ok
+SET datafusion.execution.parquet.pushdown_filters = true;
+
+# Cleanup
+statement ok
+DROP TABLE sorted_data;
+
+statement ok
+DROP TABLE sorted_parquet;
+
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = true;
+
+
+# Test 2: Sort pushdown with constant column filtering
+# This tests the case where a leading sort column becomes constant through WHERE filtering
+
+# Create a multi-column sorted dataset (like time-series data)
+statement ok
+CREATE TABLE timeseries_data(timeframe VARCHAR, period_end INT, value DOUBLE) AS VALUES
+('daily', 1, 100.0),
+('daily', 2, 150.0),
+('daily', 3, 200.0),
+('weekly', 1, 500.0),
+('weekly', 2, 600.0),
+('weekly', 3, 700.0),
+('monthly', 1, 1000.0),
+('monthly', 2, 1100.0),
+('monthly', 3, 1200.0),
+('quarterly', 1, 5000.0),
+('quarterly', 2, 5500.0),
+('quarterly', 3, 6000.0);
+
+# Copy to parquet with multi-column sorting (timeframe ASC, period_end ASC)
+query I
+COPY (SELECT * FROM timeseries_data ORDER BY timeframe ASC, period_end ASC)
+TO 'test_files/scratch/sort_pushdown/timeseries_sorted.parquet';
+----
+12
+
+statement ok
+CREATE EXTERNAL TABLE timeseries_parquet(timeframe VARCHAR, period_end INT, value DOUBLE)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/timeseries_sorted.parquet'
+WITH ORDER (timeframe ASC, period_end ASC);
+
+# Test 2.1: Query with constant prefix filter and DESC on remaining column
+# WHERE timeframe='quarterly' makes the first sort column constant
+# ORDER BY period_end DESC should trigger reverse scan because:
+# File ordering: [timeframe ASC, period_end ASC]
+# After filtering timeframe='quarterly': effectively [period_end ASC]
+# Request: [period_end DESC] -> exact reverse!
+query TT
+EXPLAIN SELECT * FROM timeseries_parquet
+WHERE timeframe = 'quarterly'
+ORDER BY period_end DESC
+LIMIT 2;
+----
+logical_plan
+01)Sort: timeseries_parquet.period_end DESC NULLS FIRST, fetch=2
+02)--Filter: timeseries_parquet.timeframe = Utf8View("quarterly")
+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("quarterly")]
+physical_plan
+01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = quarterly AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]
+
+# Test 2.2: Verify the results are correct
+query TIR
+SELECT * FROM timeseries_parquet
+WHERE timeframe = 'quarterly'
+ORDER BY period_end DESC
+LIMIT 2;
+----
+quarterly 3 6000
+quarterly 2 5500
+
+# Test 2.3: Same filter but ASC order (should not trigger reverse scan, ordering already satisfied)
+query TT
+EXPLAIN SELECT * FROM timeseries_parquet
+WHERE timeframe = 'quarterly'
+ORDER BY period_end ASC
+LIMIT 2;
+----
+logical_plan
+01)Sort: timeseries_parquet.period_end ASC NULLS LAST, fetch=2
+02)--Filter: timeseries_parquet.timeframe = Utf8View("quarterly")
+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("quarterly")]
+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], limit=2, output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=timeframe@0 = quarterly, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]
+
+# Test 2.4: Verify ASC results
+query TIR
+SELECT * FROM timeseries_parquet
+WHERE timeframe = 'quarterly'
+ORDER BY period_end ASC
+LIMIT 2;
+----
+quarterly 1 5000
+quarterly 2 5500
+
+# Test 2.5: Test with different constant value
+query TIR
+SELECT * FROM timeseries_parquet
+WHERE timeframe = 'weekly'
+ORDER BY period_end DESC;
+----
+weekly 3 700
+weekly 2 600
+weekly 1 500
+
+# Test 2.6: Test without constant filter (no reverse scan because need both columns)
+# Request: [timeframe ASC, period_end DESC]
+# File has: [timeframe ASC, period_end ASC]
+# These are NOT reverse of each other - only second column is reversed
+query TT
+EXPLAIN SELECT * FROM timeseries_parquet
+ORDER BY timeframe ASC, period_end DESC
+LIMIT 3;
+----
+logical_plan
+01)Sort: timeseries_parquet.timeframe ASC NULLS LAST, timeseries_parquet.period_end DESC NULLS FIRST, fetch=3
+02)--TableScan: timeseries_parquet projection=[timeframe, period_end, value]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[timeframe@0 ASC NULLS LAST, period_end@1 DESC], preserve_partitioning=[false], sort_prefix=[timeframe@0 ASC NULLS LAST]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]
+
+# Test 2.7: Disable sort pushdown and verify filter still works
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = false;
+
+query TT
+EXPLAIN SELECT * FROM timeseries_parquet
+WHERE timeframe = 'quarterly'
+ORDER BY period_end DESC
+LIMIT 2;
+----
+logical_plan
+01)Sort: timeseries_parquet.period_end DESC NULLS FIRST, fetch=2
+02)--Filter: timeseries_parquet.timeframe = Utf8View("quarterly")
+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("quarterly")]
+physical_plan
+01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=timeframe@0 = quarterly AND DynamicFilter [ empty ], pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]
+
+# Results should still be correct
+query TIR
+SELECT * FROM timeseries_parquet
+WHERE timeframe = 'quarterly'
+ORDER BY period_end DESC
+LIMIT 2;
+----
+quarterly 3 6000
+quarterly 2 5500
+
+# Re-enable
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = true;
+
+# Test 2.8: Test with IN clause (multiple constant values)
+# Note: IN clause with multiple values means timeframe is NOT constant
+# (could be 'daily' or 'weekly'), so the first sort column cannot be eliminated.
+# Without a constant first column, we cannot reverse scan based on just period_end DESC.
+# The physical plan should NOT show reverse_row_groups=true
+query TT
+EXPLAIN SELECT * FROM timeseries_parquet
+WHERE timeframe IN ('daily', 'weekly')
+ORDER BY period_end DESC
+LIMIT 3;
+----
+logical_plan
+01)Sort: timeseries_parquet.period_end DESC NULLS FIRST, fetch=3
+02)--Filter: timeseries_parquet.timeframe = Utf8View("daily") OR timeseries_parquet.timeframe = Utf8View("weekly")
+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("daily") OR timeseries_parquet.timeframe = Utf8View("weekly")]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[period_end@1 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=(timeframe@0 = daily OR timeframe@0 = weekly) AND DynamicFilter [ empty ], pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= daily AND daily <= timeframe_max@1 OR timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= weekly AND weekly <= timeframe_max@1, required_guarantees=[timeframe in (daily, weekly)]
+
+# Test 2.9: Complex case - literal constant in sort expression itself
+# The literal 'constant' is ignored in sort analysis
+# After stripping: ORDER BY period_end DESC
+# With WHERE timeframe='monthly' making first column constant
+# File: [period_end ASC] (after constant column removal)
+# Request: [period_end DESC] -> exact reverse, triggers reverse scan
+query TT
+EXPLAIN SELECT * FROM timeseries_parquet
+WHERE timeframe = 'monthly'
+ORDER BY 'constant', period_end DESC
+LIMIT 2;
+----
+logical_plan
+01)Sort: Utf8("constant") ASC NULLS LAST, timeseries_parquet.period_end DESC NULLS FIRST, fetch=2
+02)--Filter: timeseries_parquet.timeframe = Utf8View("monthly")
+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("monthly")]
+physical_plan
+01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = monthly AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= monthly AND monthly <= timeframe_max@1, required_guarantees=[timeframe in (monthly)]
+
+# Verify results
+query TIR
+SELECT * FROM timeseries_parquet
+WHERE timeframe = 'monthly'
+ORDER BY period_end DESC
+LIMIT 2;
+----
+monthly 3 1200
+monthly 2 1100
+
+# Test 2.10: Filter on non-leading sort column
+# File order: [timeframe ASC, period_end ASC]
+# Filter: period_end = 2 (makes second column constant)
+# Request: [timeframe DESC]
+# After constant column removal: File has [timeframe ASC], Request wants [timeframe DESC]
+# This is exact reverse -> triggers reverse scan
+query TT
+EXPLAIN SELECT * FROM timeseries_parquet
+WHERE period_end = 2
+ORDER BY timeframe DESC;
+----
+logical_plan
+01)Sort: timeseries_parquet.timeframe DESC NULLS FIRST
+02)--Filter: timeseries_parquet.period_end = Int32(2)
+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.period_end = Int32(2)]
+physical_plan
+01)SortExec: expr=[timeframe@0 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=period_end@1 = 2, reverse_row_groups=true, pruning_predicate=period_end_null_count@2 != row_count@3 AND period_end_min@0 <= 2 AND 2 <= period_end_max@1, required_guarantees=[period_end in (2)]
+
+# Cleanup
+statement ok
+DROP TABLE timeseries_data;
+
+statement ok
+DROP TABLE timeseries_parquet;
+
+# Reset to default
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = true;
+
+
+# Test 3: Sort pushdown with monotonic functions
+# This tests that reverse scan works when sort expressions involve monotonic functions
+
+# Create test data with timestamp column
+statement ok
+CREATE TABLE timestamp_data(id INT, ts TIMESTAMP, volume BIGINT, price DOUBLE) AS VALUES
+(1, TIMESTAMP '2024-01-15 10:00:00', 1000, 100.0),
+(2, TIMESTAMP '2024-01-20 11:00:00', 1500, 105.0),
+(3, TIMESTAMP '2024-01-25 12:00:00', 2000, 110.0),
+(4, TIMESTAMP '2024-02-05 09:00:00', 1200, 108.0),
+(5, TIMESTAMP '2024-02-15 14:00:00', 1800, 112.0),
+(6, TIMESTAMP '2024-02-25 15:00:00', 2200, 115.0),
+(7, TIMESTAMP '2024-03-10 09:00:00', 1300, 113.0),
+(8, TIMESTAMP '2024-03-18 14:00:00', 1900, 118.0),
+(9, TIMESTAMP '2024-03-28 15:00:00', 2300, 120.0);
+
+# Copy to parquet with sorting by timestamp ASC
+query I
+COPY (SELECT * FROM timestamp_data ORDER BY ts ASC)
+TO 'test_files/scratch/sort_pushdown/timestamp_sorted.parquet';
+----
+9
+
+# Test 3.1: Simple monotonic function - date_trunc
+# Create external table with file ordering that conceptually includes date_trunc
+# File is actually sorted by [ts ASC], but conceptually [date_trunc('month', ts) ASC, ts ASC]
+statement ok
+CREATE EXTERNAL TABLE timestamp_parquet(id INT, ts TIMESTAMP, volume BIGINT, price DOUBLE)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/timestamp_sorted.parquet'
+WITH ORDER (ts ASC);
+
+# Query with ORDER BY ts DESC
+# File ordering: [ts ASC]
+# Request: [ts DESC]
+# This should trigger reverse_row_groups=true
+query TT
+EXPLAIN SELECT * FROM timestamp_parquet
+ORDER BY ts DESC
+LIMIT 3;
+----
+logical_plan
+01)Sort: timestamp_parquet.ts DESC NULLS FIRST, fetch=3
+02)--TableScan: timestamp_parquet projection=[id, ts, volume, price]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[ts@1 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+
+# Verify results
+query IPIR
+SELECT * FROM timestamp_parquet
+ORDER BY ts DESC
+LIMIT 3;
+----
+9 2024-03-28T15:00:00 2300 120
+8 2024-03-18T14:00:00 1900 118
+7 2024-03-10T09:00:00 1300 113
+
+# Test 3.2: Monotonic function in ORDER BY - date_trunc DESC
+# File ordering: [ts ASC]
+# Request: [date_trunc('day', ts) DESC]
+# Since date_trunc is monotonic with ts, reversed file ordering [ts DESC] satisfies [date_trunc DESC]
+query TT
+EXPLAIN SELECT * FROM timestamp_parquet
+ORDER BY date_trunc('day', ts) DESC
+LIMIT 3;
+----
+logical_plan
+01)Sort: date_trunc(Utf8("day"), timestamp_parquet.ts) DESC NULLS FIRST, fetch=3
+02)--TableScan: timestamp_parquet projection=[id, ts, volume, price]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[date_trunc(day, ts@1) DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+
+# Verify results (descending day)
+query IPIR
+SELECT * FROM timestamp_parquet
+ORDER BY date_trunc('day', ts) DESC
+LIMIT 3;
+----
+9 2024-03-28T15:00:00 2300 120
+8 2024-03-18T14:00:00 1900 118
+7 2024-03-10T09:00:00 1300 113
+
+# Test 3.3: Multi-column scenario with explicit monotonic function in file ordering
+# Create a table where we explicitly declare the ordering includes the monotonic function
+# This simulates files that are partitioned/sorted by [date_trunc('month', ts) ASC, ts ASC]
+
+# Create a new parquet file sorted by [ts ASC] (which implies date_trunc ordering)
+statement ok
+CREATE TABLE multi_month_data(id INT, ts TIMESTAMP, volume BIGINT, price DOUBLE) AS VALUES
+-- January 2024
+(1, TIMESTAMP '2024-01-05 09:30:00', 1000, 100.0),
+(2, TIMESTAMP '2024-01-15 14:30:00', 1500, 105.0),
+(3, TIMESTAMP '2024-01-25 15:59:00', 2000, 110.0),
+-- February 2024
+(4, TIMESTAMP '2024-02-03 09:30:00', 1200, 108.0),
+(5, TIMESTAMP '2024-02-14 12:00:00', 1800, 112.0),
+(6, TIMESTAMP '2024-02-28 15:59:00', 2200, 115.0),
+-- March 2024
+(7, TIMESTAMP '2024-03-01 09:30:00', 1300, 113.0),
+(8, TIMESTAMP '2024-03-15 14:00:00', 1900, 118.0),
+(9, TIMESTAMP '2024-03-29 15:59:00', 2300, 120.0);
+
+query I
+COPY (SELECT * FROM multi_month_data ORDER BY ts ASC)
+TO 'test_files/scratch/sort_pushdown/multi_month_sorted.parquet';
+----
+9
+
+# Declare the file has ordering [ts ASC]
+# Conceptually this means [date_trunc('month', ts) ASC, ts ASC] due to monotonicity
+statement ok
+CREATE EXTERNAL TABLE multi_month_parquet(id INT, ts TIMESTAMP, volume BIGINT, price DOUBLE)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/multi_month_sorted.parquet'
+WITH ORDER (ts ASC);
+
+# Test 3.3a: Request ORDER BY ts DESC (opposite direction)
+# File: [ts ASC]
+# Request: [ts DESC]
+# Should trigger reverse_row_groups=true
+query TT
+EXPLAIN SELECT * FROM multi_month_parquet
+ORDER BY ts DESC
+LIMIT 2;
+----
+logical_plan
+01)Sort: multi_month_parquet.ts DESC NULLS FIRST, fetch=2
+02)--TableScan: multi_month_parquet projection=[id, ts, volume, price]
+physical_plan
+01)SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+
+query IPIR
+SELECT * FROM multi_month_parquet
+ORDER BY ts DESC
+LIMIT 2;
+----
+9 2024-03-29T15:59:00 2300 120
+8 2024-03-15T14:00:00 1900 118
+
+# Test 3.3b: Request ORDER BY date_trunc('month', ts) DESC, ts DESC
+# File: [ts ASC] (which implies [date_trunc('month', ts) ASC, ts ASC])
+# Request: [date_trunc('month', ts) DESC, ts DESC]
+# The reversed file ordering [ts DESC] satisfies this because:
+# - date_trunc is monotonic with ts
+# - So [ts DESC] implies [date_trunc('month', ts) DESC, ts DESC]
+query TT
+EXPLAIN SELECT * FROM multi_month_parquet
+ORDER BY date_trunc('month', ts) DESC, ts DESC
+LIMIT 2;
+----
+logical_plan
+01)Sort: date_trunc(Utf8("month"), multi_month_parquet.ts) DESC NULLS FIRST, multi_month_parquet.ts DESC NULLS FIRST, fetch=2
+02)--TableScan: multi_month_parquet projection=[id, ts, volume, price]
+physical_plan
+01)SortExec: TopK(fetch=2), expr=[date_trunc(month, ts@1) DESC, ts@1 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+
+query IPIR
+SELECT * FROM multi_month_parquet
+ORDER BY date_trunc('month', ts) DESC, ts DESC
+LIMIT 2;
+----
+9 2024-03-29T15:59:00 2300 120
+8 2024-03-15T14:00:00 1900 118
+
+# Test 3.4: CAST as a monotonic function
+statement ok
+CREATE TABLE int_data(id INT, small_val SMALLINT, big_val BIGINT) AS VALUES
+(1, 10, 100),
+(2, 20, 200),
+(3, 30, 300),
+(4, 40, 400),
+(5, 50, 500);
+
+query I
+COPY (SELECT * FROM int_data ORDER BY small_val ASC)
+TO 'test_files/scratch/sort_pushdown/int_sorted.parquet';
+----
+5
+
+statement ok
+CREATE EXTERNAL TABLE int_parquet(id INT, small_val SMALLINT, big_val BIGINT)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/int_sorted.parquet'
+WITH ORDER (small_val ASC);
+
+# CAST preserves ordering: CAST(small_val AS BIGINT) is monotonic with small_val
+query TT
+EXPLAIN SELECT * FROM int_parquet
+ORDER BY CAST(small_val AS BIGINT) DESC
+LIMIT 2;
+----
+logical_plan
+01)Sort: CAST(int_parquet.small_val AS Int64) DESC NULLS FIRST, fetch=2
+02)--TableScan: int_parquet projection=[id, small_val, big_val]
+physical_plan
+01)SortExec: TopK(fetch=2), expr=[CAST(small_val@1 AS Int64) DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/int_sorted.parquet]]}, projection=[id, small_val, big_val], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+
+query III
+SELECT * FROM int_parquet
+ORDER BY CAST(small_val AS BIGINT) DESC
+LIMIT 2;
+----
+5 50 500
+4 40 400
+
+# Test 3.5: CEIL as a monotonic function
+statement ok
+CREATE TABLE float_data(id INT, value DOUBLE) AS VALUES
+(1, 1.1),
+(2, 2.3),
+(3, 3.5),
+(4, 4.7),
+(5, 5.9);
+
+query I
+COPY (SELECT * FROM float_data ORDER BY value ASC)
+TO 'test_files/scratch/sort_pushdown/float_sorted.parquet';
+----
+5
+
+statement ok
+CREATE EXTERNAL TABLE float_parquet(id INT, value DOUBLE)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/float_sorted.parquet'
+WITH ORDER (value ASC);
+
+# CEIL is monotonic increasing
+query TT
+EXPLAIN SELECT * FROM float_parquet
+ORDER BY CEIL(value) DESC
+LIMIT 3;
+----
+logical_plan
+01)Sort: ceil(float_parquet.value) DESC NULLS FIRST, fetch=3
+02)--TableScan: float_parquet projection=[id, value]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[ceil(value@1) DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/float_sorted.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+
+query IR
+SELECT * FROM float_parquet
+ORDER BY CEIL(value) DESC
+LIMIT 3;
+----
+5 5.9
+4 4.7
+3 3.5
+
+# Test 3.6: Negative case - ABS is NOT monotonic over mixed positive/negative range
+statement ok
+CREATE TABLE signed_data(id INT, value DOUBLE) AS VALUES
+(1, -5.0),
+(2, -3.0),
+(3, -1.0),
+(4, 2.0),
+(5, 4.0);
+
+query I
+COPY (SELECT * FROM signed_data ORDER BY value ASC)
+TO 'test_files/scratch/sort_pushdown/signed_sorted.parquet';
+----
+5
+
+statement ok
+CREATE EXTERNAL TABLE signed_parquet(id INT, value DOUBLE)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/signed_sorted.parquet'
+WITH ORDER (value ASC);
+
+# ABS is NOT monotonic over the full range [-5, 4], so should NOT trigger reverse scan
+query TT
+EXPLAIN SELECT * FROM signed_parquet
+ORDER BY ABS(value) DESC
+LIMIT 3;
+----
+logical_plan
+01)Sort: abs(signed_parquet.value) DESC NULLS FIRST, fetch=3
+02)--TableScan: signed_parquet projection=[id, value]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[abs(value@1) DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/signed_sorted.parquet]]}, projection=[id, value], output_ordering=[value@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]
+
+# Results should still be correct (no optimization applied)
+query IR
+SELECT * FROM signed_parquet
+ORDER BY ABS(value) DESC
+LIMIT 3;
+----
+1 -5
+5 4
+2 -3
+
+# Cleanup
+statement ok
+DROP TABLE timestamp_data;
+
+statement ok
+DROP TABLE timestamp_parquet;
+
+statement ok
+DROP TABLE multi_month_data;
+
+statement ok
+DROP TABLE multi_month_parquet;
+
+statement ok
+DROP TABLE int_data;
+
+statement ok
+DROP TABLE int_parquet;
+
+statement ok
+DROP TABLE float_data;
+
+statement ok
+DROP TABLE float_parquet;
+
+statement ok
+DROP TABLE signed_data;
+
+statement ok
+DROP TABLE signed_parquet;
+
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = true;