Apache DataFusion SQL Query Engine
Redesign the try_reverse_output to support more cases (#19446)
Qi Zhu10db6b3712af
diff --git a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs b/datafusion/core/tests/physical_optimizer/pushdown_sort.rsindex 1d31374076..caef0fba05 100644--- a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs+++ b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs@@ -24,14 +24,18 @@//! 4. Early termination is enabled for TopK queries//! 5. Prefix matching works correctly+use datafusion_physical_expr::expressions;+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::pushdown_sort::PushdownSort;+use std::sync::Arc;use crate::physical_optimizer::test_utils::{OptimizationTest, coalesce_batches_exec, coalesce_partitions_exec, parquet_exec,- parquet_exec_with_sort, repartition_exec, schema, sort_exec, sort_exec_with_fetch,- sort_expr,+ parquet_exec_with_sort, projection_exec, projection_exec_with_alias,+ repartition_exec, schema, simple_projection_exec, sort_exec, sort_exec_with_fetch,+ sort_expr, sort_expr_named, test_scan_with_ordering,};#[test]@@ -619,16 +623,14 @@ fn test_pushdown_through_blocking_node() {// Middle: Aggregate (blocks pushdown from outer sort)// GROUP BY a, COUNT(b)let group_by = PhysicalGroupBy::new_single(vec![(- Arc::new(datafusion_physical_expr::expressions::Column::new("a", 0)) as _,+ Arc::new(expressions::Column::new("a", 0)) as _,"a".to_string(),)]);let count_expr = Arc::new(AggregateExprBuilder::new(count_udaf(),- vec![- Arc::new(datafusion_physical_expr::expressions::Column::new("b", 1)) as _,- ],+ vec![Arc::new(expressions::Column::new("b", 1)) as _],).schema(Arc::clone(&schema)).alias("COUNT(b)")@@ -670,3 +672,369 @@ fn test_pushdown_through_blocking_node() {");}++// ============================================================================+// PROJECTION TESTS+// ============================================================================++#[test]+fn test_sort_pushdown_through_simple_projection() {+ // Sort pushes through projection with simple column references+ let schema = schema();++ // Source has [a ASC] ordering+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Projection: SELECT a, b (simple column references)+ let projection = simple_projection_exec(source, vec![0, 1]); // columns a, b++ // Request [a DESC] - should push through projection to source+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, projection);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_sort_pushdown_through_projection_with_alias() {+ // Sort pushes through projection with column aliases+ let schema = schema();++ // Source has [a ASC] ordering+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Projection: SELECT a AS id, b AS value+ let projection = projection_exec_with_alias(source, vec![(0, "id"), (1, "value")]);++ // Request [id DESC] - should map to [a DESC] and push down+ let id_expr = sort_expr_named("id", 0);+ let desc_ordering = LexOrdering::new(vec![id_expr.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, projection);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[id@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as id, b@1 as value]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[id@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as id, b@1 as value]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_no_sort_pushdown_through_computed_projection() {+ use datafusion_expr::Operator;++ // Sort should NOT push through projection with computed columns+ let schema = schema();++ // Source has [a ASC] ordering+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Projection: SELECT a+b as sum, c+ let projection = projection_exec(+ vec![+ (+ Arc::new(expressions::BinaryExpr::new(+ Arc::new(expressions::Column::new("a", 0)),+ Operator::Plus,+ Arc::new(expressions::Column::new("b", 1)),+ )) as Arc<dyn PhysicalExpr>,+ "sum".to_string(),+ ),+ (+ Arc::new(expressions::Column::new("c", 2)) as Arc<dyn PhysicalExpr>,+ "c".to_string(),+ ),+ ],+ source,+ )+ .unwrap();++ // Request [sum DESC] - should NOT push down (sum is computed)+ let sum_expr = sort_expr_named("sum", 0);+ let desc_ordering = LexOrdering::new(vec![sum_expr.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, projection);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[sum@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 + b@1 as sum, c@2 as c]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[sum@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 + b@1 as sum, c@2 as c]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ "+ );+}++#[test]+fn test_sort_pushdown_projection_reordered_columns() {+ // Sort pushes through projection that reorders columns+ let schema = schema();++ // Source has [a ASC] ordering+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Projection: SELECT c, b, a (columns reordered)+ let projection = simple_projection_exec(source, vec![2, 1, 0]); // c, b, a++ // Request [a DESC] where a is now at index 2 in projection output+ let a_expr_at_2 = sort_expr_named("a", 2);+ let desc_ordering = LexOrdering::new(vec![a_expr_at_2.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, projection);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@2 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@2 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_sort_pushdown_projection_with_limit() {+ // Sort with LIMIT pushes through simple projection+ let schema = schema();++ // Source has [a ASC] ordering+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Projection: SELECT a, b+ let projection = simple_projection_exec(source, vec![0, 1]);++ // Request [a DESC] with LIMIT 10+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec_with_fetch(desc_ordering, Some(10), projection);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_sort_pushdown_through_projection_and_coalesce() {+ // Sort pushes through both projection and coalesce batches+ let schema = schema();++ // Source has [a ASC] ordering+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ let coalesce = coalesce_batches_exec(source, 1024);++ // Projection: SELECT a, b+ let projection = simple_projection_exec(coalesce, vec![0, 1]);++ // Request [a DESC]+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, projection);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a, b@1 as b]+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a, b@1 as b]+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_sort_pushdown_projection_subset_of_columns() {+ // Sort pushes through projection that selects subset of columns+ let schema = schema();++ // Source has [a ASC, b ASC] ordering+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone(), b.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Projection: SELECT a (subset of columns)+ let projection = simple_projection_exec(source, vec![0]);++ // Request [a DESC]+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, projection);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC, b@1 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - ProjectionExec: expr=[a@0 as a]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++// ============================================================================+// TESTSCAN DEMONSTRATION TESTS+// ============================================================================+// These tests use TestScan to demonstrate how sort pushdown works more clearly+// than ParquetExec. TestScan can accept ANY ordering (not just reverse) and+// displays the requested ordering explicitly in the output.++#[test]+fn test_sort_pushdown_with_test_scan_basic() {+ // Demonstrates TestScan showing requested ordering clearly+ let schema = schema();++ // Source has [a ASC] ordering+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = test_scan_with_ordering(schema.clone(), source_ordering);++ // Request [a DESC] ordering+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - TestScan: output_ordering=[a@0 ASC]+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - TestScan: output_ordering=[a@0 ASC], requested_ordering=[a@0 DESC NULLS LAST]+ "+ );+}++#[test]+fn test_sort_pushdown_with_test_scan_multi_column() {+ // Demonstrates TestScan with multi-column ordering+ let schema = schema();++ // Source has [a ASC, b DESC] ordering+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone(), b.clone().reverse()]).unwrap();+ let source = test_scan_with_ordering(schema.clone(), source_ordering);++ // Request [a DESC, b ASC] ordering (reverse of source)+ let reverse_ordering = LexOrdering::new(vec![a.reverse(), b]).unwrap();+ let plan = sort_exec(reverse_ordering, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST, b@1 ASC], preserve_partitioning=[false]+ - TestScan: output_ordering=[a@0 ASC, b@1 DESC NULLS LAST]+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST, b@1 ASC], preserve_partitioning=[false]+ - TestScan: output_ordering=[a@0 ASC, b@1 DESC NULLS LAST], requested_ordering=[a@0 DESC NULLS LAST, b@1 ASC]+ "+ );+}++#[test]+fn test_sort_pushdown_with_test_scan_arbitrary_ordering() {+ // Demonstrates that TestScan can accept ANY ordering (not just reverse)+ // This is different from ParquetExec which only supports reverse scans+ let schema = schema();++ // Source has [a ASC, b ASC] ordering+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone(), b.clone()]).unwrap();+ let source = test_scan_with_ordering(schema.clone(), source_ordering);++ // Request [a ASC, b DESC] - NOT a simple reverse, but TestScan accepts it+ let mixed_ordering = LexOrdering::new(vec![a, b.reverse()]).unwrap();+ let plan = sort_exec(mixed_ordering, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]+ - TestScan: output_ordering=[a@0 ASC, b@1 ASC]+ output:+ Ok:+ - SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]+ - TestScan: output_ordering=[a@0 ASC, b@1 ASC], requested_ordering=[a@0 ASC, b@1 DESC NULLS LAST]+ "+ );+}diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex c81ccb6310..5b50181d7f 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -41,6 +41,7 @@ use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::{WindowFrame, WindowFunctionDefinition};use datafusion_functions_aggregate::count::count_udaf;+use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};use datafusion_physical_expr::expressions::{self, col};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;@@ -54,6 +55,7 @@ use datafusion_physical_plan::aggregates::{};use datafusion_physical_plan::coalesce_batches::CoalesceBatchesExec;use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;+use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};use datafusion_physical_plan::filter::FilterExec;use datafusion_physical_plan::joins::utils::{JoinFilter, JoinOn};use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode, SortMergeJoinExec};@@ -68,7 +70,7 @@ use datafusion_physical_plan::union::UnionExec;use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_window_expr};use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, InputOrderMode, Partitioning,- PlanProperties, displayable,+ PlanProperties, SortOrderPushdownResult, displayable,};/// Create a non sorted parquet exec@@ -774,3 +776,206 @@ pub fn format_execution_plan(plan: &Arc<dyn ExecutionPlan>) -> Vec<String> {fn format_lines(s: &str) -> Vec<String> {s.trim().split('\n').map(|s| s.to_string()).collect()}++/// Create a simple ProjectionExec with column indices (simplified version)+pub fn simple_projection_exec(+ input: Arc<dyn ExecutionPlan>,+ columns: Vec<usize>,+) -> Arc<dyn ExecutionPlan> {+ let schema = input.schema();+ let exprs: Vec<(Arc<dyn PhysicalExpr>, String)> = columns+ .iter()+ .map(|&i| {+ let field = schema.field(i);+ (+ Arc::new(expressions::Column::new(field.name(), i))+ as Arc<dyn PhysicalExpr>,+ field.name().to_string(),+ )+ })+ .collect();++ projection_exec(exprs, input).unwrap()+}++/// Create a ProjectionExec with column aliases+pub fn projection_exec_with_alias(+ input: Arc<dyn ExecutionPlan>,+ columns: Vec<(usize, &str)>,+) -> Arc<dyn ExecutionPlan> {+ let schema = input.schema();+ let exprs: Vec<(Arc<dyn PhysicalExpr>, String)> = columns+ .iter()+ .map(|&(i, alias)| {+ (+ Arc::new(expressions::Column::new(schema.field(i).name(), i))+ as Arc<dyn PhysicalExpr>,+ alias.to_string(),+ )+ })+ .collect();++ projection_exec(exprs, input).unwrap()+}++/// Create a sort expression with custom name and index+pub fn sort_expr_named(name: &str, index: usize) -> PhysicalSortExpr {+ PhysicalSortExpr {+ expr: Arc::new(expressions::Column::new(name, index)),+ options: SortOptions::default(),+ }+}++/// A test data source that can display any requested ordering+/// This is useful for testing sort pushdown behavior+#[derive(Debug, Clone)]+pub struct TestScan {+ schema: SchemaRef,+ output_ordering: Vec<LexOrdering>,+ plan_properties: PlanProperties,+ // Store the requested ordering for display+ requested_ordering: Option<LexOrdering>,+}++impl TestScan {+ /// Create a new TestScan with the given schema and output ordering+ pub fn new(schema: SchemaRef, output_ordering: Vec<LexOrdering>) -> Self {+ let eq_properties = if !output_ordering.is_empty() {+ // Convert Vec<LexOrdering> to the format expected by new_with_orderings+ // We need to extract the inner Vec<PhysicalSortExpr> from each LexOrdering+ let orderings: Vec<Vec<PhysicalSortExpr>> = output_ordering+ .iter()+ .map(|lex_ordering| {+ // LexOrdering implements IntoIterator, so we can collect it+ lex_ordering.iter().cloned().collect()+ })+ .collect();++ EquivalenceProperties::new_with_orderings(Arc::clone(&schema), orderings)+ } else {+ EquivalenceProperties::new(Arc::clone(&schema))+ };++ let plan_properties = PlanProperties::new(+ eq_properties,+ Partitioning::UnknownPartitioning(1),+ EmissionType::Incremental,+ Boundedness::Bounded,+ );++ Self {+ schema,+ output_ordering,+ plan_properties,+ requested_ordering: None,+ }+ }++ /// Create a TestScan with a single output ordering+ pub fn with_ordering(schema: SchemaRef, ordering: LexOrdering) -> Self {+ Self::new(schema, vec![ordering])+ }+}++impl DisplayAs for TestScan {+ fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {+ match t {+ DisplayFormatType::Default | DisplayFormatType::Verbose => {+ write!(f, "TestScan")?;+ if !self.output_ordering.is_empty() {+ write!(f, ": output_ordering=[")?;+ // Format the ordering in a readable way+ for (i, sort_expr) in self.output_ordering[0].iter().enumerate() {+ if i > 0 {+ write!(f, ", ")?;+ }+ write!(f, "{sort_expr}")?;+ }+ write!(f, "]")?;+ }+ // This is the key part - show what ordering was requested+ if let Some(ref req) = self.requested_ordering {+ write!(f, ", requested_ordering=[")?;+ for (i, sort_expr) in req.iter().enumerate() {+ if i > 0 {+ write!(f, ", ")?;+ }+ write!(f, "{sort_expr}")?;+ }+ write!(f, "]")?;+ }+ Ok(())+ }+ DisplayFormatType::TreeRender => {+ write!(f, "TestScan")+ }+ }+ }+}++impl ExecutionPlan for TestScan {+ fn name(&self) -> &str {+ "TestScan"+ }++ fn as_any(&self) -> &dyn Any {+ self+ }++ fn properties(&self) -> &PlanProperties {+ &self.plan_properties+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ vec![]+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ if children.is_empty() {+ Ok(self)+ } else {+ internal_err!("TestScan should have no children")+ }+ }++ fn execute(+ &self,+ _partition: usize,+ _context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ internal_err!("TestScan is for testing optimizer only, not for execution")+ }++ fn partition_statistics(&self, _partition: Option<usize>) -> Result<Statistics> {+ Ok(Statistics::new_unknown(&self.schema))+ }++ // This is the key method - implement sort pushdown+ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {+ // For testing purposes, accept ANY ordering request+ // and create a new TestScan that shows what was requested+ let requested_ordering = LexOrdering::new(order.to_vec());++ let mut new_scan = self.clone();+ new_scan.requested_ordering = requested_ordering;++ // Always return Inexact to keep the Sort node (like Phase 1 behavior)+ Ok(SortOrderPushdownResult::Inexact {+ inner: Arc::new(new_scan),+ })+ }+}++/// Helper function to create a TestScan with ordering+pub fn test_scan_with_ordering(+ schema: SchemaRef,+ ordering: LexOrdering,+) -> Arc<dyn ExecutionPlan> {+ Arc::new(TestScan::with_ordering(schema, ordering))+}diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 7d7662a4a9..2e0919b144 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -38,8 +38,8 @@ use datafusion_common::config::TableParquetOptions;use datafusion_datasource::TableSchema;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;-use datafusion_physical_expr::conjunction;use datafusion_physical_expr::projection::ProjectionExprs;+use datafusion_physical_expr::{EquivalenceProperties, conjunction};use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::physical_expr::fmt_sql;@@ -54,7 +54,7 @@ use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;#[cfg(feature = "parquet_encryption")]use datafusion_execution::parquet_encryption::EncryptionFactory;-use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use itertools::Itertools;use object_store::ObjectStore;#[cfg(feature = "parquet_encryption")]@@ -288,9 +288,6 @@ pub struct ParquetSource {pub(crate) projection: ProjectionExprs,#[cfg(feature = "parquet_encryption")]pub(crate) encryption_factory: Option<Arc<dyn EncryptionFactory>>,- /// The ordering of data within the files- /// This is set by FileScanConfig when it knows the file ordering- file_ordering: Option<LexOrdering>,/// If true, read files in reverse order and reverse row groups within files./// But it's not guaranteed that rows within row groups are in reverse order,/// so we still need to sort them after reading, so the reverse scan is inexact.@@ -320,7 +317,6 @@ impl ParquetSource {metadata_size_hint: None,#[cfg(feature = "parquet_encryption")]encryption_factory: None,- file_ordering: None,reverse_row_groups: false,}}@@ -397,12 +393,6 @@ impl ParquetSource {self}- /// If set, indicates the ordering of data within the files being read.- pub fn with_file_ordering(mut self, ordering: Option<LexOrdering>) -> Self {- self.file_ordering = ordering;- self- }-/// Return the value described in [`Self::with_pushdown_filters`]pub(crate) fn pushdown_filters(&self) -> bool {self.table_parquet_options.global.pushdown_filters@@ -769,44 +759,61 @@ impl FileSource for ParquetSource {fn try_reverse_output(&self,order: &[PhysicalSortExpr],+ eq_properties: &EquivalenceProperties,) -> datafusion_common::Result<SortOrderPushdownResult<Arc<dyn FileSource>>> {- // Check if we have file ordering information- let file_ordering = match &self.file_ordering {- Some(ordering) => ordering,- None => return Ok(SortOrderPushdownResult::Unsupported),- };-- // Create a LexOrdering from the requested order to use the is_reverse method- let Some(requested_ordering) = LexOrdering::new(order.to_vec()) else {- // Empty ordering requested, cannot optimize+ if order.is_empty() {return Ok(SortOrderPushdownResult::Unsupported);- };+ }- // Check if reversing the file ordering would satisfy the requested ordering- if file_ordering.is_reverse(&requested_ordering) {- // Phase 1: Enable reverse row group scanning- let new_source = self.clone().with_reverse_row_groups(true);+ // Build new equivalence properties with the reversed ordering.+ // This allows us to check if the reversed ordering satisfies the request+ // by leveraging:+ // - Function monotonicity (e.g., extract_year_month preserves ordering)+ // - Constant columns (from filters)+ // - Other equivalence relationships+ //+ // Example flow:+ // 1. File ordering: [extract_year_month(ws) DESC, ws DESC]+ // 2. After reversal: [extract_year_month(ws) ASC, ws ASC]+ // 3. Requested: [ws ASC]+ // 4. Through extract_year_month's monotonicity property, the reversed+ // ordering satisfies [ws ASC] even though it has additional prefix+ let reversed_eq_properties = {+ let mut new = eq_properties.clone();+ new.clear_orderings();++ // Reverse each ordering in the equivalence properties+ let reversed_orderings = eq_properties+ .oeq_class()+ .iter()+ .map(|ordering| {+ ordering+ .iter()+ .map(|expr| expr.reverse())+ .collect::<Vec<_>>()+ })+ .collect::<Vec<_>>();++ new.add_orderings(reversed_orderings);+ new+ };- // Return Inexact because we're only reversing row group order,- // not guaranteeing perfect row-level ordering- return Ok(SortOrderPushdownResult::Inexact {- inner: Arc::new(new_source) as Arc<dyn FileSource>,- });+ // Check if the reversed ordering satisfies the requested ordering+ if !reversed_eq_properties.ordering_satisfy(order.iter().cloned())? {+ return Ok(SortOrderPushdownResult::Unsupported);}+ // Return Inexact because we're only reversing row group order,+ // not guaranteeing perfect row-level ordering+ let new_source = self.clone().with_reverse_row_groups(true);+ Ok(SortOrderPushdownResult::Inexact {+ inner: Arc::new(new_source) as Arc<dyn FileSource>,+ })+// TODO Phase 2: Add support for other optimizations:// - File reordering based on min/max statistics// - Detection of exact ordering (return Exact to remove Sort operator)// - Partial sort pushdown for prefix matches-- Ok(SortOrderPushdownResult::Unsupported)- }-- fn with_file_ordering_info(- &self,- ordering: Option<LexOrdering>,- ) -> datafusion_common::Result<Arc<dyn FileSource>> {- Ok(Arc::new(self.clone().with_file_ordering(ordering)))}}diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex 4aac3d46d4..f5380c27ec 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -30,7 +30,7 @@ use crate::schema_adapter::SchemaAdapterFactory;use datafusion_common::config::ConfigOptions;use datafusion_common::{Result, not_impl_err};use datafusion_physical_expr::projection::ProjectionExprs;-use datafusion_physical_expr::{LexOrdering, PhysicalExpr};+use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, PhysicalExpr};use datafusion_physical_plan::DisplayFormatType;use datafusion_physical_plan::SortOrderPushdownResult;use datafusion_physical_plan::filter_pushdown::{FilterPushdownPropagation, PushedDown};@@ -150,15 +150,50 @@ pub trait FileSource: Send + Sync {/// Try to create a new FileSource that can produce data in the specified sort order.///+ /// This method attempts to optimize data retrieval to match the requested ordering.+ /// It receives both the requested ordering and equivalence properties that describe+ /// the output data from this file source.+ ///+ /// # Parameters+ /// * `order` - The requested sort ordering from the query+ /// * `eq_properties` - Equivalence properties of the data that will be produced by this+ /// file source. These properties describe the ordering, constant columns, and other+ /// relationships in the output data, allowing the implementation to determine if+ /// optimizations like reversed scanning can help satisfy the requested ordering.+ /// This includes information about:+ /// - The file's natural ordering (from output_ordering in FileScanConfig)+ /// - Constant columns (e.g., from filters like `ticker = 'AAPL'`)+ /// - Monotonic functions (e.g., `extract_year_month(timestamp)`)+ /// - Other equivalence relationships+ ///+ /// # Examples+ ///+ /// ## Example 1: Simple reverse+ /// ```text+ /// File ordering: [a ASC, b DESC]+ /// Requested: [a DESC]+ /// Reversed file: [a DESC, b ASC]+ /// Result: Satisfies request (prefix match) → Inexact+ /// ```+ ///+ /// ## Example 2: Monotonic function+ /// ```text+ /// File ordering: [extract_year_month(ts) ASC, ts ASC]+ /// Requested: [ts DESC]+ /// Reversed file: [extract_year_month(ts) DESC, ts DESC]+ /// Result: Through monotonicity, satisfies [ts DESC] → Inexact+ /// ```+ ////// # Returns/// * `Exact` - Created a source that guarantees perfect ordering- /// * `Inexact` - Created a source optimized for ordering (e.g., reordered files) but not perfectly sorted+ /// * `Inexact` - Created a source optimized for ordering (e.g., reversed row groups) but not perfectly sorted/// * `Unsupported` - Cannot optimize for this ordering////// Default implementation returns `Unsupported`.fn try_reverse_output(&self,_order: &[PhysicalSortExpr],+ _eq_properties: &EquivalenceProperties,) -> Result<SortOrderPushdownResult<Arc<dyn FileSource>>> {Ok(SortOrderPushdownResult::Unsupported)}@@ -192,24 +227,6 @@ pub trait FileSource: Send + Sync {Ok(None)}- /// Set the file ordering information- ///- /// This allows the file source to know how the files are sorted,- /// enabling it to make informed decisions about sort pushdown.- ///- /// # Default Implementation- ///- /// Returns `not_impl_err!`. FileSource implementations that support- /// sort optimization should override this method.- fn with_file_ordering_info(- &self,- _ordering: Option<LexOrdering>,- ) -> Result<Arc<dyn FileSource>> {- // Default: clone self without modification- // ParquetSource will override this- not_impl_err!("with_file_ordering_info not implemented for this FileSource")- }-/// Deprecated: Set optional schema adapter factory.////// `SchemaAdapterFactory` has been removed. Use `PhysicalExprAdapterFactory` instead.diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex b17b5fba68..c8636343cc 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -829,23 +829,10 @@ impl DataSource for FileScanConfig {&self,order: &[PhysicalSortExpr],) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {- let file_ordering = self.output_ordering.first().cloned();-- if file_ordering.is_none() {- return Ok(SortOrderPushdownResult::Unsupported);- }-- // Use the trait method instead of downcasting- // Try to provide file ordering info to the source- // If not supported (e.g., CsvSource), fall back to original source- let file_source_with_ordering = self+ // Delegate to FileSource to check if reverse scanning can satisfy the request.+ let pushdown_result = self.file_source- .with_file_ordering_info(file_ordering)- .unwrap_or_else(|_| Arc::clone(&self.file_source));-- // Try to reverse the datasource with ordering info,- // and currently only ParquetSource supports it with inexact reverse with row groups.- let pushdown_result = file_source_with_ordering.try_reverse_output(order)?;+ .try_reverse_output(order, &self.eq_properties())?;match pushdown_result {SortOrderPushdownResult::Exact { inner } => {diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rsindex d929c78850..a1fad86777 100644--- a/datafusion/physical-plan/src/coop.rs+++ b/datafusion/physical-plan/src/coop.rs@@ -82,7 +82,7 @@ use crate::filter_pushdown::{use crate::projection::ProjectionExec;use crate::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, RecordBatchStream,- SendableRecordBatchStream,+ SendableRecordBatchStream, SortOrderPushdownResult,};use arrow::record_batch::RecordBatch;use arrow_schema::Schema;@@ -91,6 +91,7 @@ use datafusion_execution::TaskContext;use crate::execution_plan::SchedulingType;use crate::stream::RecordBatchStreamAdapter;+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use futures::{Stream, StreamExt};/// A stream that passes record batches through unchanged while cooperating with the Tokio runtime.@@ -328,6 +329,27 @@ impl ExecutionPlan for CooperativeExec {) -> Result<FilterPushdownPropagation<Arc<dyn ExecutionPlan>>> {Ok(FilterPushdownPropagation::if_all(child_pushdown_result))}++ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {+ let child = self.input();++ match child.try_pushdown_sort(order)? {+ SortOrderPushdownResult::Exact { inner } => {+ let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;+ Ok(SortOrderPushdownResult::Exact { inner: new_exec })+ }+ SortOrderPushdownResult::Inexact { inner } => {+ let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;+ Ok(SortOrderPushdownResult::Inexact { inner: new_exec })+ }+ SortOrderPushdownResult::Unsupported => {+ Ok(SortOrderPushdownResult::Unsupported)+ }+ }+ }}/// Creates a [`CooperativeStream`] wrapper around the given [`RecordBatchStream`].diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex a56e9272f1..135f78677f 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -24,7 +24,7 @@ use super::expressions::{Column, Literal};use super::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};use super::{DisplayAs, ExecutionPlanProperties, PlanProperties, RecordBatchStream,- SendableRecordBatchStream, Statistics,+ SendableRecordBatchStream, SortOrderPushdownResult, Statistics,};use crate::execution_plan::CardinalityEffect;use crate::filter_pushdown::{@@ -51,7 +51,9 @@ use datafusion_physical_expr::equivalence::ProjectionMapping;use datafusion_physical_expr::projection::Projector;use datafusion_physical_expr::utils::collect_columns;use datafusion_physical_expr_common::physical_expr::{PhysicalExprRef, fmt_sql};-use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement};+use datafusion_physical_expr_common::sort_expr::{+ LexOrdering, LexRequirement, PhysicalSortExpr,+};// Re-exported from datafusion-physical-expr for backwards compatibility// We recommend updating your imports to use datafusion-physical-expr directlypub use datafusion_physical_expr::projection::{@@ -357,6 +359,72 @@ impl ExecutionPlan for ProjectionExec {) -> Result<FilterPushdownPropagation<Arc<dyn ExecutionPlan>>> {Ok(FilterPushdownPropagation::if_all(child_pushdown_result))}++ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {+ let child = self.input();+ let mut child_order = Vec::new();++ // Check and transform sort expressions+ for sort_expr in order {+ // Recursively transform the expression+ let mut can_pushdown = true;+ let transformed = Arc::clone(&sort_expr.expr).transform(|expr| {+ if let Some(col) = expr.as_any().downcast_ref::<Column>() {+ // Check if column index is valid.+ // This should always be true but fail gracefully if it's not.+ if col.index() >= self.expr().len() {+ can_pushdown = false;+ return Ok(Transformed::no(expr));+ }++ let proj_expr = &self.expr()[col.index()];++ // Check if projection expression is a simple column+ // We cannot push down order by clauses that depend on+ // projected computations as they would have nothing to reference.+ if let Some(child_col) =+ proj_expr.expr.as_any().downcast_ref::<Column>()+ {+ // Replace with the child column+ Ok(Transformed::yes(Arc::new(child_col.clone()) as _))+ } else {+ // Projection involves computation, cannot push down+ can_pushdown = false;+ Ok(Transformed::no(expr))+ }+ } else {+ Ok(Transformed::no(expr))+ }+ })?;++ if !can_pushdown {+ return Ok(SortOrderPushdownResult::Unsupported);+ }++ child_order.push(PhysicalSortExpr {+ expr: transformed.data,+ options: sort_expr.options,+ });+ }++ // Recursively push down to child node+ match child.try_pushdown_sort(&child_order)? {+ SortOrderPushdownResult::Exact { inner } => {+ let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;+ Ok(SortOrderPushdownResult::Exact { inner: new_exec })+ }+ SortOrderPushdownResult::Inexact { inner } => {+ let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;+ Ok(SortOrderPushdownResult::Inexact { inner: new_exec })+ }+ SortOrderPushdownResult::Unsupported => {+ Ok(SortOrderPushdownResult::Unsupported)+ }+ }+ }}impl ProjectionStream {diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex 3d08cdf751..3e403171e0 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -426,329 +426,3 @@ SET datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = true;statement okSET datafusion.optimizer.enable_dynamic_filter_pushdown = true;--# Test 6: Sort Pushdown for ordered Parquet files--# Create a sorted dataset-statement ok-CREATE TABLE sorted_data(id INT, value INT, name VARCHAR) AS VALUES-(1, 100, 'a'),-(2, 200, 'b'),-(3, 300, 'c'),-(4, 400, 'd'),-(5, 500, 'e'),-(6, 600, 'f'),-(7, 700, 'g'),-(8, 800, 'h'),-(9, 900, 'i'),-(10, 1000, 'j');--# Copy to parquet with sorting-query I-COPY (SELECT * FROM sorted_data ORDER BY id ASC)-TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet';------10--statement ok-CREATE EXTERNAL TABLE sorted_parquet(id INT, value INT, name VARCHAR)-STORED AS PARQUET-LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet'-WITH ORDER (id ASC);--# Test 6.1: Sort pushdown with DESC (opposite of ASC)-# Should show reverse_row_groups=true-query TT-EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;------logical_plan-01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3-02)--TableScan: sorted_parquet projection=[id, value, name]-physical_plan-01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true--# Test 6.2: Verify results are correct-query IIT-SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;------10 1000 j-9 900 i-8 800 h--# Test 6.3: Should NOT apply for ASC (same direction)-query TT-EXPLAIN SELECT * FROM sorted_parquet ORDER BY id ASC LIMIT 3;------logical_plan-01)Sort: sorted_parquet.id ASC NULLS LAST, fetch=3-02)--TableScan: sorted_parquet projection=[id, value, name]-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet--# Test 6.4: Disable sort pushdown-statement ok-SET datafusion.optimizer.enable_sort_pushdown = false;--query TT-EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;------logical_plan-01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3-02)--TableScan: sorted_parquet projection=[id, value, name]-physical_plan-01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]--# Re-enable-statement ok-SET datafusion.optimizer.enable_sort_pushdown = true;--# Test 6.5: With OFFSET-query TT-EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;------logical_plan-01)Limit: skip=2, fetch=3-02)--Sort: sorted_parquet.id DESC NULLS FIRST, fetch=5-03)----TableScan: sorted_parquet projection=[id, value, name]-physical_plan-01)GlobalLimitExec: skip=2, fetch=3-02)--SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true--query IIT-SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;------8 800 h-7 700 g-6 600 f--# Test 6.6: Reverse scan with row selection (page index pruning)-# This tests that when reverse_row_groups=true, the RowSelection is also properly reversed--# Create a dataset with multiple row groups and enable page index-statement ok-CREATE TABLE multi_rg_data(id INT, category VARCHAR, value INT) AS VALUES-(1, 'alpha', 10),-(2, 'alpha', 20),-(3, 'beta', 30),-(4, 'beta', 40),-(5, 'gamma', 50),-(6, 'gamma', 60),-(7, 'delta', 70),-(8, 'delta', 80);--# Write with small row groups (2 rows each = 4 row groups)-statement ok-SET datafusion.execution.parquet.max_row_group_size = 2;--query I-COPY (SELECT * FROM multi_rg_data ORDER BY id ASC)-TO 'test_files/scratch/dynamic_filter_pushdown_config/multi_rg_sorted.parquet';------8--# Reset row group size-statement ok-SET datafusion.execution.parquet.max_row_group_size = 1048576;--statement ok-CREATE EXTERNAL TABLE multi_rg_sorted(id INT, category VARCHAR, value INT)-STORED AS PARQUET-LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/multi_rg_sorted.parquet'-WITH ORDER (id ASC);--# Enable page index for better pruning-statement ok-SET datafusion.execution.parquet.enable_page_index = true;--statement ok-SET datafusion.execution.parquet.pushdown_filters = true;--# Test with reverse scan and filter that prunes some row groups-# This will create a RowSelection with partial row group scans-query TT-EXPLAIN SELECT * FROM multi_rg_sorted-WHERE category IN ('alpha', 'gamma')-ORDER BY id DESC LIMIT 5;------logical_plan-01)Sort: multi_rg_sorted.id DESC NULLS FIRST, fetch=5-02)--Filter: multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")-03)----TableScan: multi_rg_sorted projection=[id, category, value], partial_filters=[multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")]-physical_plan-01)SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/multi_rg_sorted.parquet]]}, projection=[id, category, value], file_type=parquet, predicate=(category@1 = alpha OR category@1 = gamma) AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=category_null_count@2 != row_count@3 AND category_min@0 <= alpha AND alpha <= category_max@1 OR category_null_count@2 != row_count@3 AND category_min@0 <= gamma AND gamma <= category_max@1, required_guarantees=[category in (alpha, gamma)]--# Verify the results are correct despite reverse scanning with row selection-# Expected: gamma values (6, 5) then alpha values (2, 1), in DESC order by id-query ITI-SELECT * FROM multi_rg_sorted-WHERE category IN ('alpha', 'gamma')-ORDER BY id DESC LIMIT 5;------6 gamma 60-5 gamma 50-2 alpha 20-1 alpha 10--# Test with more complex selection pattern-query ITI-SELECT * FROM multi_rg_sorted-WHERE category IN ('beta', 'delta')-ORDER BY id DESC;------8 delta 80-7 delta 70-4 beta 40-3 beta 30--# Test forward scan for comparison (should give same logical results in ASC order)-query ITI-SELECT * FROM multi_rg_sorted-WHERE category IN ('alpha', 'gamma')-ORDER BY id ASC;------1 alpha 10-2 alpha 20-5 gamma 50-6 gamma 60--# Disable reverse scan and verify it still works-statement ok-SET datafusion.optimizer.enable_sort_pushdown = false;--query ITI-SELECT * FROM multi_rg_sorted-WHERE category IN ('alpha', 'gamma')-ORDER BY id DESC LIMIT 5;------6 gamma 60-5 gamma 50-2 alpha 20-1 alpha 10--# Re-enable-statement ok-SET datafusion.optimizer.enable_sort_pushdown = true;--# Test 6.7: Sort pushdown with more than one partition-# Create multiple parquet files to trigger it--# Split data into multiple files-statement ok-CREATE TABLE sorted_data_part1(id INT, value INT, name VARCHAR) AS VALUES-(1, 100, 'a'),-(2, 200, 'b'),-(3, 300, 'c');--statement ok-CREATE TABLE sorted_data_part2(id INT, value INT, name VARCHAR) AS VALUES-(4, 400, 'd'),-(5, 500, 'e'),-(6, 600, 'f');--statement ok-CREATE TABLE sorted_data_part3(id INT, value INT, name VARCHAR) AS VALUES-(7, 700, 'g'),-(8, 800, 'h'),-(9, 900, 'i'),-(10, 1000, 'j');--# Create directory for multi-file parquet-query I-COPY (SELECT * FROM sorted_data_part1 ORDER BY id ASC)-TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part1.parquet';------3--query I-COPY (SELECT * FROM sorted_data_part2 ORDER BY id ASC)-TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part2.parquet';------3--query I-COPY (SELECT * FROM sorted_data_part3 ORDER BY id ASC)-TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part3.parquet';------4--# Create external table pointing to directory with multiple files-statement ok-CREATE EXTERNAL TABLE sorted_parquet_multi(id INT, value INT, name VARCHAR)-STORED AS PARQUET-LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/'-WITH ORDER (id ASC);--# Enable multiple partitions-statement ok-SET datafusion.execution.target_partitions = 4;--# Now we should see RepartitionExec because we have 3 input partitions (3 files)-query TT-EXPLAIN SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;------logical_plan-01)Sort: sorted_parquet_multi.id DESC NULLS FIRST, fetch=3-02)--TableScan: sorted_parquet_multi projection=[id, value, name]-physical_plan-01)SortPreservingMergeExec: [id@0 DESC], fetch=3-02)--SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part3.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true--# Verify correctness with repartitioning and multiple files-query IIT-SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;------10 1000 j-9 900 i-8 800 h--# Test ASC order (should not trigger reverse scan)-query IIT-SELECT * FROM sorted_parquet_multi ORDER BY id ASC LIMIT 3;------1 100 a-2 200 b-3 300 c--# Cleanup-statement ok-DROP TABLE sorted_data_part1;--statement ok-DROP TABLE sorted_data_part2;--statement ok-DROP TABLE sorted_data_part3;--statement ok-DROP TABLE sorted_parquet_multi;--# Reset to default-statement ok-SET datafusion.execution.target_partitions = 4;--# Cleanup-statement ok-DROP TABLE multi_rg_data;--statement ok-DROP TABLE multi_rg_sorted;--statement ok-SET datafusion.execution.parquet.enable_page_index = false;--statement ok-SET datafusion.execution.parquet.pushdown_filters = false;--# Cleanup-statement ok-DROP TABLE sorted_data;--statement ok-DROP TABLE sorted_parquet;--statement ok-SET datafusion.optimizer.enable_sort_pushdown = true;diff --git a/datafusion/sqllogictest/test_files/sort_pushdown.slt b/datafusion/sqllogictest/test_files/sort_pushdown.sltnew file mode 100644index 0000000000..58d9915a24--- /dev/null+++ b/datafusion/sqllogictest/test_files/sort_pushdown.slt@@ -0,0 +1,886 @@+#Sort Pushdown for ordered Parquet files+statement ok+SET datafusion.execution.parquet.pushdown_filters = true;++statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;++# Test 1: Sort Pushdown for ordered Parquet files+# Create a sorted dataset+statement ok+CREATE TABLE sorted_data(id INT, value INT, name VARCHAR) AS VALUES+(1, 100, 'a'),+(2, 200, 'b'),+(3, 300, 'c'),+(4, 400, 'd'),+(5, 500, 'e'),+(6, 600, 'f'),+(7, 700, 'g'),+(8, 800, 'h'),+(9, 900, 'i'),+(10, 1000, 'j');++# Copy to parquet with sorting+query I+COPY (SELECT * FROM sorted_data ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/sorted_data.parquet';+----+10++statement ok+CREATE EXTERNAL TABLE sorted_parquet(id INT, value INT, name VARCHAR)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/sorted_data.parquet'+WITH ORDER (id ASC);++# Test 1.1: Sort pushdown with DESC (opposite of ASC)+# Should show reverse_row_groups=true+query TT+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;+----+logical_plan+01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3+02)--TableScan: sorted_parquet projection=[id, value, name]+physical_plan+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++# Test 1.2: Verify results are correct+query IIT+SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;+----+10 1000 j+9 900 i+8 800 h++# Test 1.3: Should NOT apply for ASC (same direction)+query TT+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id ASC LIMIT 3;+----+logical_plan+01)Sort: sorted_parquet.id ASC NULLS LAST, fetch=3+02)--TableScan: sorted_parquet projection=[id, value, name]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++# Test 1.4: Disable sort pushdown+statement ok+SET datafusion.optimizer.enable_sort_pushdown = false;++query TT+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;+----+logical_plan+01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3+02)--TableScan: sorted_parquet projection=[id, value, name]+physical_plan+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]++# Re-enable+statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;++# Test 1.5: With OFFSET+query TT+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;+----+logical_plan+01)Limit: skip=2, fetch=3+02)--Sort: sorted_parquet.id DESC NULLS FIRST, fetch=5+03)----TableScan: sorted_parquet projection=[id, value, name]+physical_plan+01)GlobalLimitExec: skip=2, fetch=3+02)--SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++query IIT+SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;+----+8 800 h+7 700 g+6 600 f++# Test 1.6: Reverse scan with row selection (page index pruning)+# This tests that when reverse_row_groups=true, the RowSelection is also properly reversed++# Create a dataset with multiple row groups and enable page index+statement ok+CREATE TABLE multi_rg_data(id INT, category VARCHAR, value INT) AS VALUES+(1, 'alpha', 10),+(2, 'alpha', 20),+(3, 'beta', 30),+(4, 'beta', 40),+(5, 'gamma', 50),+(6, 'gamma', 60),+(7, 'delta', 70),+(8, 'delta', 80);++# Write with small row groups (2 rows each = 4 row groups)+statement ok+SET datafusion.execution.parquet.max_row_group_size = 2;++query I+COPY (SELECT * FROM multi_rg_data ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/multi_rg_sorted.parquet';+----+8++# Reset row group size+statement ok+SET datafusion.execution.parquet.max_row_group_size = 1048576;++statement ok+CREATE EXTERNAL TABLE multi_rg_sorted(id INT, category VARCHAR, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/multi_rg_sorted.parquet'+WITH ORDER (id ASC);++# Enable page index for better pruning+statement ok+SET datafusion.execution.parquet.enable_page_index = true;++statement ok+SET datafusion.execution.parquet.pushdown_filters = true;++# Test with reverse scan and filter that prunes some row groups+# This will create a RowSelection with partial row group scans+query TT+EXPLAIN SELECT * FROM multi_rg_sorted+WHERE category IN ('alpha', 'gamma')+ORDER BY id DESC LIMIT 5;+----+logical_plan+01)Sort: multi_rg_sorted.id DESC NULLS FIRST, fetch=5+02)--Filter: multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")+03)----TableScan: multi_rg_sorted projection=[id, category, value], partial_filters=[multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")]+physical_plan+01)SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_rg_sorted.parquet]]}, projection=[id, category, value], file_type=parquet, predicate=(category@1 = alpha OR category@1 = gamma) AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=category_null_count@2 != row_count@3 AND category_min@0 <= alpha AND alpha <= category_max@1 OR category_null_count@2 != row_count@3 AND category_min@0 <= gamma AND gamma <= category_max@1, required_guarantees=[category in (alpha, gamma)]++# Verify the results are correct despite reverse scanning with row selection+# Expected: gamma values (6, 5) then alpha values (2, 1), in DESC order by id+query ITI+SELECT * FROM multi_rg_sorted+WHERE category IN ('alpha', 'gamma')+ORDER BY id DESC LIMIT 5;+----+6 gamma 60+5 gamma 50+2 alpha 20+1 alpha 10++# Test with more complex selection pattern+query ITI+SELECT * FROM multi_rg_sorted+WHERE category IN ('beta', 'delta')+ORDER BY id DESC;+----+8 delta 80+7 delta 70+4 beta 40+3 beta 30++# Test forward scan for comparison (should give same logical results in ASC order)+query ITI+SELECT * FROM multi_rg_sorted+WHERE category IN ('alpha', 'gamma')+ORDER BY id ASC;+----+1 alpha 10+2 alpha 20+5 gamma 50+6 gamma 60++# Disable reverse scan and verify it still works+statement ok+SET datafusion.optimizer.enable_sort_pushdown = false;++query ITI+SELECT * FROM multi_rg_sorted+WHERE category IN ('alpha', 'gamma')+ORDER BY id DESC LIMIT 5;+----+6 gamma 60+5 gamma 50+2 alpha 20+1 alpha 10++# Re-enable+statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;++# Test 1.7: Sort pushdown with more than one partition+# Create multiple parquet files to trigger it++# Split data into multiple files+statement ok+CREATE TABLE sorted_data_part1(id INT, value INT, name VARCHAR) AS VALUES+(1, 100, 'a'),+(2, 200, 'b'),+(3, 300, 'c');++statement ok+CREATE TABLE sorted_data_part2(id INT, value INT, name VARCHAR) AS VALUES+(4, 400, 'd'),+(5, 500, 'e'),+(6, 600, 'f');++statement ok+CREATE TABLE sorted_data_part3(id INT, value INT, name VARCHAR) AS VALUES+(7, 700, 'g'),+(8, 800, 'h'),+(9, 900, 'i'),+(10, 1000, 'j');++# Create directory for multi-file parquet+query I+COPY (SELECT * FROM sorted_data_part1 ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/sorted_multi/part1.parquet';+----+3++query I+COPY (SELECT * FROM sorted_data_part2 ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/sorted_multi/part2.parquet';+----+3++query I+COPY (SELECT * FROM sorted_data_part3 ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/sorted_multi/part3.parquet';+----+4++# Create external table pointing to directory with multiple files+statement ok+CREATE EXTERNAL TABLE sorted_parquet_multi(id INT, value INT, name VARCHAR)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/sorted_multi/'+WITH ORDER (id ASC);++# Enable multiple partitions+statement ok+SET datafusion.execution.target_partitions = 4;++# Now we should see RepartitionExec because we have 3 input partitions (3 files)+query TT+EXPLAIN SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;+----+logical_plan+01)Sort: sorted_parquet_multi.id DESC NULLS FIRST, fetch=3+02)--TableScan: sorted_parquet_multi projection=[id, value, name]+physical_plan+01)SortPreservingMergeExec: [id@0 DESC], fetch=3+02)--SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[true]+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part3.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++# Verify correctness with repartitioning and multiple files+query IIT+SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;+----+10 1000 j+9 900 i+8 800 h++# Test ASC order (should not trigger reverse scan)+query IIT+SELECT * FROM sorted_parquet_multi ORDER BY id ASC LIMIT 3;+----+1 100 a+2 200 b+3 300 c++# Cleanup+statement ok+DROP TABLE sorted_data_part1;++statement ok+DROP TABLE sorted_data_part2;++statement ok+DROP TABLE sorted_data_part3;++statement ok+DROP TABLE sorted_parquet_multi;++# Reset to default+statement ok+SET datafusion.execution.target_partitions = 4;++# Cleanup+statement ok+DROP TABLE multi_rg_data;++statement ok+DROP TABLE multi_rg_sorted;++statement ok+SET datafusion.execution.parquet.enable_page_index = false;++statement ok+SET datafusion.execution.parquet.pushdown_filters = true;++# Cleanup+statement ok+DROP TABLE sorted_data;++statement ok+DROP TABLE sorted_parquet;++statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;+++# Test 2: Sort pushdown with constant column filtering+# This tests the case where a leading sort column becomes constant through WHERE filtering++# Create a multi-column sorted dataset (like time-series data)+statement ok+CREATE TABLE timeseries_data(timeframe VARCHAR, period_end INT, value DOUBLE) AS VALUES+('daily', 1, 100.0),+('daily', 2, 150.0),+('daily', 3, 200.0),+('weekly', 1, 500.0),+('weekly', 2, 600.0),+('weekly', 3, 700.0),+('monthly', 1, 1000.0),+('monthly', 2, 1100.0),+('monthly', 3, 1200.0),+('quarterly', 1, 5000.0),+('quarterly', 2, 5500.0),+('quarterly', 3, 6000.0);++# Copy to parquet with multi-column sorting (timeframe ASC, period_end ASC)+query I+COPY (SELECT * FROM timeseries_data ORDER BY timeframe ASC, period_end ASC)+TO 'test_files/scratch/sort_pushdown/timeseries_sorted.parquet';+----+12++statement ok+CREATE EXTERNAL TABLE timeseries_parquet(timeframe VARCHAR, period_end INT, value DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/timeseries_sorted.parquet'+WITH ORDER (timeframe ASC, period_end ASC);++# Test 2.1: Query with constant prefix filter and DESC on remaining column+# WHERE timeframe='quarterly' makes the first sort column constant+# ORDER BY period_end DESC should trigger reverse scan because:+# File ordering: [timeframe ASC, period_end ASC]+# After filtering timeframe='quarterly': effectively [period_end ASC]+# Request: [period_end DESC] -> exact reverse!+query TT+EXPLAIN SELECT * FROM timeseries_parquet+WHERE timeframe = 'quarterly'+ORDER BY period_end DESC+LIMIT 2;+----+logical_plan+01)Sort: timeseries_parquet.period_end DESC NULLS FIRST, fetch=2+02)--Filter: timeseries_parquet.timeframe = Utf8View("quarterly")+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("quarterly")]+physical_plan+01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = quarterly AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]++# Test 2.2: Verify the results are correct+query TIR+SELECT * FROM timeseries_parquet+WHERE timeframe = 'quarterly'+ORDER BY period_end DESC+LIMIT 2;+----+quarterly 3 6000+quarterly 2 5500++# Test 2.3: Same filter but ASC order (should not trigger reverse scan, ordering already satisfied)+query TT+EXPLAIN SELECT * FROM timeseries_parquet+WHERE timeframe = 'quarterly'+ORDER BY period_end ASC+LIMIT 2;+----+logical_plan+01)Sort: timeseries_parquet.period_end ASC NULLS LAST, fetch=2+02)--Filter: timeseries_parquet.timeframe = Utf8View("quarterly")+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("quarterly")]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], limit=2, output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=timeframe@0 = quarterly, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]++# Test 2.4: Verify ASC results+query TIR+SELECT * FROM timeseries_parquet+WHERE timeframe = 'quarterly'+ORDER BY period_end ASC+LIMIT 2;+----+quarterly 1 5000+quarterly 2 5500++# Test 2.5: Test with different constant value+query TIR+SELECT * FROM timeseries_parquet+WHERE timeframe = 'weekly'+ORDER BY period_end DESC;+----+weekly 3 700+weekly 2 600+weekly 1 500++# Test 2.6: Test without constant filter (no reverse scan because need both columns)+# Request: [timeframe ASC, period_end DESC]+# File has: [timeframe ASC, period_end ASC]+# These are NOT reverse of each other - only second column is reversed+query TT+EXPLAIN SELECT * FROM timeseries_parquet+ORDER BY timeframe ASC, period_end DESC+LIMIT 3;+----+logical_plan+01)Sort: timeseries_parquet.timeframe ASC NULLS LAST, timeseries_parquet.period_end DESC NULLS FIRST, fetch=3+02)--TableScan: timeseries_parquet projection=[timeframe, period_end, value]+physical_plan+01)SortExec: TopK(fetch=3), expr=[timeframe@0 ASC NULLS LAST, period_end@1 DESC], preserve_partitioning=[false], sort_prefix=[timeframe@0 ASC NULLS LAST]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]++# Test 2.7: Disable sort pushdown and verify filter still works+statement ok+SET datafusion.optimizer.enable_sort_pushdown = false;++query TT+EXPLAIN SELECT * FROM timeseries_parquet+WHERE timeframe = 'quarterly'+ORDER BY period_end DESC+LIMIT 2;+----+logical_plan+01)Sort: timeseries_parquet.period_end DESC NULLS FIRST, fetch=2+02)--Filter: timeseries_parquet.timeframe = Utf8View("quarterly")+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("quarterly")]+physical_plan+01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=timeframe@0 = quarterly AND DynamicFilter [ empty ], pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]++# Results should still be correct+query TIR+SELECT * FROM timeseries_parquet+WHERE timeframe = 'quarterly'+ORDER BY period_end DESC+LIMIT 2;+----+quarterly 3 6000+quarterly 2 5500++# Re-enable+statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;++# Test 2.8: Test with IN clause (multiple constant values)+# Note: IN clause with multiple values means timeframe is NOT constant+# (could be 'daily' or 'weekly'), so the first sort column cannot be eliminated.+# Without a constant first column, we cannot reverse scan based on just period_end DESC.+# The physical plan should NOT show reverse_row_groups=true+query TT+EXPLAIN SELECT * FROM timeseries_parquet+WHERE timeframe IN ('daily', 'weekly')+ORDER BY period_end DESC+LIMIT 3;+----+logical_plan+01)Sort: timeseries_parquet.period_end DESC NULLS FIRST, fetch=3+02)--Filter: timeseries_parquet.timeframe = Utf8View("daily") OR timeseries_parquet.timeframe = Utf8View("weekly")+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("daily") OR timeseries_parquet.timeframe = Utf8View("weekly")]+physical_plan+01)SortExec: TopK(fetch=3), expr=[period_end@1 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=(timeframe@0 = daily OR timeframe@0 = weekly) AND DynamicFilter [ empty ], pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= daily AND daily <= timeframe_max@1 OR timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= weekly AND weekly <= timeframe_max@1, required_guarantees=[timeframe in (daily, weekly)]++# Test 2.9: Complex case - literal constant in sort expression itself+# The literal 'constant' is ignored in sort analysis+# After stripping: ORDER BY period_end DESC+# With WHERE timeframe='monthly' making first column constant+# File: [period_end ASC] (after constant column removal)+# Request: [period_end DESC] -> exact reverse, triggers reverse scan+query TT+EXPLAIN SELECT * FROM timeseries_parquet+WHERE timeframe = 'monthly'+ORDER BY 'constant', period_end DESC+LIMIT 2;+----+logical_plan+01)Sort: Utf8("constant") ASC NULLS LAST, timeseries_parquet.period_end DESC NULLS FIRST, fetch=2+02)--Filter: timeseries_parquet.timeframe = Utf8View("monthly")+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("monthly")]+physical_plan+01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = monthly AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= monthly AND monthly <= timeframe_max@1, required_guarantees=[timeframe in (monthly)]++# Verify results+query TIR+SELECT * FROM timeseries_parquet+WHERE timeframe = 'monthly'+ORDER BY period_end DESC+LIMIT 2;+----+monthly 3 1200+monthly 2 1100++# Test 2.10: Filter on non-leading sort column+# File order: [timeframe ASC, period_end ASC]+# Filter: period_end = 2 (makes second column constant)+# Request: [timeframe DESC]+# After constant column removal: File has [timeframe ASC], Request wants [timeframe DESC]+# This is exact reverse -> triggers reverse scan+query TT+EXPLAIN SELECT * FROM timeseries_parquet+WHERE period_end = 2+ORDER BY timeframe DESC;+----+logical_plan+01)Sort: timeseries_parquet.timeframe DESC NULLS FIRST+02)--Filter: timeseries_parquet.period_end = Int32(2)+03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.period_end = Int32(2)]+physical_plan+01)SortExec: expr=[timeframe@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=period_end@1 = 2, reverse_row_groups=true, pruning_predicate=period_end_null_count@2 != row_count@3 AND period_end_min@0 <= 2 AND 2 <= period_end_max@1, required_guarantees=[period_end in (2)]++# Cleanup+statement ok+DROP TABLE timeseries_data;++statement ok+DROP TABLE timeseries_parquet;++# Reset to default+statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;+++# Test 3: Sort pushdown with monotonic functions+# This tests that reverse scan works when sort expressions involve monotonic functions++# Create test data with timestamp column+statement ok+CREATE TABLE timestamp_data(id INT, ts TIMESTAMP, volume BIGINT, price DOUBLE) AS VALUES+(1, TIMESTAMP '2024-01-15 10:00:00', 1000, 100.0),+(2, TIMESTAMP '2024-01-20 11:00:00', 1500, 105.0),+(3, TIMESTAMP '2024-01-25 12:00:00', 2000, 110.0),+(4, TIMESTAMP '2024-02-05 09:00:00', 1200, 108.0),+(5, TIMESTAMP '2024-02-15 14:00:00', 1800, 112.0),+(6, TIMESTAMP '2024-02-25 15:00:00', 2200, 115.0),+(7, TIMESTAMP '2024-03-10 09:00:00', 1300, 113.0),+(8, TIMESTAMP '2024-03-18 14:00:00', 1900, 118.0),+(9, TIMESTAMP '2024-03-28 15:00:00', 2300, 120.0);++# Copy to parquet with sorting by timestamp ASC+query I+COPY (SELECT * FROM timestamp_data ORDER BY ts ASC)+TO 'test_files/scratch/sort_pushdown/timestamp_sorted.parquet';+----+9++# Test 3.1: Simple monotonic function - date_trunc+# Create external table with file ordering that conceptually includes date_trunc+# File is actually sorted by [ts ASC], but conceptually [date_trunc('month', ts) ASC, ts ASC]+statement ok+CREATE EXTERNAL TABLE timestamp_parquet(id INT, ts TIMESTAMP, volume BIGINT, price DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/timestamp_sorted.parquet'+WITH ORDER (ts ASC);++# Query with ORDER BY ts DESC+# File ordering: [ts ASC]+# Request: [ts DESC]+# This should trigger reverse_row_groups=true+query TT+EXPLAIN SELECT * FROM timestamp_parquet+ORDER BY ts DESC+LIMIT 3;+----+logical_plan+01)Sort: timestamp_parquet.ts DESC NULLS FIRST, fetch=3+02)--TableScan: timestamp_parquet projection=[id, ts, volume, price]+physical_plan+01)SortExec: TopK(fetch=3), expr=[ts@1 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++# Verify results+query IPIR+SELECT * FROM timestamp_parquet+ORDER BY ts DESC+LIMIT 3;+----+9 2024-03-28T15:00:00 2300 120+8 2024-03-18T14:00:00 1900 118+7 2024-03-10T09:00:00 1300 113++# Test 3.2: Monotonic function in ORDER BY - date_trunc DESC+# File ordering: [ts ASC]+# Request: [date_trunc('day', ts) DESC]+# Since date_trunc is monotonic with ts, reversed file ordering [ts DESC] satisfies [date_trunc DESC]+query TT+EXPLAIN SELECT * FROM timestamp_parquet+ORDER BY date_trunc('day', ts) DESC+LIMIT 3;+----+logical_plan+01)Sort: date_trunc(Utf8("day"), timestamp_parquet.ts) DESC NULLS FIRST, fetch=3+02)--TableScan: timestamp_parquet projection=[id, ts, volume, price]+physical_plan+01)SortExec: TopK(fetch=3), expr=[date_trunc(day, ts@1) DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++# Verify results (descending day)+query IPIR+SELECT * FROM timestamp_parquet+ORDER BY date_trunc('day', ts) DESC+LIMIT 3;+----+9 2024-03-28T15:00:00 2300 120+8 2024-03-18T14:00:00 1900 118+7 2024-03-10T09:00:00 1300 113++# Test 3.3: Multi-column scenario with explicit monotonic function in file ordering+# Create a table where we explicitly declare the ordering includes the monotonic function+# This simulates files that are partitioned/sorted by [date_trunc('month', ts) ASC, ts ASC]++# Create a new parquet file sorted by [ts ASC] (which implies date_trunc ordering)+statement ok+CREATE TABLE multi_month_data(id INT, ts TIMESTAMP, volume BIGINT, price DOUBLE) AS VALUES+-- January 2024+(1, TIMESTAMP '2024-01-05 09:30:00', 1000, 100.0),+(2, TIMESTAMP '2024-01-15 14:30:00', 1500, 105.0),+(3, TIMESTAMP '2024-01-25 15:59:00', 2000, 110.0),+-- February 2024+(4, TIMESTAMP '2024-02-03 09:30:00', 1200, 108.0),+(5, TIMESTAMP '2024-02-14 12:00:00', 1800, 112.0),+(6, TIMESTAMP '2024-02-28 15:59:00', 2200, 115.0),+-- March 2024+(7, TIMESTAMP '2024-03-01 09:30:00', 1300, 113.0),+(8, TIMESTAMP '2024-03-15 14:00:00', 1900, 118.0),+(9, TIMESTAMP '2024-03-29 15:59:00', 2300, 120.0);++query I+COPY (SELECT * FROM multi_month_data ORDER BY ts ASC)+TO 'test_files/scratch/sort_pushdown/multi_month_sorted.parquet';+----+9++# Declare the file has ordering [ts ASC]+# Conceptually this means [date_trunc('month', ts) ASC, ts ASC] due to monotonicity+statement ok+CREATE EXTERNAL TABLE multi_month_parquet(id INT, ts TIMESTAMP, volume BIGINT, price DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/multi_month_sorted.parquet'+WITH ORDER (ts ASC);++# Test 3.3a: Request ORDER BY ts DESC (opposite direction)+# File: [ts ASC]+# Request: [ts DESC]+# Should trigger reverse_row_groups=true+query TT+EXPLAIN SELECT * FROM multi_month_parquet+ORDER BY ts DESC+LIMIT 2;+----+logical_plan+01)Sort: multi_month_parquet.ts DESC NULLS FIRST, fetch=2+02)--TableScan: multi_month_parquet projection=[id, ts, volume, price]+physical_plan+01)SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++query IPIR+SELECT * FROM multi_month_parquet+ORDER BY ts DESC+LIMIT 2;+----+9 2024-03-29T15:59:00 2300 120+8 2024-03-15T14:00:00 1900 118++# Test 3.3b: Request ORDER BY date_trunc('month', ts) DESC, ts DESC+# File: [ts ASC] (which implies [date_trunc('month', ts) ASC, ts ASC])+# Request: [date_trunc('month', ts) DESC, ts DESC]+# The reversed file ordering [ts DESC] satisfies this because:+# - date_trunc is monotonic with ts+# - So [ts DESC] implies [date_trunc('month', ts) DESC, ts DESC]+query TT+EXPLAIN SELECT * FROM multi_month_parquet+ORDER BY date_trunc('month', ts) DESC, ts DESC+LIMIT 2;+----+logical_plan+01)Sort: date_trunc(Utf8("month"), multi_month_parquet.ts) DESC NULLS FIRST, multi_month_parquet.ts DESC NULLS FIRST, fetch=2+02)--TableScan: multi_month_parquet projection=[id, ts, volume, price]+physical_plan+01)SortExec: TopK(fetch=2), expr=[date_trunc(month, ts@1) DESC, ts@1 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++query IPIR+SELECT * FROM multi_month_parquet+ORDER BY date_trunc('month', ts) DESC, ts DESC+LIMIT 2;+----+9 2024-03-29T15:59:00 2300 120+8 2024-03-15T14:00:00 1900 118++# Test 3.4: CAST as a monotonic function+statement ok+CREATE TABLE int_data(id INT, small_val SMALLINT, big_val BIGINT) AS VALUES+(1, 10, 100),+(2, 20, 200),+(3, 30, 300),+(4, 40, 400),+(5, 50, 500);++query I+COPY (SELECT * FROM int_data ORDER BY small_val ASC)+TO 'test_files/scratch/sort_pushdown/int_sorted.parquet';+----+5++statement ok+CREATE EXTERNAL TABLE int_parquet(id INT, small_val SMALLINT, big_val BIGINT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/int_sorted.parquet'+WITH ORDER (small_val ASC);++# CAST preserves ordering: CAST(small_val AS BIGINT) is monotonic with small_val+query TT+EXPLAIN SELECT * FROM int_parquet+ORDER BY CAST(small_val AS BIGINT) DESC+LIMIT 2;+----+logical_plan+01)Sort: CAST(int_parquet.small_val AS Int64) DESC NULLS FIRST, fetch=2+02)--TableScan: int_parquet projection=[id, small_val, big_val]+physical_plan+01)SortExec: TopK(fetch=2), expr=[CAST(small_val@1 AS Int64) DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/int_sorted.parquet]]}, projection=[id, small_val, big_val], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++query III+SELECT * FROM int_parquet+ORDER BY CAST(small_val AS BIGINT) DESC+LIMIT 2;+----+5 50 500+4 40 400++# Test 3.5: CEIL as a monotonic function+statement ok+CREATE TABLE float_data(id INT, value DOUBLE) AS VALUES+(1, 1.1),+(2, 2.3),+(3, 3.5),+(4, 4.7),+(5, 5.9);++query I+COPY (SELECT * FROM float_data ORDER BY value ASC)+TO 'test_files/scratch/sort_pushdown/float_sorted.parquet';+----+5++statement ok+CREATE EXTERNAL TABLE float_parquet(id INT, value DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/float_sorted.parquet'+WITH ORDER (value ASC);++# CEIL is monotonic increasing+query TT+EXPLAIN SELECT * FROM float_parquet+ORDER BY CEIL(value) DESC+LIMIT 3;+----+logical_plan+01)Sort: ceil(float_parquet.value) DESC NULLS FIRST, fetch=3+02)--TableScan: float_parquet projection=[id, value]+physical_plan+01)SortExec: TopK(fetch=3), expr=[ceil(value@1) DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/float_sorted.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++query IR+SELECT * FROM float_parquet+ORDER BY CEIL(value) DESC+LIMIT 3;+----+5 5.9+4 4.7+3 3.5++# Test 3.6: Negative case - ABS is NOT monotonic over mixed positive/negative range+statement ok+CREATE TABLE signed_data(id INT, value DOUBLE) AS VALUES+(1, -5.0),+(2, -3.0),+(3, -1.0),+(4, 2.0),+(5, 4.0);++query I+COPY (SELECT * FROM signed_data ORDER BY value ASC)+TO 'test_files/scratch/sort_pushdown/signed_sorted.parquet';+----+5++statement ok+CREATE EXTERNAL TABLE signed_parquet(id INT, value DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/signed_sorted.parquet'+WITH ORDER (value ASC);++# ABS is NOT monotonic over the full range [-5, 4], so should NOT trigger reverse scan+query TT+EXPLAIN SELECT * FROM signed_parquet+ORDER BY ABS(value) DESC+LIMIT 3;+----+logical_plan+01)Sort: abs(signed_parquet.value) DESC NULLS FIRST, fetch=3+02)--TableScan: signed_parquet projection=[id, value]+physical_plan+01)SortExec: TopK(fetch=3), expr=[abs(value@1) DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/signed_sorted.parquet]]}, projection=[id, value], output_ordering=[value@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]++# Results should still be correct (no optimization applied)+query IR+SELECT * FROM signed_parquet+ORDER BY ABS(value) DESC+LIMIT 3;+----+1 -5+5 4+2 -3++# Cleanup+statement ok+DROP TABLE timestamp_data;++statement ok+DROP TABLE timestamp_parquet;++statement ok+DROP TABLE multi_month_data;++statement ok+DROP TABLE multi_month_parquet;++statement ok+DROP TABLE int_data;++statement ok+DROP TABLE int_parquet;++statement ok+DROP TABLE float_data;++statement ok+DROP TABLE float_parquet;++statement ok+DROP TABLE signed_data;++statement ok+DROP TABLE signed_parquet;++statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;