Apache DataFusion SQL Query Engine
refactor: Rename `FileSource::try_reverse_output` to `FileSource::try_pushdown_sort` (#20043)
Kumar Ujjawal030621c6b6b6
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 6f92d567c8..f87a30265a 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -581,7 +581,7 @@ impl FileOpener for ParquetOpener {// ----------------------------------------------------------// Step: potentially reverse the access plan for performance.- // See `ParquetSource::try_reverse_output` for the rationale.+ // See `ParquetSource::try_pushdown_sort` for the rationale.// ----------------------------------------------------------if reverse_row_groups {prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 07f58db185..75d87a4cd1 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -757,7 +757,7 @@ impl FileSource for ParquetSource {/// # Returns/// - `Inexact`: Created an optimized source (e.g., reversed scan) that approximates the order/// - `Unsupported`: Cannot optimize for this ordering- fn try_reverse_output(+ fn try_pushdown_sort(&self,order: &[PhysicalSortExpr],eq_properties: &EquivalenceProperties,diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex f5380c27ec..c6282c3c7c 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -189,7 +189,29 @@ pub trait FileSource: Send + Sync {/// * `Inexact` - Created a source optimized for ordering (e.g., reversed row groups) but not perfectly sorted/// * `Unsupported` - Cannot optimize for this ordering///- /// Default implementation returns `Unsupported`.+ /// # Deprecation / migration notes+ /// - [`Self::try_reverse_output`] was renamed to this method and deprecated since `53.0.0`.+ /// Per DataFusion's deprecation guidelines, it will be removed in `59.0.0` or later+ /// (6 major versions or 6 months, whichever is longer).+ /// - New implementations should override [`Self::try_pushdown_sort`] directly.+ /// - For backwards compatibility, the default implementation of+ /// [`Self::try_pushdown_sort`] delegates to the deprecated+ /// [`Self::try_reverse_output`] until it is removed. After that point, the+ /// default implementation will return [`SortOrderPushdownResult::Unsupported`].+ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ eq_properties: &EquivalenceProperties,+ ) -> Result<SortOrderPushdownResult<Arc<dyn FileSource>>> {+ #[expect(deprecated)]+ self.try_reverse_output(order, eq_properties)+ }++ /// Deprecated: Renamed to [`Self::try_pushdown_sort`].+ #[deprecated(+ since = "53.0.0",+ note = "Renamed to try_pushdown_sort. This method was never limited to reversing output. It will be removed in 59.0.0 or later."+ )]fn try_reverse_output(&self,_order: &[PhysicalSortExpr],@@ -232,7 +254,7 @@ pub trait FileSource: Send + Sync {/// `SchemaAdapterFactory` has been removed. Use `PhysicalExprAdapterFactory` instead./// See `upgrading.md` for more details.#[deprecated(- since = "52.0.0",+ since = "53.0.0",note = "SchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details.")]#[expect(deprecated)]@@ -250,7 +272,7 @@ pub trait FileSource: Send + Sync {/// `SchemaAdapterFactory` has been removed. Use `PhysicalExprAdapterFactory` instead./// See `upgrading.md` for more details.#[deprecated(- since = "52.0.0",+ since = "53.0.0",note = "SchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details.")]#[expect(deprecated)]diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex 51b9ba9e06..fe78c0e526 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -851,20 +851,20 @@ impl DataSource for FileScanConfig {&self,order: &[PhysicalSortExpr],) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {- // Delegate to FileSource to check if reverse scanning can satisfy the request.+ // Delegate to FileSource to see if it can optimize for the requested ordering.let pushdown_result = self.file_source- .try_reverse_output(order, &self.eq_properties())?;+ .try_pushdown_sort(order, &self.eq_properties())?;match pushdown_result {SortOrderPushdownResult::Exact { inner } => {Ok(SortOrderPushdownResult::Exact {- inner: self.rebuild_with_source(inner, true)?,+ inner: self.rebuild_with_source(inner, true, order)?,})}SortOrderPushdownResult::Inexact { inner } => {Ok(SortOrderPushdownResult::Inexact {- inner: self.rebuild_with_source(inner, false)?,+ inner: self.rebuild_with_source(inner, false, order)?,})}SortOrderPushdownResult::Unsupported => {@@ -1157,19 +1157,44 @@ impl FileScanConfig {&self,new_file_source: Arc<dyn FileSource>,is_exact: bool,+ order: &[PhysicalSortExpr],) -> Result<Arc<dyn DataSource>> {let mut new_config = self.clone();- // Reverse file groups (FileScanConfig's responsibility)- new_config.file_groups = new_config- .file_groups- .into_iter()- .map(|group| {- let mut files = group.into_inner();- files.reverse();- files.into()- })- .collect();+ // Reverse file order (within each group) if the caller is requesting a reversal of this+ // scan's declared output ordering.+ //+ // Historically this function always reversed `file_groups` because it was only reached+ // via `FileSource::try_reverse_output` (where a reversal was the only supported+ // optimization).+ //+ // Now that `FileSource::try_pushdown_sort` is generic, we must not assume reversal: other+ // optimizations may become possible (e.g. already-sorted data, statistics-based file+ // reordering). Therefore we only reverse files when it is known to help satisfy the+ // requested ordering.+ let reverse_file_groups = if self.output_ordering.is_empty() {+ false+ } else if let Some(requested) = LexOrdering::new(order.iter().cloned()) {+ let projected_schema = self.projected_schema()?;+ let orderings = project_orderings(&self.output_ordering, &projected_schema);+ orderings+ .iter()+ .any(|ordering| ordering.is_reverse(&requested))+ } else {+ false+ };++ if reverse_file_groups {+ new_config.file_groups = new_config+ .file_groups+ .into_iter()+ .map(|group| {+ let mut files = group.into_inner();+ files.reverse();+ files.into()+ })+ .collect();+ }new_config.file_source = new_file_source;@@ -1392,6 +1417,62 @@ mod tests {use datafusion_physical_expr::projection::ProjectionExpr;use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+ #[derive(Clone)]+ struct InexactSortPushdownSource {+ metrics: ExecutionPlanMetricsSet,+ table_schema: TableSchema,+ }++ impl InexactSortPushdownSource {+ fn new(table_schema: TableSchema) -> Self {+ Self {+ metrics: ExecutionPlanMetricsSet::new(),+ table_schema,+ }+ }+ }++ impl FileSource for InexactSortPushdownSource {+ fn create_file_opener(+ &self,+ _object_store: Arc<dyn object_store::ObjectStore>,+ _base_config: &FileScanConfig,+ _partition: usize,+ ) -> Result<Arc<dyn crate::file_stream::FileOpener>> {+ unimplemented!()+ }++ fn as_any(&self) -> &dyn Any {+ self+ }++ fn table_schema(&self) -> &TableSchema {+ &self.table_schema+ }++ fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {+ Arc::new(self.clone())+ }++ fn metrics(&self) -> &ExecutionPlanMetricsSet {+ &self.metrics+ }++ fn file_type(&self) -> &str {+ "mock"+ }++ fn try_pushdown_sort(+ &self,+ _order: &[PhysicalSortExpr],+ _eq_properties: &EquivalenceProperties,+ ) -> Result<SortOrderPushdownResult<Arc<dyn FileSource>>> {+ Ok(SortOrderPushdownResult::Inexact {+ inner: Arc::new(self.clone()) as Arc<dyn FileSource>,+ })+ }+ }+#[test]fn physical_plan_config_no_projection_tab_cols_as_field() {let file_schema = aggr_test_schema();@@ -2337,4 +2418,56 @@ mod tests {_ => panic!("Expected Hash partitioning"),}}++ #[test]+ fn try_pushdown_sort_reverses_file_groups_only_when_requested_is_reverse()+ -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));++ let table_schema = TableSchema::new(Arc::clone(&file_schema), vec![]);+ let file_source = Arc::new(InexactSortPushdownSource::new(table_schema));++ let file_groups = vec![FileGroup::new(vec![+ PartitionedFile::new("file1", 1),+ PartitionedFile::new("file2", 1),+ ])];++ let sort_expr_asc = PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)));+ let config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(file_groups)+ .with_output_ordering(vec![+ LexOrdering::new(vec![sort_expr_asc.clone()]).unwrap(),+ ])+ .build();++ let requested_asc = vec![sort_expr_asc.clone()];+ let result = config.try_pushdown_sort(&requested_asc)?;+ let SortOrderPushdownResult::Inexact { inner } = result else {+ panic!("Expected Inexact result");+ };+ let pushed_config = inner+ .as_any()+ .downcast_ref::<FileScanConfig>()+ .expect("Expected FileScanConfig");+ let pushed_files = pushed_config.file_groups[0].files();+ assert_eq!(pushed_files[0].object_meta.location.as_ref(), "file1");+ assert_eq!(pushed_files[1].object_meta.location.as_ref(), "file2");++ let requested_desc = vec![sort_expr_asc.reverse()];+ let result = config.try_pushdown_sort(&requested_desc)?;+ let SortOrderPushdownResult::Inexact { inner } = result else {+ panic!("Expected Inexact result");+ };+ let pushed_config = inner+ .as_any()+ .downcast_ref::<FileScanConfig>()+ .expect("Expected FileScanConfig");+ let pushed_files = pushed_config.file_groups[0].files();+ assert_eq!(pushed_files[0].object_meta.location.as_ref(), "file2");+ assert_eq!(pushed_files[1].object_meta.location.as_ref(), "file1");++ Ok(())+ }}