Apache DataFusion SQL Query Engine
perf: embed projections in ASOF joins (#24873)
Xuanwo06e9fae2ef37
diff --git a/datafusion/core/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rsindex dd32830dd5..c013b70c04 100644--- a/datafusion/core/tests/sql/joins.rs+++ b/datafusion/core/tests/sql/joins.rs@@ -490,7 +490,7 @@ async fn asof_join_broadcasts_multi_partition_right_input() -> Result<()> {register_asof_test_tables(&ctx)?;let df = ctx.sql(- "SELECT t.trade_id, p.price FROM trades t ASOF JOIN \+ "SELECT t.trade_id, t.ts, p.price FROM trades t ASOF JOIN \(SELECT ts, price FROM prices WHERE symbol = 'A') p \MATCH_CONDITION (t.ts >= p.ts)",)@@ -519,6 +519,7 @@ async fn asof_join_broadcasts_multi_partition_right_input() -> Result<()> {.to_string();assert_contains!(right_plan.as_str(), "SortPreservingMergeExec");assert_contains!(right_plan.as_str(), "DataSourceExec: partitions=2");+ // Keep the left sort key in the embedded projection so the join can expose it.assert!(asof.output_ordering().is_some());assert!(matches!(&asof.input_distribution_requirements().into_per_child()[..],@@ -530,22 +531,49 @@ async fn asof_join_broadcasts_multi_partition_right_input() -> Result<()> {let batches = collect(plan, ctx.task_ctx()).await?;assert_batches_sorted_eq!([- "+----------+-------+",- "| trade_id | price |",- "+----------+-------+",- "| 1 | |",- "| 2 | 40 |",- "| 3 | 60 |",- "| 4 | 20 |",- "| 5 | 60 |",- "| 6 | 20 |",- "+----------+-------+",+ "+----------+----+-------+",+ "| trade_id | ts | price |",+ "+----------+----+-------+",+ "| 1 | 1 | |",+ "| 2 | 4 | 40 |",+ "| 3 | 7 | 60 |",+ "| 4 | 2 | 20 |",+ "| 5 | 8 | 60 |",+ "| 6 | 3 | 20 |",+ "+----------+----+-------+",],&batches);Ok(())}+#[tokio::test]+async fn asof_join_projection_drops_output_ordering() -> Result<()> {+ let ctx = SessionContext::new();+ register_asof_test_tables(&ctx)?;+ let plan = ctx+ .sql(+ "SELECT t.trade_id, p.price FROM trades t ASOF JOIN \+ (SELECT ts, price FROM prices WHERE symbol = 'A') p \+ MATCH_CONDITION (t.ts >= p.ts)",+ )+ .await?+ .create_physical_plan()+ .await?;+ let asof = find_asof_exec(&plan).expect("physical ASOF join must be present");++ assert_eq!(+ asof.schema()+ .fields()+ .iter()+ .map(|field| field.name().as_str())+ .collect::<Vec<_>>(),+ vec!["trade_id", "price"]+ );+ assert!(asof.output_ordering().is_none());+ Ok(())+}+#[tokio::test]async fn asof_join_explain_names_equality_and_match_conditions() -> Result<()> {let ctx = SessionContext::new();diff --git a/datafusion/physical-plan/src/joins/asof_join.rs b/datafusion/physical-plan/src/joins/asof_join.rsindex abf1b37f1f..cb6e71eea7 100644--- a/datafusion/physical-plan/src/joins/asof_join.rs+++ b/datafusion/physical-plan/src/joins/asof_join.rs@@ -110,6 +110,7 @@ use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricsSet,RecordOutput, Time,};+use crate::projection::{EmbeddedProjection, ProjectionExec, try_embed_projection};use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::RecordBatchStreamAdapter;use crate::{@@ -250,6 +251,18 @@ impl AsOfJoinExec {})}+ /// Returns this join emitting only the columns in `projection`, in that order.+ /// The indices address the join's own schema, before any projection.+ pub fn with_projection(&self, projection: Option<Vec<usize>>) -> Result<Self> {+ Self::try_new(+ Arc::clone(&self.left),+ Arc::clone(&self.right),+ self.on.clone(),+ self.match_condition.clone(),+ projection,+ )+ }+fn compute_properties(left: &Arc<dyn ExecutionPlan>,join_schema: &SchemaRef,@@ -293,6 +306,12 @@ impl AsOfJoinExec {}}+impl EmbeddedProjection for AsOfJoinExec {+ fn with_projection(&self, projection: Option<Vec<usize>>) -> Result<Self> {+ self.with_projection(projection)+ }+}+impl DisplayAs for AsOfJoinExec {fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter<'_>) -> std::fmt::Result {let on = self@@ -386,6 +405,16 @@ impl ExecutionPlan for AsOfJoinExec {vec![&self.left, &self.right]}+ fn try_swapping_with_projection(+ &self,+ projection: &ProjectionExec,+ ) -> Result<Option<Arc<dyn ExecutionPlan>>> {+ if self.projection.is_some() {+ return Ok(None);+ }+ try_embed_projection(projection, self)+ }+fn apply_expressions(&self,f: &mut dyn FnMut(&Arc<dyn crate::PhysicalExpr>) -> Result<TreeNodeRecursion>,@@ -1493,6 +1522,7 @@ mod tests {use datafusion_execution::runtime_env::RuntimeEnvBuilder;use datafusion_expr::ColumnarValue;use datafusion_physical_expr::expressions::{BinaryExpr, CastExpr};+ use datafusion_physical_expr::projection::ProjectionExpr;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use insta::assert_snapshot;@@ -1670,6 +1700,95 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn embeds_output_projection() -> Result<()> {+ let exec = Arc::new(test_exec()?.with_projection(None)?);+ let input = Arc::clone(&exec) as Arc<dyn ExecutionPlan>;+ let projection = ProjectionExec::try_new(+ [+ ProjectionExpr {+ expr: Arc::new(PhysicalColumn::new("id", 2)),+ alias: "id".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(PhysicalColumn::new("price", 5)),+ alias: "price".to_string(),+ },+ ],+ input,+ )?;++ let embedded = exec+ .try_swapping_with_projection(&projection)?+ .expect("projection should be embedded");+ let embedded_exec = embedded+ .downcast_ref::<AsOfJoinExec>()+ .expect("identity projection should be removed");+ assert_eq!(embedded_exec.projection.as_deref(), Some(&[2, 5][..]));+ assert_eq!(+ embedded_exec+ .schema()+ .fields()+ .iter()+ .map(|field| field.name().as_str())+ .collect::<Vec<_>>(),+ vec!["id", "price"]+ );++ let batches = collect(embedded, Arc::new(TaskContext::default())).await?;+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +----+-------++ | id | price |+ +----+-------++ | 0 | |+ | 1 | |+ | 2 | |+ | 3 | 40 |+ | 4 | 60 |+ | 5 | 101 |+ | 6 | |+ +----+-------++ ");+ Ok(())+ }++ #[tokio::test]+ async fn empty_projection_preserves_row_count() -> Result<()> {+ let exec = Arc::new(test_exec()?.with_projection(None)?);+ let input = Arc::clone(&exec) as Arc<dyn ExecutionPlan>;+ let projection = ProjectionExec::try_new(Vec::<ProjectionExpr>::new(), input)?;++ let embedded = exec+ .try_swapping_with_projection(&projection)?+ .expect("empty projection should be embedded");+ let embedded_exec = embedded+ .downcast_ref::<AsOfJoinExec>()+ .expect("empty projection should remove ProjectionExec");+ assert_eq!(embedded_exec.projection.as_deref(), Some(&[][..]));+ assert!(embedded_exec.schema().fields().is_empty());++ let batches = collect(embedded, Arc::new(TaskContext::default())).await?;+ assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 7);+ assert!(batches.iter().all(|batch| batch.num_columns() == 0));+ Ok(())+ }++ #[test]+ fn declines_projection_when_already_embedded() -> Result<()> {+ let exec = test_exec()?;+ let input = Arc::clone(&exec) as Arc<dyn ExecutionPlan>;+ let projection = ProjectionExec::try_new(+ [ProjectionExpr {+ expr: Arc::new(PhysicalColumn::new("id", 2)),+ alias: "id".to_string(),+ }],+ input,+ )?;++ assert!(exec.try_swapping_with_projection(&projection)?.is_none());+ Ok(())+ }+fn exec_without_equality_keys(left_times: Vec<i64>,right_times: Vec<i64>,diff --git a/datafusion/sqllogictest/test_files/asof_join.slt b/datafusion/sqllogictest/test_files/asof_join.sltindex 3e84cd0376..bcfa41b902 100644--- a/datafusion/sqllogictest/test_files/asof_join.slt+++ b/datafusion/sqllogictest/test_files/asof_join.slt@@ -214,14 +214,13 @@ logical_plan07)------Projection: column1 AS grp, column2 AS ts, column3 AS payload08)--------Values: (Int64(5), Int64(9), Int64(90))physical_plan-01)ProjectionExec: expr=[id@0 as id, payload@5 as payload]-02)--AsOfJoinExec: on=[(CAST(grp AS Int64) = grp)], match=[CAST(ts AS Int64) >= ts]-03)----SortExec: expr=[CAST(grp@1 AS Int64) ASC, CAST(ts@2 AS Int64) ASC], preserve_partitioning=[false]-04)------ProjectionExec: expr=[column1@0 as id, column2@1 as grp, column3@2 as ts]-05)--------DataSourceExec: partitions=1, partition_sizes=[1]-06)----ProjectionExec: expr=[column1@0 as grp, column2@1 as ts, column3@2 as payload]-07)------SortExec: expr=[column1@0 ASC, column2@1 ASC], preserve_partitioning=[false]-08)--------DataSourceExec: partitions=1, partition_sizes=[1]+01)AsOfJoinExec: on=[(CAST(grp AS Int64) = grp)], match=[CAST(ts AS Int64) >= ts], projection=[id@0, payload@5]+02)--SortExec: expr=[CAST(grp@1 AS Int64) ASC, CAST(ts@2 AS Int64) ASC], preserve_partitioning=[false]+03)----ProjectionExec: expr=[column1@0 as id, column2@1 as grp, column3@2 as ts]+04)------DataSourceExec: partitions=1, partition_sizes=[1]+05)--ProjectionExec: expr=[column1@0 as grp, column2@1 as ts, column3@2 as payload]+06)----SortExec: expr=[column1@0 ASC, column2@1 ASC], preserve_partitioning=[false]+07)------DataSourceExec: partitions=1, partition_sizes=[1]# Equality operands can name the right input first.query IT@@ -251,12 +250,11 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: asof_right projection=[grp, ts, val]physical_plan-01)ProjectionExec: expr=[id@0 as id, val@5 as val]-02)--AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts]-03)----SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[false]-04)------DataSourceExec: partitions=1, partition_sizes=[1]-05)----SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]-06)------DataSourceExec: partitions=1, partition_sizes=[1]+01)AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0, val@5]+02)--SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[false]+03)----DataSourceExec: partitions=1, partition_sizes=[1]+04)--SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]+05)----DataSourceExec: partitions=1, partition_sizes=[1]query error ASOF MATCH_CONDITION requires <, <=, >, or >=SELECT *