Skip to content

Apache DataFusion SQL Query Engine

perf: embed projections in ASOF joins (#24873)

Xuanwo06e9fae2ef37
diff --git a/datafusion/core/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rs
index dd32830dd5..c013b70c04 100644
--- a/datafusion/core/tests/sql/joins.rs
+++ b/datafusion/core/tests/sql/joins.rs
@@ -490,7 +490,7 @@ async fn asof_join_broadcasts_multi_partition_right_input() -> Result<()> {
register_asof_test_tables(&ctx)?;
let df = ctx
.sql(
- "SELECT t.trade_id, p.price FROM trades t ASOF JOIN \
+ "SELECT t.trade_id, t.ts, p.price FROM trades t ASOF JOIN \
(SELECT ts, price FROM prices WHERE symbol = 'A') p \
MATCH_CONDITION (t.ts >= p.ts)",
)
@@ -519,6 +519,7 @@ async fn asof_join_broadcasts_multi_partition_right_input() -> Result<()> {
.to_string();
assert_contains!(right_plan.as_str(), "SortPreservingMergeExec");
assert_contains!(right_plan.as_str(), "DataSourceExec: partitions=2");
+ // Keep the left sort key in the embedded projection so the join can expose it.
assert!(asof.output_ordering().is_some());
assert!(matches!(
&asof.input_distribution_requirements().into_per_child()[..],
@@ -530,22 +531,49 @@ async fn asof_join_broadcasts_multi_partition_right_input() -> Result<()> {
let batches = collect(plan, ctx.task_ctx()).await?;
assert_batches_sorted_eq!(
[
- "+----------+-------+",
- "| trade_id | price |",
- "+----------+-------+",
- "| 1 | |",
- "| 2 | 40 |",
- "| 3 | 60 |",
- "| 4 | 20 |",
- "| 5 | 60 |",
- "| 6 | 20 |",
- "+----------+-------+",
+ "+----------+----+-------+",
+ "| trade_id | ts | price |",
+ "+----------+----+-------+",
+ "| 1 | 1 | |",
+ "| 2 | 4 | 40 |",
+ "| 3 | 7 | 60 |",
+ "| 4 | 2 | 20 |",
+ "| 5 | 8 | 60 |",
+ "| 6 | 3 | 20 |",
+ "+----------+----+-------+",
],
&batches
);
Ok(())
}
+#[tokio::test]
+async fn asof_join_projection_drops_output_ordering() -> Result<()> {
+ let ctx = SessionContext::new();
+ register_asof_test_tables(&ctx)?;
+ let plan = ctx
+ .sql(
+ "SELECT t.trade_id, p.price FROM trades t ASOF JOIN \
+ (SELECT ts, price FROM prices WHERE symbol = 'A') p \
+ MATCH_CONDITION (t.ts >= p.ts)",
+ )
+ .await?
+ .create_physical_plan()
+ .await?;
+ let asof = find_asof_exec(&plan).expect("physical ASOF join must be present");
+
+ assert_eq!(
+ asof.schema()
+ .fields()
+ .iter()
+ .map(|field| field.name().as_str())
+ .collect::<Vec<_>>(),
+ vec!["trade_id", "price"]
+ );
+ assert!(asof.output_ordering().is_none());
+ Ok(())
+}
+
#[tokio::test]
async fn asof_join_explain_names_equality_and_match_conditions() -> Result<()> {
let ctx = SessionContext::new();
diff --git a/datafusion/physical-plan/src/joins/asof_join.rs b/datafusion/physical-plan/src/joins/asof_join.rs
index abf1b37f1f..cb6e71eea7 100644
--- a/datafusion/physical-plan/src/joins/asof_join.rs
+++ b/datafusion/physical-plan/src/joins/asof_join.rs
@@ -110,6 +110,7 @@ use crate::metrics::{
BaselineMetrics, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricsSet,
RecordOutput, Time,
};
+use crate::projection::{EmbeddedProjection, ProjectionExec, try_embed_projection};
use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::RecordBatchStreamAdapter;
use crate::{
@@ -250,6 +251,18 @@ impl AsOfJoinExec {
})
}
+ /// Returns this join emitting only the columns in `projection`, in that order.
+ /// The indices address the join's own schema, before any projection.
+ pub fn with_projection(&self, projection: Option<Vec<usize>>) -> Result<Self> {
+ Self::try_new(
+ Arc::clone(&self.left),
+ Arc::clone(&self.right),
+ self.on.clone(),
+ self.match_condition.clone(),
+ projection,
+ )
+ }
+
fn compute_properties(
left: &Arc<dyn ExecutionPlan>,
join_schema: &SchemaRef,
@@ -293,6 +306,12 @@ impl AsOfJoinExec {
}
}
+impl EmbeddedProjection for AsOfJoinExec {
+ fn with_projection(&self, projection: Option<Vec<usize>>) -> Result<Self> {
+ self.with_projection(projection)
+ }
+}
+
impl DisplayAs for AsOfJoinExec {
fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter<'_>) -> std::fmt::Result {
let on = self
@@ -386,6 +405,16 @@ impl ExecutionPlan for AsOfJoinExec {
vec![&self.left, &self.right]
}
+ fn try_swapping_with_projection(
+ &self,
+ projection: &ProjectionExec,
+ ) -> Result<Option<Arc<dyn ExecutionPlan>>> {
+ if self.projection.is_some() {
+ return Ok(None);
+ }
+ try_embed_projection(projection, self)
+ }
+
fn apply_expressions(
&self,
f: &mut dyn FnMut(&Arc<dyn crate::PhysicalExpr>) -> Result<TreeNodeRecursion>,
@@ -1493,6 +1522,7 @@ mod tests {
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use datafusion_expr::ColumnarValue;
use datafusion_physical_expr::expressions::{BinaryExpr, CastExpr};
+ use datafusion_physical_expr::projection::ProjectionExpr;
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use insta::assert_snapshot;
@@ -1670,6 +1700,95 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn embeds_output_projection() -> Result<()> {
+ let exec = Arc::new(test_exec()?.with_projection(None)?);
+ let input = Arc::clone(&exec) as Arc<dyn ExecutionPlan>;
+ let projection = ProjectionExec::try_new(
+ [
+ ProjectionExpr {
+ expr: Arc::new(PhysicalColumn::new("id", 2)),
+ alias: "id".to_string(),
+ },
+ ProjectionExpr {
+ expr: Arc::new(PhysicalColumn::new("price", 5)),
+ alias: "price".to_string(),
+ },
+ ],
+ input,
+ )?;
+
+ let embedded = exec
+ .try_swapping_with_projection(&projection)?
+ .expect("projection should be embedded");
+ let embedded_exec = embedded
+ .downcast_ref::<AsOfJoinExec>()
+ .expect("identity projection should be removed");
+ assert_eq!(embedded_exec.projection.as_deref(), Some(&[2, 5][..]));
+ assert_eq!(
+ embedded_exec
+ .schema()
+ .fields()
+ .iter()
+ .map(|field| field.name().as_str())
+ .collect::<Vec<_>>(),
+ vec!["id", "price"]
+ );
+
+ let batches = collect(embedded, Arc::new(TaskContext::default())).await?;
+ assert_snapshot!(batches_to_sort_string(&batches), @r"
+ +----+-------+
+ | id | price |
+ +----+-------+
+ | 0 | |
+ | 1 | |
+ | 2 | |
+ | 3 | 40 |
+ | 4 | 60 |
+ | 5 | 101 |
+ | 6 | |
+ +----+-------+
+ ");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn empty_projection_preserves_row_count() -> Result<()> {
+ let exec = Arc::new(test_exec()?.with_projection(None)?);
+ let input = Arc::clone(&exec) as Arc<dyn ExecutionPlan>;
+ let projection = ProjectionExec::try_new(Vec::<ProjectionExpr>::new(), input)?;
+
+ let embedded = exec
+ .try_swapping_with_projection(&projection)?
+ .expect("empty projection should be embedded");
+ let embedded_exec = embedded
+ .downcast_ref::<AsOfJoinExec>()
+ .expect("empty projection should remove ProjectionExec");
+ assert_eq!(embedded_exec.projection.as_deref(), Some(&[][..]));
+ assert!(embedded_exec.schema().fields().is_empty());
+
+ let batches = collect(embedded, Arc::new(TaskContext::default())).await?;
+ assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 7);
+ assert!(batches.iter().all(|batch| batch.num_columns() == 0));
+ Ok(())
+ }
+
+ #[test]
+ fn declines_projection_when_already_embedded() -> Result<()> {
+ let exec = test_exec()?;
+ let input = Arc::clone(&exec) as Arc<dyn ExecutionPlan>;
+ let projection = ProjectionExec::try_new(
+ [ProjectionExpr {
+ expr: Arc::new(PhysicalColumn::new("id", 2)),
+ alias: "id".to_string(),
+ }],
+ input,
+ )?;
+
+ assert!(exec.try_swapping_with_projection(&projection)?.is_none());
+ Ok(())
+ }
+
fn exec_without_equality_keys(
left_times: Vec<i64>,
right_times: Vec<i64>,
diff --git a/datafusion/sqllogictest/test_files/asof_join.slt b/datafusion/sqllogictest/test_files/asof_join.slt
index 3e84cd0376..bcfa41b902 100644
--- a/datafusion/sqllogictest/test_files/asof_join.slt
+++ b/datafusion/sqllogictest/test_files/asof_join.slt
@@ -214,14 +214,13 @@ logical_plan
07)------Projection: column1 AS grp, column2 AS ts, column3 AS payload
08)--------Values: (Int64(5), Int64(9), Int64(90))
physical_plan
-01)ProjectionExec: expr=[id@0 as id, payload@5 as payload]
-02)--AsOfJoinExec: on=[(CAST(grp AS Int64) = grp)], match=[CAST(ts AS Int64) >= ts]
-03)----SortExec: expr=[CAST(grp@1 AS Int64) ASC, CAST(ts@2 AS Int64) ASC], preserve_partitioning=[false]
-04)------ProjectionExec: expr=[column1@0 as id, column2@1 as grp, column3@2 as ts]
-05)--------DataSourceExec: partitions=1, partition_sizes=[1]
-06)----ProjectionExec: expr=[column1@0 as grp, column2@1 as ts, column3@2 as payload]
-07)------SortExec: expr=[column1@0 ASC, column2@1 ASC], preserve_partitioning=[false]
-08)--------DataSourceExec: partitions=1, partition_sizes=[1]
+01)AsOfJoinExec: on=[(CAST(grp AS Int64) = grp)], match=[CAST(ts AS Int64) >= ts], projection=[id@0, payload@5]
+02)--SortExec: expr=[CAST(grp@1 AS Int64) ASC, CAST(ts@2 AS Int64) ASC], preserve_partitioning=[false]
+03)----ProjectionExec: expr=[column1@0 as id, column2@1 as grp, column3@2 as ts]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+05)--ProjectionExec: expr=[column1@0 as grp, column2@1 as ts, column3@2 as payload]
+06)----SortExec: expr=[column1@0 ASC, column2@1 ASC], preserve_partitioning=[false]
+07)------DataSourceExec: partitions=1, partition_sizes=[1]
# Equality operands can name the right input first.
query IT
@@ -251,12 +250,11 @@ logical_plan
05)----SubqueryAlias: r
06)------TableScan: asof_right projection=[grp, ts, val]
physical_plan
-01)ProjectionExec: expr=[id@0 as id, val@5 as val]
-02)--AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts]
-03)----SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[false]
-04)------DataSourceExec: partitions=1, partition_sizes=[1]
-05)----SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]
-06)------DataSourceExec: partitions=1, partition_sizes=[1]
+01)AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0, val@5]
+02)--SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[false]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+04)--SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]
+05)----DataSourceExec: partitions=1, partition_sizes=[1]
query error ASOF MATCH_CONDITION requires <, <=, >, or >=
SELECT *