Apache DataFusion SQL Query Engine
perf: mirror compound ASOF key predicates to right (#25728)
Xuanwo242b96c23163
diff --git a/benchmarks/sql_benchmarks/asof_join/asof_join.suite b/benchmarks/sql_benchmarks/asof_join/asof_join.suiteindex 9dff295976..f3d1ec19fc 100644--- a/benchmarks/sql_benchmarks/asof_join/asof_join.suite+++ b/benchmarks/sql_benchmarks/asof_join/asof_join.suite@@ -29,3 +29,7 @@ description = "Run the selective left-filter ASOF join query."[[examples]]command = "cargo run --release --bin benchmark_runner -- asof_join --query 9"description = "Run the equality-key filter ASOF join query."++[[examples]]+command = "cargo run --release --bin benchmark_runner -- asof_join --query 10"+description = "Run the compound equality-key filter ASOF join query."diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q10.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q10.benchmarknew file mode 100644index 0000000000..e09cee241e--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q10.benchmark@@ -0,0 +1,25 @@+name Q10+group asof_join++expect_plan AsOfJoinExec++run+-- Both sides: 50M rows in 500K equality groups. The IN predicate retains+-- two groups; mirroring it prunes the right input before sorting and broadcast.+WITH left_input AS (+ SELECT value % 500000 AS key,+ value / 500000 + 1 AS ts,+ value AS payload+ FROM range(50000000)+),+right_input AS (+ SELECT value % 500000 AS key,+ value / 500000 AS ts,+ value AS payload+ FROM range(50000000)+)+SELECT l.key, l.ts, l.payload, r.payload AS right_payload+FROM left_input l+ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts)+ON l.key = r.key+WHERE l.key IN (42, 43);diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex 64effaea82..b358c8abd2 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -38,7 +38,7 @@ use datafusion_expr::utils::{conjunction, expr_to_columns, split_conjunction, split_conjunction_owned,};use datafusion_expr::{- BinaryExpr, Distinct, Expr, ExprSchemable, Filter, Operator, Projection,+ BinaryExpr, Distinct, Expr, Filter, Operator, Projection,TableProviderFilterPushDown, and, or,};@@ -1168,41 +1168,27 @@ impl OptimizerRule for PushDownFilter {})});- // A literal comparison on an equal, same-typed key has the- // same value for every matching pair. Mirroring it to the- // right can prune groups without changing the ASOF candidate.+ // Every matching pair has equal key values, so a deterministic+ // predicate that refers only to left keys also holds for the+ // corresponding right keys. Expression-based join keys are not+ // mirrored because they cannot be replaced column-for-column.+ let key_replacements = join+ .on+ .iter()+ .filter_map(|(left, right)| {+ Some((left.try_as_col()?, right.try_as_col()?))+ })+ .collect::<HashMap<_, _>>();let mut right_predicates = Vec::new();for predicate in &push_predicates {- let Expr::BinaryExpr(BinaryExpr {- left,- op: Operator::Eq,- right,- }) = predicate- else {- continue;- };- let ((Expr::Column(left_column), Expr::Literal(_, _))- | (Expr::Literal(_, _), Expr::Column(left_column))) =- (left.as_ref(), right.as_ref())- else {- continue;- };- for (left_key, right_key) in &join.on {- let (Some(left_key_column), Some(right_key_column)) =- (left_key.try_as_col(), right_key.try_as_col())- else {- continue;- };- if left_column == left_key_column- && left_key.get_type(join.left.schema())?- == right_key.get_type(join.right.schema())?- {- let replacements =- HashMap::from([(left_key_column, right_key_column)]);- right_predicates- .push(replace_col(predicate.clone(), &replacements)?);- break;- }+ let columns = predicate.column_refs();+ if !columns.is_empty()+ && columns+ .iter()+ .all(|column| key_replacements.contains_key(column))+ {+ right_predicates+ .push(replace_col(predicate.clone(), &key_replacements)?);}}if let Some(predicate) = conjunction(right_predicates) {diff --git a/datafusion/sqllogictest/test_files/asof_join.slt b/datafusion/sqllogictest/test_files/asof_join.sltindex e745a95b0e..2e5160f4e5 100644--- a/datafusion/sqllogictest/test_files/asof_join.slt+++ b/datafusion/sqllogictest/test_files/asof_join.slt@@ -413,6 +413,62 @@ physical_plan07)----FilterExec: grp@0 = A08)------DataSourceExec: partitions=1, partition_sizes=[1]+# Any deterministic predicate that only references equality keys can be+# mirrored, including compound IN, range, and OR expressions.+query TT+EXPLAIN SELECT l.id, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp+WHERE l.grp IN ('A', 'B') OR l.grp > 'C';+----+logical_plan+01)Projection: l.id, r.val+02)--AsOf Join: match=[l.ts >= r.ts], constraint=On, on=[l.grp = r.grp]+03)----SubqueryAlias: l+04)------Filter: asof_left.grp = Utf8View("A") OR asof_left.grp = Utf8View("B") OR asof_left.grp > Utf8View("C")+05)--------TableScan: asof_left projection=[id, grp, ts]+06)----SubqueryAlias: r+07)------Filter: asof_right.grp = Utf8View("A") OR asof_right.grp = Utf8View("B") OR asof_right.grp > Utf8View("C")+08)--------TableScan: asof_right projection=[grp, ts, val]+physical_plan+01)AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0, val@5]+02)--SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[true]+03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+04)------FilterExec: grp@1 = A OR grp@1 = B OR grp@1 > C+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)--SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]+07)----FilterExec: grp@0 = A OR grp@0 = B OR grp@0 > C+08)------DataSourceExec: partitions=1, partition_sizes=[1]++# If any referenced column is not an equality key, the entire predicate must+# stay off the right input.+query TT+EXPLAIN SELECT l.id, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp+WHERE l.grp = 'A' OR l.id = 4;+----+logical_plan+01)Projection: l.id, r.val+02)--AsOf Join: match=[l.ts >= r.ts], constraint=On, on=[l.grp = r.grp]+03)----SubqueryAlias: l+04)------Filter: asof_left.grp = Utf8View("A") OR asof_left.id = Int32(4)+05)--------TableScan: asof_left projection=[id, grp, ts]+06)----SubqueryAlias: r+07)------TableScan: asof_right projection=[grp, ts, val]+physical_plan+01)AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0, val@5]+02)--SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[true]+03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+04)------FilterExec: grp@1 = A OR id@0 = 4+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)--SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]+07)----DataSourceExec: partitions=1, partition_sizes=[1]+# An ASOF join emits each left row exactly once, so the primary key from# asof_left remains unique. Because the outer LEFT JOIN does not use columns# from the ASOF side, it can be removed entirely.