Skip to content

Apache DataFusion SQL Query Engine

perf: mirror compound ASOF key predicates to right (#25728)

Xuanwo242b96c23163
diff --git a/benchmarks/sql_benchmarks/asof_join/asof_join.suite b/benchmarks/sql_benchmarks/asof_join/asof_join.suite
index 9dff295976..f3d1ec19fc 100644
--- a/benchmarks/sql_benchmarks/asof_join/asof_join.suite
+++ b/benchmarks/sql_benchmarks/asof_join/asof_join.suite
@@ -29,3 +29,7 @@ description = "Run the selective left-filter ASOF join query."
[[examples]]
command = "cargo run --release --bin benchmark_runner -- asof_join --query 9"
description = "Run the equality-key filter ASOF join query."
+
+[[examples]]
+command = "cargo run --release --bin benchmark_runner -- asof_join --query 10"
+description = "Run the compound equality-key filter ASOF join query."
diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q10.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q10.benchmark
new file mode 100644
index 0000000000..e09cee241e
--- /dev/null
+++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q10.benchmark
@@ -0,0 +1,25 @@
+name Q10
+group asof_join
+
+expect_plan AsOfJoinExec
+
+run
+-- Both sides: 50M rows in 500K equality groups. The IN predicate retains
+-- two groups; mirroring it prunes the right input before sorting and broadcast.
+WITH left_input AS (
+ SELECT value % 500000 AS key,
+ value / 500000 + 1 AS ts,
+ value AS payload
+ FROM range(50000000)
+),
+right_input AS (
+ SELECT value % 500000 AS key,
+ value / 500000 AS ts,
+ value AS payload
+ FROM range(50000000)
+)
+SELECT l.key, l.ts, l.payload, r.payload AS right_payload
+FROM left_input l
+ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts)
+ON l.key = r.key
+WHERE l.key IN (42, 43);
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs
index 64effaea82..b358c8abd2 100644
--- a/datafusion/optimizer/src/push_down_filter.rs
+++ b/datafusion/optimizer/src/push_down_filter.rs
@@ -38,7 +38,7 @@ use datafusion_expr::utils::{
conjunction, expr_to_columns, split_conjunction, split_conjunction_owned,
};
use datafusion_expr::{
- BinaryExpr, Distinct, Expr, ExprSchemable, Filter, Operator, Projection,
+ BinaryExpr, Distinct, Expr, Filter, Operator, Projection,
TableProviderFilterPushDown, and, or,
};
@@ -1168,41 +1168,27 @@ impl OptimizerRule for PushDownFilter {
})
});
- // A literal comparison on an equal, same-typed key has the
- // same value for every matching pair. Mirroring it to the
- // right can prune groups without changing the ASOF candidate.
+ // Every matching pair has equal key values, so a deterministic
+ // predicate that refers only to left keys also holds for the
+ // corresponding right keys. Expression-based join keys are not
+ // mirrored because they cannot be replaced column-for-column.
+ let key_replacements = join
+ .on
+ .iter()
+ .filter_map(|(left, right)| {
+ Some((left.try_as_col()?, right.try_as_col()?))
+ })
+ .collect::<HashMap<_, _>>();
let mut right_predicates = Vec::new();
for predicate in &push_predicates {
- let Expr::BinaryExpr(BinaryExpr {
- left,
- op: Operator::Eq,
- right,
- }) = predicate
- else {
- continue;
- };
- let ((Expr::Column(left_column), Expr::Literal(_, _))
- | (Expr::Literal(_, _), Expr::Column(left_column))) =
- (left.as_ref(), right.as_ref())
- else {
- continue;
- };
- for (left_key, right_key) in &join.on {
- let (Some(left_key_column), Some(right_key_column)) =
- (left_key.try_as_col(), right_key.try_as_col())
- else {
- continue;
- };
- if left_column == left_key_column
- && left_key.get_type(join.left.schema())?
- == right_key.get_type(join.right.schema())?
- {
- let replacements =
- HashMap::from([(left_key_column, right_key_column)]);
- right_predicates
- .push(replace_col(predicate.clone(), &replacements)?);
- break;
- }
+ let columns = predicate.column_refs();
+ if !columns.is_empty()
+ && columns
+ .iter()
+ .all(|column| key_replacements.contains_key(column))
+ {
+ right_predicates
+ .push(replace_col(predicate.clone(), &key_replacements)?);
}
}
if let Some(predicate) = conjunction(right_predicates) {
diff --git a/datafusion/sqllogictest/test_files/asof_join.slt b/datafusion/sqllogictest/test_files/asof_join.slt
index e745a95b0e..2e5160f4e5 100644
--- a/datafusion/sqllogictest/test_files/asof_join.slt
+++ b/datafusion/sqllogictest/test_files/asof_join.slt
@@ -413,6 +413,62 @@ physical_plan
07)----FilterExec: grp@0 = A
08)------DataSourceExec: partitions=1, partition_sizes=[1]
+# Any deterministic predicate that only references equality keys can be
+# mirrored, including compound IN, range, and OR expressions.
+query TT
+EXPLAIN SELECT l.id, r.val
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = r.grp
+WHERE l.grp IN ('A', 'B') OR l.grp > 'C';
+----
+logical_plan
+01)Projection: l.id, r.val
+02)--AsOf Join: match=[l.ts >= r.ts], constraint=On, on=[l.grp = r.grp]
+03)----SubqueryAlias: l
+04)------Filter: asof_left.grp = Utf8View("A") OR asof_left.grp = Utf8View("B") OR asof_left.grp > Utf8View("C")
+05)--------TableScan: asof_left projection=[id, grp, ts]
+06)----SubqueryAlias: r
+07)------Filter: asof_right.grp = Utf8View("A") OR asof_right.grp = Utf8View("B") OR asof_right.grp > Utf8View("C")
+08)--------TableScan: asof_right projection=[grp, ts, val]
+physical_plan
+01)AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0, val@5]
+02)--SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[true]
+03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+04)------FilterExec: grp@1 = A OR grp@1 = B OR grp@1 > C
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+06)--SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]
+07)----FilterExec: grp@0 = A OR grp@0 = B OR grp@0 > C
+08)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# If any referenced column is not an equality key, the entire predicate must
+# stay off the right input.
+query TT
+EXPLAIN SELECT l.id, r.val
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = r.grp
+WHERE l.grp = 'A' OR l.id = 4;
+----
+logical_plan
+01)Projection: l.id, r.val
+02)--AsOf Join: match=[l.ts >= r.ts], constraint=On, on=[l.grp = r.grp]
+03)----SubqueryAlias: l
+04)------Filter: asof_left.grp = Utf8View("A") OR asof_left.id = Int32(4)
+05)--------TableScan: asof_left projection=[id, grp, ts]
+06)----SubqueryAlias: r
+07)------TableScan: asof_right projection=[grp, ts, val]
+physical_plan
+01)AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0, val@5]
+02)--SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[true]
+03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+04)------FilterExec: grp@1 = A OR id@0 = 4
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
+06)--SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]
+07)----DataSourceExec: partitions=1, partition_sizes=[1]
+
# An ASOF join emits each left row exactly once, so the primary key from
# asof_left remains unique. Because the outer LEFT JOIN does not use columns
# from the ASOF side, it can be removed entirely.