Apache DataFusion SQL Query Engine
test: consolidate ASOF SQL coverage (#24964)
Xuanwo549439f50aa4
diff --git a/datafusion/core/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rsindex b8bb32f0ed..534abad5f5 100644--- a/datafusion/core/tests/sql/joins.rs+++ b/datafusion/core/tests/sql/joins.rs@@ -521,33 +521,6 @@ async fn asof_join_all_match_directions_across_batches() -> Result<()> {Ok(())}-#[tokio::test]-async fn asof_join_coerces_equality_and_match_types() -> Result<()> {- let ctx = SessionContext::new();- let batches = ctx- .sql(- "SELECT t.id, p.price \- FROM (VALUES (CAST(1 AS INT), CAST(4 AS INT), 7)) t(k, ts, id) \- ASOF JOIN \- (VALUES (CAST(1 AS BIGINT), CAST(2 AS BIGINT), 20)) p(k, ts, price) \- MATCH_CONDITION (t.ts >= p.ts) ON t.k = p.k",- )- .await?- .collect()- .await?;- assert_batches_eq!(- [- "+----+-------+",- "| id | price |",- "+----+-------+",- "| 7 | 20 |",- "+----+-------+",- ],- &batches- );- Ok(())-}-#[tokio::test]async fn asof_join_broadcasts_multi_partition_right_input() -> Result<()> {let config = SessionConfig::new().with_target_partitions(4);@@ -639,28 +612,6 @@ async fn asof_join_projection_drops_output_ordering() -> Result<()> {Ok(())}-#[tokio::test]-async fn asof_join_explain_names_equality_and_match_conditions() -> Result<()> {- let ctx = SessionContext::new();- register_asof_test_tables(&ctx)?;- let batches = ctx- .sql(- "EXPLAIN SELECT t.trade_id, p.price FROM trades t \- ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) \- ON t.symbol = p.symbol",- )- .await?- .collect()- .await?;- let explain = arrow::util::pretty::pretty_format_batches(&batches)?.to_string();- assert_contains!(explain.as_str(), "AsOf Join: match=[t.ts >= p.ts]");- assert_contains!(explain.as_str(), "on=[t.symbol = p.symbol]");- assert_contains!(explain.as_str(), "AsOfJoinExec:");- assert_contains!(explain.as_str(), "on=[(symbol = symbol)]");- assert_contains!(explain.as_str(), "match=[ts >= ts]");- Ok(())-}-#[tokio::test]async fn asof_join_rejects_unbounded_inputs_during_physical_planning() -> Result<()> {let ctx = SessionContext::new();@@ -698,7 +649,7 @@ async fn asof_join_rejects_unbounded_inputs_during_physical_planning() -> Result}#[tokio::test]-async fn asof_join_using_preserves_key_access_and_unparser_round_trips() -> Result<()> {+async fn asof_join_using_unparser_round_trips() -> Result<()> {let ctx = SessionContext::new();register_asof_test_tables(&ctx)?;let df = ctx@@ -707,45 +658,11 @@ async fn asof_join_using_preserves_key_access_and_unparser_round_trips() -> ResuMATCH_CONDITION (t.ts >= p.ts) USING (symbol)",).await?;- assert_eq!(- df.schema()- .fields()- .iter()- .map(|field| field.name())- .collect::<Vec<_>>(),- vec!["ts", "trade_id", "symbol", "ts", "price"]- );let sql = plan_to_sql(df.logical_plan())?.to_string();assert!(sql.contains("ASOF JOIN"));assert!(sql.contains("MATCH_CONDITION"));assert!(sql.contains("USING(symbol)"), "unexpected SQL: {sql}");ctx.sql(&sql).await?;-- let batches = ctx- .sql(- "SELECT t.trade_id, t.symbol AS left_symbol, p.symbol AS right_symbol \- FROM trades t ASOF JOIN prices p \- MATCH_CONDITION (t.ts >= p.ts) USING (symbol) \- ORDER BY t.trade_id",- )- .await?- .collect()- .await?;- assert_batches_eq!(- [- "+----------+-------------+--------------+",- "| trade_id | left_symbol | right_symbol |",- "+----------+-------------+--------------+",- "| 1 | A | |",- "| 2 | A | A |",- "| 3 | A | A |",- "| 4 | B | B |",- "| 5 | B | B |",- "| 6 | | |",- "+----------+-------------+--------------+",- ],- &batches- );Ok(())}@@ -784,20 +701,3 @@ async fn asof_join_unparser_preserves_right_preselection() -> Result<()> {}Ok(())}--#[tokio::test]-async fn asof_join_rejects_invalid_contracts() -> Result<()> {- let ctx = SessionContext::new();- register_asof_test_tables(&ctx)?;- for sql in [- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts = p.ts) ON t.symbol = p.symbol",- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (p.ts >= t.ts) ON t.symbol = p.symbol",- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) ON t.symbol > p.symbol",- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (1 >= p.ts) ON t.symbol = p.symbol",- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) ON 1 = 1",- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts AND t.ts > p.ts) ON t.symbol = p.symbol",- ] {- assert!(ctx.sql(sql).await.is_err(), "query should fail: {sql}");- }- Ok(())-}diff --git a/datafusion/sqllogictest/test_files/asof_join.slt b/datafusion/sqllogictest/test_files/asof_join.sltindex 55fb3b52aa..5d851c7020 100644--- a/datafusion/sqllogictest/test_files/asof_join.slt+++ b/datafusion/sqllogictest/test_files/asof_join.slt@@ -138,6 +138,38 @@ ORDER BY l.id;6 NULL NULL7 A NULL+# SELECT * exposes the USING key once, keeping the key whose qualifier sorts+# first (`l` < `r` keeps the left key).+query ITPPT+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+USING (grp)+ORDER BY l.id;+----+1 A 2024-01-01T09:00:01 NULL NULL+2 A 2024-01-01T09:00:04 2024-01-01T09:00:04 a4+3 A 2024-01-01T09:00:07 2024-01-01T09:00:06 a6+4 B 2024-01-01T09:00:02 2024-01-01T09:00:01 b1+5 B 2024-01-01T09:00:08 2024-01-01T09:00:06 b6+6 NULL 2024-01-01T09:00:03 NULL NULL+7 A NULL NULL NULL++# With the right alias sorting first, SELECT * keeps the right key, which is+# NULL for unmatched left rows.+query IPTPT+SELECT *+FROM asof_left x+ASOF JOIN asof_right a+MATCH_CONDITION (x.ts >= a.ts)+USING (grp)+WHERE x.id IN (1, 2)+ORDER BY x.id;+----+1 2024-01-01T09:00:01 NULL NULL NULL+2 2024-01-01T09:00:04 A 2024-01-01T09:00:04 a4+# Equality keys are optional.query ITSELECT l.id, r.label@@ -169,6 +201,25 @@ ORDER BY l.id;1 x-a22 y-a3+# USING accepts multiple equality keys.+query IT+SELECT l.id, r.val+FROM (VALUES+ (1, 'X', 'A', TIMESTAMP '2024-01-01 09:00:04'),+ (2, 'Y', 'A', TIMESTAMP '2024-01-01 09:00:04')+) AS l(id, venue, grp, ts)+ASOF JOIN (VALUES+ ('X', 'A', TIMESTAMP '2024-01-01 09:00:02', 'x-a2'),+ ('Y', 'A', TIMESTAMP '2024-01-01 09:00:03', 'y-a3'),+ ('X', 'B', TIMESTAMP '2024-01-01 09:00:04', 'x-b4')+) AS r(venue, grp, ts, val)+MATCH_CONDITION (l.ts >= r.ts)+USING (venue, grp)+ORDER BY l.id;+----+1 x-a2+2 y-a3+# Candidate selection sees the right input after subquery filtering.query ITSELECT l.id, r.val@@ -182,6 +233,150 @@ ORDER BY l.id;3 a45 b6+# A filter on a right output column is applied after candidate selection.+query IT+SELECT l.id, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp+WHERE r.val <> 'a6'+ORDER BY l.id;+----+2 a4+4 b1+5 b6++query TT+EXPLAIN SELECT l.id, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp+WHERE r.val <> 'a6';+----+logical_plan+01)Filter: r.val != Utf8View("a6")+02)--Projection: l.id, r.val+03)----AsOf Join: match=[l.ts >= r.ts], constraint=On, on=[l.grp = r.grp]+04)------SubqueryAlias: l+05)--------TableScan: asof_left projection=[id, grp, ts]+06)------SubqueryAlias: r+07)--------TableScan: asof_right projection=[grp, ts, val]+physical_plan+01)FilterExec: val@1 != a6+02)--AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0, val@5]+03)----SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[false]+04)------DataSourceExec: partitions=1, partition_sizes=[1]+05)----SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]+06)------DataSourceExec: partitions=1, partition_sizes=[1]++# Projection pruning keeps only columns required by the ASOF contract.+query TT+EXPLAIN SELECT l.id+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp;+----+logical_plan+01)Projection: l.id+02)--AsOf Join: match=[l.ts >= r.ts], constraint=On, on=[l.grp = r.grp]+03)----SubqueryAlias: l+04)------TableScan: asof_left projection=[id, grp, ts]+05)----SubqueryAlias: r+06)------TableScan: asof_right projection=[grp, ts]+physical_plan+01)AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0]+02)--SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[false]+03)----DataSourceExec: partitions=1, partition_sizes=[1]+04)--SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]+05)----DataSourceExec: partitions=1, partition_sizes=[1]++# Equality and ordered operands may be expressions.+query IT+SELECT l.id, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts + INTERVAL '1 second')+ON lower(l.grp) = lower(r.grp)+WHERE l.id IN (2, 3, 4)+ORDER BY l.id;+----+2 a2+3 a6+4 b1++# Empty inputs preserve left-join semantics.+query IT+SELECT l.id, r.val+FROM asof_left l+ASOF JOIN (SELECT * FROM asof_right WHERE false) r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp+ORDER BY l.id;+----+1 NULL+2 NULL+3 NULL+4 NULL+5 NULL+6 NULL+7 NULL++query I+SELECT l.id+FROM (SELECT * FROM asof_left WHERE false) l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp;+----++# Duplicate left rows are emitted independently.+query IT+SELECT l.id, r.val+FROM (+ SELECT * FROM asof_left+ UNION ALL+ SELECT * FROM asof_left+) l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp+ORDER BY l.id, r.val;+----+1 NULL+1 NULL+2 a4+2 a4+3 a6+3 a6+4 b1+4 b1+5 b6+5 b6+6 NULL+6 NULL+7 NULL+7 NULL++# ASOF JOIN supports independently aliased self joins.+query II+SELECT a.id, b.id+FROM asof_left a+ASOF JOIN asof_left b+MATCH_CONDITION (a.ts > b.ts)+ON a.grp = b.grp+ORDER BY a.id;+----+1 NULL+2 1+3 2+4 NULL+5 4+6 NULL+7 NULL+# Equality and match operands use the planner's common coercion types.query IISELECT l.id, r.payload@@ -552,6 +747,48 @@ ASOF JOIN asof_right rMATCH_CONDITION (l.ts)ON l.grp = r.grp;+query error ASOF MATCH_CONDITION left operand must reference only the left input+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (1 >= r.ts)+ON l.grp = r.grp;++query error Each ASOF equality condition must compare one left expression with one right expression+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON 1 = 1;++query error ASOF MATCH_CONDITION requires <, <=, >, or >=+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts AND l.ts > r.ts)+ON l.grp = r.grp;++query error Each ASOF equality condition must compare one left expression with one right expression+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = l.grp;++query error ASOF ON accepts only equality conditions combined with AND+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp AND l.ts >= r.ts;++query error Ambiguous reference to unqualified field ts+SELECT ts+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+USING (grp);+# Correlated subqueries around an ASOF join. See# https://github.com/apache/datafusion/issues/25834.statement ok