Skip to content

Apache DataFusion SQL Query Engine

test: consolidate ASOF SQL coverage (#24964)

Xuanwo549439f50aa4
diff --git a/datafusion/core/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rs
index b8bb32f0ed..534abad5f5 100644
--- a/datafusion/core/tests/sql/joins.rs
+++ b/datafusion/core/tests/sql/joins.rs
@@ -521,33 +521,6 @@ async fn asof_join_all_match_directions_across_batches() -> Result<()> {
Ok(())
}
-#[tokio::test]
-async fn asof_join_coerces_equality_and_match_types() -> Result<()> {
- let ctx = SessionContext::new();
- let batches = ctx
- .sql(
- "SELECT t.id, p.price \
- FROM (VALUES (CAST(1 AS INT), CAST(4 AS INT), 7)) t(k, ts, id) \
- ASOF JOIN \
- (VALUES (CAST(1 AS BIGINT), CAST(2 AS BIGINT), 20)) p(k, ts, price) \
- MATCH_CONDITION (t.ts >= p.ts) ON t.k = p.k",
- )
- .await?
- .collect()
- .await?;
- assert_batches_eq!(
- [
- "+----+-------+",
- "| id | price |",
- "+----+-------+",
- "| 7 | 20 |",
- "+----+-------+",
- ],
- &batches
- );
- Ok(())
-}
-
#[tokio::test]
async fn asof_join_broadcasts_multi_partition_right_input() -> Result<()> {
let config = SessionConfig::new().with_target_partitions(4);
@@ -639,28 +612,6 @@ async fn asof_join_projection_drops_output_ordering() -> Result<()> {
Ok(())
}
-#[tokio::test]
-async fn asof_join_explain_names_equality_and_match_conditions() -> Result<()> {
- let ctx = SessionContext::new();
- register_asof_test_tables(&ctx)?;
- let batches = ctx
- .sql(
- "EXPLAIN SELECT t.trade_id, p.price FROM trades t \
- ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) \
- ON t.symbol = p.symbol",
- )
- .await?
- .collect()
- .await?;
- let explain = arrow::util::pretty::pretty_format_batches(&batches)?.to_string();
- assert_contains!(explain.as_str(), "AsOf Join: match=[t.ts >= p.ts]");
- assert_contains!(explain.as_str(), "on=[t.symbol = p.symbol]");
- assert_contains!(explain.as_str(), "AsOfJoinExec:");
- assert_contains!(explain.as_str(), "on=[(symbol = symbol)]");
- assert_contains!(explain.as_str(), "match=[ts >= ts]");
- Ok(())
-}
-
#[tokio::test]
async fn asof_join_rejects_unbounded_inputs_during_physical_planning() -> Result<()> {
let ctx = SessionContext::new();
@@ -698,7 +649,7 @@ async fn asof_join_rejects_unbounded_inputs_during_physical_planning() -> Result
}
#[tokio::test]
-async fn asof_join_using_preserves_key_access_and_unparser_round_trips() -> Result<()> {
+async fn asof_join_using_unparser_round_trips() -> Result<()> {
let ctx = SessionContext::new();
register_asof_test_tables(&ctx)?;
let df = ctx
@@ -707,45 +658,11 @@ async fn asof_join_using_preserves_key_access_and_unparser_round_trips() -> Resu
MATCH_CONDITION (t.ts >= p.ts) USING (symbol)",
)
.await?;
- assert_eq!(
- df.schema()
- .fields()
- .iter()
- .map(|field| field.name())
- .collect::<Vec<_>>(),
- vec!["ts", "trade_id", "symbol", "ts", "price"]
- );
let sql = plan_to_sql(df.logical_plan())?.to_string();
assert!(sql.contains("ASOF JOIN"));
assert!(sql.contains("MATCH_CONDITION"));
assert!(sql.contains("USING(symbol)"), "unexpected SQL: {sql}");
ctx.sql(&sql).await?;
-
- let batches = ctx
- .sql(
- "SELECT t.trade_id, t.symbol AS left_symbol, p.symbol AS right_symbol \
- FROM trades t ASOF JOIN prices p \
- MATCH_CONDITION (t.ts >= p.ts) USING (symbol) \
- ORDER BY t.trade_id",
- )
- .await?
- .collect()
- .await?;
- assert_batches_eq!(
- [
- "+----------+-------------+--------------+",
- "| trade_id | left_symbol | right_symbol |",
- "+----------+-------------+--------------+",
- "| 1 | A | |",
- "| 2 | A | A |",
- "| 3 | A | A |",
- "| 4 | B | B |",
- "| 5 | B | B |",
- "| 6 | | |",
- "+----------+-------------+--------------+",
- ],
- &batches
- );
Ok(())
}
@@ -784,20 +701,3 @@ async fn asof_join_unparser_preserves_right_preselection() -> Result<()> {
}
Ok(())
}
-
-#[tokio::test]
-async fn asof_join_rejects_invalid_contracts() -> Result<()> {
- let ctx = SessionContext::new();
- register_asof_test_tables(&ctx)?;
- for sql in [
- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts = p.ts) ON t.symbol = p.symbol",
- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (p.ts >= t.ts) ON t.symbol = p.symbol",
- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) ON t.symbol > p.symbol",
- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (1 >= p.ts) ON t.symbol = p.symbol",
- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) ON 1 = 1",
- "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts AND t.ts > p.ts) ON t.symbol = p.symbol",
- ] {
- assert!(ctx.sql(sql).await.is_err(), "query should fail: {sql}");
- }
- Ok(())
-}
diff --git a/datafusion/sqllogictest/test_files/asof_join.slt b/datafusion/sqllogictest/test_files/asof_join.slt
index 55fb3b52aa..5d851c7020 100644
--- a/datafusion/sqllogictest/test_files/asof_join.slt
+++ b/datafusion/sqllogictest/test_files/asof_join.slt
@@ -138,6 +138,38 @@ ORDER BY l.id;
6 NULL NULL
7 A NULL
+# SELECT * exposes the USING key once, keeping the key whose qualifier sorts
+# first (`l` < `r` keeps the left key).
+query ITPPT
+SELECT *
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+USING (grp)
+ORDER BY l.id;
+----
+1 A 2024-01-01T09:00:01 NULL NULL
+2 A 2024-01-01T09:00:04 2024-01-01T09:00:04 a4
+3 A 2024-01-01T09:00:07 2024-01-01T09:00:06 a6
+4 B 2024-01-01T09:00:02 2024-01-01T09:00:01 b1
+5 B 2024-01-01T09:00:08 2024-01-01T09:00:06 b6
+6 NULL 2024-01-01T09:00:03 NULL NULL
+7 A NULL NULL NULL
+
+# With the right alias sorting first, SELECT * keeps the right key, which is
+# NULL for unmatched left rows.
+query IPTPT
+SELECT *
+FROM asof_left x
+ASOF JOIN asof_right a
+MATCH_CONDITION (x.ts >= a.ts)
+USING (grp)
+WHERE x.id IN (1, 2)
+ORDER BY x.id;
+----
+1 2024-01-01T09:00:01 NULL NULL NULL
+2 2024-01-01T09:00:04 A 2024-01-01T09:00:04 a4
+
# Equality keys are optional.
query IT
SELECT l.id, r.label
@@ -169,6 +201,25 @@ ORDER BY l.id;
1 x-a2
2 y-a3
+# USING accepts multiple equality keys.
+query IT
+SELECT l.id, r.val
+FROM (VALUES
+ (1, 'X', 'A', TIMESTAMP '2024-01-01 09:00:04'),
+ (2, 'Y', 'A', TIMESTAMP '2024-01-01 09:00:04')
+) AS l(id, venue, grp, ts)
+ASOF JOIN (VALUES
+ ('X', 'A', TIMESTAMP '2024-01-01 09:00:02', 'x-a2'),
+ ('Y', 'A', TIMESTAMP '2024-01-01 09:00:03', 'y-a3'),
+ ('X', 'B', TIMESTAMP '2024-01-01 09:00:04', 'x-b4')
+) AS r(venue, grp, ts, val)
+MATCH_CONDITION (l.ts >= r.ts)
+USING (venue, grp)
+ORDER BY l.id;
+----
+1 x-a2
+2 y-a3
+
# Candidate selection sees the right input after subquery filtering.
query IT
SELECT l.id, r.val
@@ -182,6 +233,150 @@ ORDER BY l.id;
3 a4
5 b6
+# A filter on a right output column is applied after candidate selection.
+query IT
+SELECT l.id, r.val
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = r.grp
+WHERE r.val <> 'a6'
+ORDER BY l.id;
+----
+2 a4
+4 b1
+5 b6
+
+query TT
+EXPLAIN SELECT l.id, r.val
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = r.grp
+WHERE r.val <> 'a6';
+----
+logical_plan
+01)Filter: r.val != Utf8View("a6")
+02)--Projection: l.id, r.val
+03)----AsOf Join: match=[l.ts >= r.ts], constraint=On, on=[l.grp = r.grp]
+04)------SubqueryAlias: l
+05)--------TableScan: asof_left projection=[id, grp, ts]
+06)------SubqueryAlias: r
+07)--------TableScan: asof_right projection=[grp, ts, val]
+physical_plan
+01)FilterExec: val@1 != a6
+02)--AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0, val@5]
+03)----SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[false]
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+05)----SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]
+06)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+# Projection pruning keeps only columns required by the ASOF contract.
+query TT
+EXPLAIN SELECT l.id
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = r.grp;
+----
+logical_plan
+01)Projection: l.id
+02)--AsOf Join: match=[l.ts >= r.ts], constraint=On, on=[l.grp = r.grp]
+03)----SubqueryAlias: l
+04)------TableScan: asof_left projection=[id, grp, ts]
+05)----SubqueryAlias: r
+06)------TableScan: asof_right projection=[grp, ts]
+physical_plan
+01)AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts], projection=[id@0]
+02)--SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[false]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+04)--SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]
+05)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+# Equality and ordered operands may be expressions.
+query IT
+SELECT l.id, r.val
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts + INTERVAL '1 second')
+ON lower(l.grp) = lower(r.grp)
+WHERE l.id IN (2, 3, 4)
+ORDER BY l.id;
+----
+2 a2
+3 a6
+4 b1
+
+# Empty inputs preserve left-join semantics.
+query IT
+SELECT l.id, r.val
+FROM asof_left l
+ASOF JOIN (SELECT * FROM asof_right WHERE false) r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = r.grp
+ORDER BY l.id;
+----
+1 NULL
+2 NULL
+3 NULL
+4 NULL
+5 NULL
+6 NULL
+7 NULL
+
+query I
+SELECT l.id
+FROM (SELECT * FROM asof_left WHERE false) l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = r.grp;
+----
+
+# Duplicate left rows are emitted independently.
+query IT
+SELECT l.id, r.val
+FROM (
+ SELECT * FROM asof_left
+ UNION ALL
+ SELECT * FROM asof_left
+) l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = r.grp
+ORDER BY l.id, r.val;
+----
+1 NULL
+1 NULL
+2 a4
+2 a4
+3 a6
+3 a6
+4 b1
+4 b1
+5 b6
+5 b6
+6 NULL
+6 NULL
+7 NULL
+7 NULL
+
+# ASOF JOIN supports independently aliased self joins.
+query II
+SELECT a.id, b.id
+FROM asof_left a
+ASOF JOIN asof_left b
+MATCH_CONDITION (a.ts > b.ts)
+ON a.grp = b.grp
+ORDER BY a.id;
+----
+1 NULL
+2 1
+3 2
+4 NULL
+5 4
+6 NULL
+7 NULL
+
# Equality and match operands use the planner's common coercion types.
query II
SELECT l.id, r.payload
@@ -552,6 +747,48 @@ ASOF JOIN asof_right r
MATCH_CONDITION (l.ts)
ON l.grp = r.grp;
+query error ASOF MATCH_CONDITION left operand must reference only the left input
+SELECT *
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (1 >= r.ts)
+ON l.grp = r.grp;
+
+query error Each ASOF equality condition must compare one left expression with one right expression
+SELECT *
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON 1 = 1;
+
+query error ASOF MATCH_CONDITION requires <, <=, >, or >=
+SELECT *
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts AND l.ts > r.ts)
+ON l.grp = r.grp;
+
+query error Each ASOF equality condition must compare one left expression with one right expression
+SELECT *
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = l.grp;
+
+query error ASOF ON accepts only equality conditions combined with AND
+SELECT *
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+ON l.grp = r.grp AND l.ts >= r.ts;
+
+query error Ambiguous reference to unqualified field ts
+SELECT ts
+FROM asof_left l
+ASOF JOIN asof_right r
+MATCH_CONDITION (l.ts >= r.ts)
+USING (grp);
+
# Correlated subqueries around an ASOF join. See
# https://github.com/apache/datafusion/issues/25834.
statement ok