Apache DataFusion SQL Query Engine
feat: support ASOF JOIN SQL (#23830)
Xuanwof40d978a4c72
diff --git a/datafusion/core/tests/sql/joins.rs b/datafusion/core/tests/sql/joins.rsindex bb526895b6..dd32830dd5 100644--- a/datafusion/core/tests/sql/joins.rs+++ b/datafusion/core/tests/sql/joins.rs@@ -17,10 +17,12 @@use insta::assert_snapshot;-use datafusion::assert_batches_eq;use datafusion::catalog::MemTable;use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};+use datafusion::physical_plan::joins::AsOfJoinExec;+use datafusion::physical_plan::{Distribution, ExecutionPlanProperties};use datafusion::test_util::register_unbounded_file_with_ordering;+use datafusion::{assert_batches_eq, assert_batches_sorted_eq};use datafusion_sql::unparser::plan_to_sql;use super::*;@@ -297,3 +299,412 @@ async fn unparse_cross_join() -> Result<()> {Ok(())}++fn register_asof_test_tables(ctx: &SessionContext) -> Result<()> {+ let trades_schema = Arc::new(Schema::new(vec![+ Field::new("symbol", DataType::Utf8, true),+ Field::new("ts", DataType::Int64, true),+ Field::new("trade_id", DataType::Int32, false),+ ]));+ let trades = vec![+ RecordBatch::try_new(+ Arc::clone(&trades_schema),+ vec![+ Arc::new(StringArray::from(vec![Some("A"), Some("B"), None])),+ Arc::new(Int64Array::from(vec![Some(7), Some(2), Some(3)])),+ Arc::new(Int32Array::from(vec![3, 4, 6])),+ ],+ )?,+ RecordBatch::try_new(+ Arc::clone(&trades_schema),+ vec![+ Arc::new(StringArray::from(vec![Some("A"), Some("A"), Some("B")])),+ Arc::new(Int64Array::from(vec![Some(1), Some(4), Some(8)])),+ Arc::new(Int32Array::from(vec![1, 2, 5])),+ ],+ )?,+ ];+ ctx.register_table(+ "trades",+ Arc::new(MemTable::try_new(+ trades_schema,+ trades.into_iter().map(|batch| vec![batch]).collect(),+ )?),+ )?;++ let prices_schema = Arc::new(Schema::new(vec![+ Field::new("symbol", DataType::Utf8, true),+ Field::new("ts", DataType::Int64, true),+ Field::new("price", DataType::Int32, false),+ ]));+ let prices = vec![+ RecordBatch::try_new(+ Arc::clone(&prices_schema),+ vec![+ Arc::new(StringArray::from(vec![Some("A"), Some("B"), None])),+ Arc::new(Int64Array::from(vec![Some(6), Some(1), Some(2)])),+ Arc::new(Int32Array::from(vec![60, 101, 999])),+ ],+ )?,+ RecordBatch::try_new(+ Arc::clone(&prices_schema),+ vec![+ Arc::new(StringArray::from(vec![Some("A"), Some("A"), Some("B")])),+ Arc::new(Int64Array::from(vec![Some(2), Some(4), Some(6)])),+ Arc::new(Int32Array::from(vec![20, 40, 106])),+ ],+ )?,+ ];+ ctx.register_table(+ "prices",+ Arc::new(MemTable::try_new(+ prices_schema,+ prices.into_iter().map(|batch| vec![batch]).collect(),+ )?),+ )?;+ Ok(())+}++fn find_asof_exec(plan: &Arc<dyn ExecutionPlan>) -> Option<Arc<dyn ExecutionPlan>> {+ if plan.downcast_ref::<AsOfJoinExec>().is_some() {+ return Some(Arc::clone(plan));+ }+ plan.children().into_iter().find_map(find_asof_exec)+}++#[tokio::test]+async fn asof_join_all_match_directions_across_batches() -> Result<()> {+ let config = SessionConfig::new()+ .with_batch_size(2)+ .with_target_partitions(2);+ let ctx = SessionContext::new_with_config(config);+ register_asof_test_tables(&ctx)?;++ for (op, expected) in [+ (+ ">=",+ [+ "+----------+-------+",+ "| trade_id | price |",+ "+----------+-------+",+ "| 1 | |",+ "| 2 | 40 |",+ "| 3 | 60 |",+ "| 4 | 101 |",+ "| 5 | 106 |",+ "| 6 | |",+ "+----------+-------+",+ ],+ ),+ (+ ">",+ [+ "+----------+-------+",+ "| trade_id | price |",+ "+----------+-------+",+ "| 1 | |",+ "| 2 | 20 |",+ "| 3 | 60 |",+ "| 4 | 101 |",+ "| 5 | 106 |",+ "| 6 | |",+ "+----------+-------+",+ ],+ ),+ (+ "<=",+ [+ "+----------+-------+",+ "| trade_id | price |",+ "+----------+-------+",+ "| 1 | 20 |",+ "| 2 | 40 |",+ "| 3 | |",+ "| 4 | 106 |",+ "| 5 | |",+ "| 6 | |",+ "+----------+-------+",+ ],+ ),+ (+ "<",+ [+ "+----------+-------+",+ "| trade_id | price |",+ "+----------+-------+",+ "| 1 | 20 |",+ "| 2 | 60 |",+ "| 3 | |",+ "| 4 | 106 |",+ "| 5 | |",+ "| 6 | |",+ "+----------+-------+",+ ],+ ),+ ] {+ let batches = ctx+ .sql(&format!(+ "SELECT t.trade_id, p.price FROM trades t \+ ASOF JOIN prices p MATCH_CONDITION (t.ts {op} p.ts) \+ ON t.symbol = p.symbol ORDER BY t.trade_id"+ ))+ .await?+ .collect()+ .await?;+ assert_batches_eq!(expected, &batches);+ }+ Ok(())+}++#[tokio::test]+async fn asof_join_coerces_equality_and_match_types() -> Result<()> {+ let ctx = SessionContext::new();+ let batches = ctx+ .sql(+ "SELECT t.id, p.price \+ FROM (VALUES (CAST(1 AS INT), CAST(4 AS INT), 7)) t(k, ts, id) \+ ASOF JOIN \+ (VALUES (CAST(1 AS BIGINT), CAST(2 AS BIGINT), 20)) p(k, ts, price) \+ MATCH_CONDITION (t.ts >= p.ts) ON t.k = p.k",+ )+ .await?+ .collect()+ .await?;+ assert_batches_eq!(+ [+ "+----+-------+",+ "| id | price |",+ "+----+-------+",+ "| 7 | 20 |",+ "+----+-------+",+ ],+ &batches+ );+ Ok(())+}++#[tokio::test]+async fn asof_join_broadcasts_multi_partition_right_input() -> Result<()> {+ let config = SessionConfig::new().with_target_partitions(4);+ let ctx = SessionContext::new_with_config(config);+ register_asof_test_tables(&ctx)?;+ let df = ctx+ .sql(+ "SELECT t.trade_id, p.price FROM trades t ASOF JOIN \+ (SELECT ts, price FROM prices WHERE symbol = 'A') p \+ MATCH_CONDITION (t.ts >= p.ts)",+ )+ .await?;+ let sql = plan_to_sql(df.logical_plan())?.to_string();+ assert_contains!(sql.as_str(), "ASOF JOIN");+ assert!(!sql.contains(" ON "), "unexpected equality clause: {sql}");+ ctx.sql(&sql).await?;+ let plan = df.create_physical_plan().await?;+ let asof = find_asof_exec(&plan).expect("physical ASOF join must be present");+ let output_partitions = asof.output_partitioning().partition_count();+ assert_eq!(+ output_partitions,+ asof.children()[0].output_partitioning().partition_count()+ );+ assert!(+ output_partitions > 1,+ "ASOF join did not preserve left-side parallelism"+ );+ assert_eq!(+ asof.children()[1].output_partitioning().partition_count(),+ 1+ );+ let right_plan = displayable(asof.children()[1].as_ref())+ .indent(true)+ .to_string();+ assert_contains!(right_plan.as_str(), "SortPreservingMergeExec");+ assert_contains!(right_plan.as_str(), "DataSourceExec: partitions=2");+ assert!(asof.output_ordering().is_some());+ assert!(matches!(+ &asof.input_distribution_requirements().into_per_child()[..],+ [+ Distribution::UnspecifiedDistribution,+ Distribution::SinglePartition+ ]+ ));+ let batches = collect(plan, ctx.task_ctx()).await?;+ assert_batches_sorted_eq!(+ [+ "+----------+-------+",+ "| trade_id | price |",+ "+----------+-------+",+ "| 1 | |",+ "| 2 | 40 |",+ "| 3 | 60 |",+ "| 4 | 20 |",+ "| 5 | 60 |",+ "| 6 | 20 |",+ "+----------+-------+",+ ],+ &batches+ );+ Ok(())+}++#[tokio::test]+async fn asof_join_explain_names_equality_and_match_conditions() -> Result<()> {+ let ctx = SessionContext::new();+ register_asof_test_tables(&ctx)?;+ let batches = ctx+ .sql(+ "EXPLAIN SELECT t.trade_id, p.price FROM trades t \+ ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) \+ ON t.symbol = p.symbol",+ )+ .await?+ .collect()+ .await?;+ let explain = arrow::util::pretty::pretty_format_batches(&batches)?.to_string();+ assert_contains!(explain.as_str(), "AsOf Join: match=[t.ts >= p.ts]");+ assert_contains!(explain.as_str(), "on=[t.symbol = p.symbol]");+ assert_contains!(explain.as_str(), "AsOfJoinExec:");+ assert_contains!(explain.as_str(), "on=[(symbol = symbol)]");+ assert_contains!(explain.as_str(), "match=[ts >= ts]");+ Ok(())+}++#[tokio::test]+async fn asof_join_rejects_unbounded_inputs_during_physical_planning() -> Result<()> {+ let ctx = SessionContext::new();+ let tmp_dir = TempDir::new()?;+ let schema = Arc::new(Schema::new(vec![+ Field::new("symbol", DataType::UInt32, false),+ Field::new("ts", DataType::UInt32, false),+ ]));+ let ordering = vec![vec![+ col("symbol").sort(true, true),+ col("ts").sort(true, true),+ ]];+ for table in ["left_stream", "right_stream"] {+ let path = tmp_dir.path().join(format!("{table}.csv"));+ File::create(&path)?;+ register_unbounded_file_with_ordering(+ &ctx,+ Arc::clone(&schema),+ &path,+ table,+ ordering.clone(),+ )?;+ }+ let error = ctx+ .sql(+ "SELECT * FROM left_stream l ASOF JOIN right_stream r \+ MATCH_CONDITION (l.ts >= r.ts) ON l.symbol = r.symbol",+ )+ .await?+ .create_physical_plan()+ .await+ .expect_err("ASOF physical planning must reject unbounded inputs");+ assert_contains!(error.to_string(), "AsOfJoinExec requires bounded inputs");+ Ok(())+}++#[tokio::test]+async fn asof_join_using_preserves_key_access_and_unparser_round_trips() -> Result<()> {+ let ctx = SessionContext::new();+ register_asof_test_tables(&ctx)?;+ let df = ctx+ .sql(+ "SELECT * FROM trades t ASOF JOIN prices p \+ MATCH_CONDITION (t.ts >= p.ts) USING (symbol)",+ )+ .await?;+ assert_eq!(+ df.schema()+ .fields()+ .iter()+ .map(|field| field.name())+ .collect::<Vec<_>>(),+ vec!["ts", "trade_id", "symbol", "ts", "price"]+ );+ let sql = plan_to_sql(df.logical_plan())?.to_string();+ assert!(sql.contains("ASOF JOIN"));+ assert!(sql.contains("MATCH_CONDITION"));+ assert!(sql.contains("USING(symbol)"), "unexpected SQL: {sql}");+ ctx.sql(&sql).await?;++ let batches = ctx+ .sql(+ "SELECT t.trade_id, t.symbol AS left_symbol, p.symbol AS right_symbol \+ FROM trades t ASOF JOIN prices p \+ MATCH_CONDITION (t.ts >= p.ts) USING (symbol) \+ ORDER BY t.trade_id",+ )+ .await?+ .collect()+ .await?;+ assert_batches_eq!(+ [+ "+----------+-------------+--------------+",+ "| trade_id | left_symbol | right_symbol |",+ "+----------+-------------+--------------+",+ "| 1 | A | |",+ "| 2 | A | A |",+ "| 3 | A | A |",+ "| 4 | B | B |",+ "| 5 | B | B |",+ "| 6 | | |",+ "+----------+-------------+--------------+",+ ],+ &batches+ );+ Ok(())+}++#[tokio::test]+async fn asof_join_unparser_preserves_right_preselection() -> Result<()> {+ let ctx = SessionContext::new();+ register_asof_test_tables(&ctx)?;+ for query in [+ "SELECT t.trade_id, p.price FROM trades t \+ ASOF JOIN (SELECT * FROM prices WHERE price < 100) p \+ MATCH_CONDITION (t.ts >= p.ts) ON t.symbol = p.symbol \+ ORDER BY t.trade_id",+ "SELECT * FROM trades t \+ ASOF JOIN (SELECT * FROM prices WHERE price < 100) p \+ MATCH_CONDITION (t.ts >= p.ts) USING (symbol) \+ ORDER BY t.trade_id",+ "SELECT t.trade_id, p.price FROM trades t \+ JOIN prices q ON t.symbol = q.symbol AND t.ts = q.ts \+ ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) \+ ON q.symbol = p.symbol ORDER BY t.trade_id",+ "SELECT t.trade_id, q.trade_id FROM trades t \+ ASOF JOIN (prices p JOIN trades q \+ ON p.symbol = q.symbol AND p.ts = q.ts) \+ MATCH_CONDITION (t.ts >= q.ts) ON t.symbol = p.symbol \+ ORDER BY t.trade_id",+ ] {+ let expected = ctx.sql(query).await?.collect().await?;+ let plan = ctx.sql(query).await?.into_optimized_plan()?;+ let sql = plan_to_sql(&plan)?.to_string();+ let actual = ctx.sql(&sql).await?.collect().await?;+ assert_eq!(+ datafusion_common::test_util::batches_to_string(&expected),+ datafusion_common::test_util::batches_to_string(&actual),+ "unparsed SQL changed ASOF candidate preselection: {sql}"+ );+ }+ Ok(())+}++#[tokio::test]+async fn asof_join_rejects_invalid_contracts() -> Result<()> {+ let ctx = SessionContext::new();+ register_asof_test_tables(&ctx)?;+ for sql in [+ "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts = p.ts) ON t.symbol = p.symbol",+ "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (p.ts >= t.ts) ON t.symbol = p.symbol",+ "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) ON t.symbol > p.symbol",+ "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (1 >= p.ts) ON t.symbol = p.symbol",+ "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts) ON 1 = 1",+ "SELECT * FROM trades t ASOF JOIN prices p MATCH_CONDITION (t.ts >= p.ts AND t.ts > p.ts) ON t.symbol = p.symbol",+ ] {+ assert!(ctx.sql(sql).await.is_err(), "query should fail: {sql}");+ }+ Ok(())+}diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex 330509a5eb..b2522c8533 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -38,13 +38,15 @@ use crate::logical_plan::{};use crate::select_expr::SelectExpr;use crate::utils::{- can_hash, columnize_expr, compare_sort_expr, expand_qualified_wildcard,- expand_wildcard, expr_to_columns, find_valid_equijoin_key_pair,- group_window_expr_by_sort_keys,+ can_hash, check_all_columns_from_schema, columnize_expr, compare_sort_expr,+ expand_qualified_wildcard, expand_wildcard, expr_to_columns,+ find_valid_equijoin_key_pair, group_window_expr_by_sort_keys,+ split_conjunction_owned,};use crate::{- DmlStatement, ExplainOption, Expr, ExprSchemable, Operator, RecursiveQuery,- Statement, TableProviderFilterPushDown, TableSource, WriteOp, and, binary_expr, lit,+ BinaryExpr, DmlStatement, ExplainOption, Expr, ExprSchemable, Operator,+ RecursiveQuery, Statement, TableProviderFilterPushDown, TableSource, WriteOp, and,+ binary_expr, lit,};use super::dml::InsertOp;@@ -1007,23 +1009,67 @@ impl LogicalPlanBuilder {)}- /// Apply a left-preserving ASOF join using equality expressions and one- /// ordered match condition.- pub fn asof_join(+ /// Apply a left-preserving ASOF join using an optional equality condition+ /// and one ordered match condition.+ ///+ /// When present, `on_expr` must contain equality comparisons combined with+ /// `AND`. Each comparison must have one operand that references only the+ /// left input and one that references only `right`; their order does not+ /// matter. `match_condition` must be a single `<`, `<=`, `>`, or `>=`+ /// comparison whose left operand references only the left input and whose+ /// right operand references only `right`.+ pub fn asof_join_on(self,right: LogicalPlan,- on: Vec<(Expr, Expr)>,- match_condition: AsOfMatch,+ on_expr: Option<Expr>,+ match_condition: Expr,) -> Result<Self> {- self.asof_join_with_constraint(right, on, match_condition, JoinConstraint::On)+ let on = on_expr+ .into_iter()+ .flat_map(split_conjunction_owned)+ .map(|predicate| {+ let Expr::BinaryExpr(BinaryExpr {+ left,+ op: Operator::Eq,+ right: right_expr,+ }) = predicate+ else {+ return plan_err!(+ "ASOF ON accepts only equality conditions combined with AND"+ );+ };+ find_valid_equijoin_key_pair(+ &left,+ &right_expr,+ self.plan.schema(),+ right.schema(),+ )?+ .ok_or_else(|| {+ plan_datafusion_err!(+ "Each ASOF equality condition must compare one left expression with one right expression"+ )+ })+ })+ .collect::<Result<_>>()?;+ self.asof_join_with_constraint(+ right,+ on,+ AsOfMatch::try_from(match_condition)?,+ JoinConstraint::On,+ )}- /// Apply a left-preserving ASOF join using `USING` equality keys.+ /// Apply a left-preserving ASOF join using `USING` equality keys and one+ /// ordered match condition.+ ///+ /// Every key in `using_keys` must resolve in both inputs.+ /// `match_condition` follows the same operand and operator requirements as+ /// [`asof_join_on`](Self::asof_join_on).pub fn asof_join_using(self,right: LogicalPlan,using_keys: Vec<Column>,- match_condition: AsOfMatch,+ match_condition: Expr,) -> Result<Self> {let on = using_keys.into_iter()@@ -1033,7 +1079,12 @@ impl LogicalPlanBuilder {Ok((Expr::Column(left), Expr::Column(right)))}).collect::<Result<_>>()?;- self.asof_join_with_constraint(right, on, match_condition, JoinConstraint::Using)+ self.asof_join_with_constraint(+ right,+ on,+ AsOfMatch::try_from(match_condition)?,+ JoinConstraint::Using,+ )}fn asof_join_with_constraint(@@ -1043,6 +1094,17 @@ impl LogicalPlanBuilder {match_condition: AsOfMatch,join_constraint: JoinConstraint,) -> Result<Self> {+ let left_columns = match_condition.left.column_refs();+ let right_columns = match_condition.right.column_refs();+ if left_columns.is_empty()+ || right_columns.is_empty()+ || !check_all_columns_from_schema(&left_columns, self.plan.schema())?+ || !check_all_columns_from_schema(&right_columns, right.schema())?+ {+ return plan_err!(+ "ASOF MATCH_CONDITION left operand must reference only the left input and right operand only the right input"+ );+ }let normalize = |expr, schema: &DFSchema| {normalize_col_with_schemas_and_ambiguity_check(expr, &[&[schema]], &[])};@@ -2905,6 +2967,70 @@ mod tests {Ok(())}+ #[test]+ fn asof_join_on_extracts_and_validates_conditions() -> Result<()> {+ let values = vec![vec![lit(1), lit(2)]];+ let left = LogicalPlanBuilder::values(values.clone())?+ .alias("l")?+ .build()?;+ let right = LogicalPlanBuilder::values(values)?.alias("r")?.build()?;++ let plan = LogicalPlanBuilder::from(left.clone())+ .asof_join_on(+ right.clone(),+ Some(+ col("r.column1")+ .eq(col("l.column1"))+ .and(col("l.column2").eq(col("r.column2"))),+ ),+ col("l.column2").gt_eq(col("r.column2")),+ )?+ .build()?;+ let LogicalPlan::AsOfJoin(join) = plan else {+ panic!("expected ASOF join")+ };+ assert_eq!(+ join.on,+ vec![+ (col("l.column1"), col("r.column1")),+ (col("l.column2"), col("r.column2")),+ ]+ );+ assert_eq!(+ join.match_condition.as_ref(),+ &AsOfMatch::new(col("l.column2"), Operator::GtEq, col("r.column2"))+ );++ let invalid_on = LogicalPlanBuilder::from(left.clone())+ .asof_join_on(+ right.clone(),+ Some(col("l.column1").gt(col("r.column1"))),+ col("l.column2").gt_eq(col("r.column2")),+ )+ .expect_err("non-equality ASOF ON should fail");+ assert_snapshot!(invalid_on.strip_backtrace(), @r#"Error during planning: ASOF ON accepts only equality conditions combined with AND"#);++ let invalid_match = LogicalPlanBuilder::from(left.clone())+ .asof_join_on(+ right.clone(),+ Some(col("l.column1").eq(col("r.column1"))),+ col("l.column2").eq(col("r.column2")),+ )+ .expect_err("equality ASOF MATCH_CONDITION should fail");+ assert_snapshot!(invalid_match.strip_backtrace(), @r#"Error during planning: ASOF MATCH_CONDITION requires <, <=, >, or >=, found ="#);++ let reversed_match = LogicalPlanBuilder::from(left)+ .asof_join_on(+ right,+ Some(col("l.column1").eq(col("r.column1"))),+ col("r.column2").gt_eq(col("l.column2")),+ )+ .expect_err("reversed ASOF MATCH_CONDITION should fail");+ assert_snapshot!(reversed_match.strip_backtrace(), @r#"Error during planning: ASOF MATCH_CONDITION left operand must reference only the left input and right operand only the right input"#);++ Ok(())+ }+#[test]fn plan_builder_from_logical_plan() -> Result<()> {let plan =diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex 649c82a16a..ab1ee1cdf4 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -4462,6 +4462,25 @@ impl AsOfMatch {}}+impl TryFrom<Expr> for AsOfMatch {+ type Error = DataFusionError;++ fn try_from(condition: Expr) -> Result<Self> {+ let Expr::BinaryExpr(BinaryExpr { left, op, right }) = condition else {+ return plan_err!("ASOF MATCH_CONDITION must be a single comparison");+ };+ if !matches!(+ op,+ Operator::Lt | Operator::LtEq | Operator::Gt | Operator::GtEq+ ) {+ return plan_err!(+ "ASOF MATCH_CONDITION requires <, <=, >, or >=, found {op}"+ );+ }+ Ok(Self::new(*left, op, *right))+ }+}+impl Display for AsOfMatch {fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {write!(f, "{} {} {}", self.left, self.op, self.right)@@ -6179,10 +6198,10 @@ mod tests {assert_eq!(cross_join.min_rows(), 2);let asof_join = LogicalPlanBuilder::from(two_rows.clone())- .asof_join(+ .asof_join_on(one_row.clone(),- vec![],- AsOfMatch::new(col("l.column1"), Operator::GtEq, col("r.column1")),+ None,+ col("l.column1").gt_eq(col("r.column1")),)?.build()?;assert_eq!(asof_join.min_rows(), 2);diff --git a/datafusion/sql/src/relation/join.rs b/datafusion/sql/src/relation/join.rsindex 475d9a5b38..73c2f05504 100644--- a/datafusion/sql/src/relation/join.rs+++ b/datafusion/sql/src/relation/join.rs@@ -16,7 +16,7 @@// under the License.use crate::planner::{ContextProvider, PlannerContext, SqlToRel};-use datafusion_common::{Column, Result, not_impl_err, plan_datafusion_err};+use datafusion_common::{Column, Result, not_impl_err, plan_datafusion_err, plan_err};use datafusion_expr::{JoinType, LogicalPlan, LogicalPlanBuilder};use sqlparser::ast::{Join, JoinConstraint, JoinOperator, ObjectName, TableFactor, TableWithJoins,@@ -98,10 +98,75 @@ impl<S: ContextProvider> SqlToRel<'_, S> {JoinOperator::CrossJoin(JoinConstraint::None) => {self.parse_cross_join(left, right)}+ JoinOperator::AsOf {+ match_condition,+ constraint,+ } => self.parse_asof_join(+ left,+ right,+ match_condition,+ constraint,+ planner_context,+ ),other => not_impl_err!("Unsupported JOIN operator {other:?}"),}}+ fn parse_asof_join(+ &self,+ left: LogicalPlan,+ right: LogicalPlan,+ sql_match_condition: sqlparser::ast::Expr,+ constraint: JoinConstraint,+ planner_context: &mut PlannerContext,+ ) -> Result<LogicalPlan> {+ let join_schema = left.schema().join(right.schema())?;+ let match_condition =+ self.sql_to_expr(sql_match_condition, &join_schema, planner_context)?;++ match constraint {+ JoinConstraint::On(sql_on) => {+ let on = self.sql_to_expr(sql_on, &join_schema, planner_context)?;+ LogicalPlanBuilder::from(left)+ .asof_join_on(right, Some(on), match_condition)?+ .build()+ }+ JoinConstraint::Using(object_names) => {+ let keys = object_names+ .into_iter()+ .map(|object_name| {+ let ObjectName(mut object_names) = object_name;+ if object_names.len() != 1 {+ return not_impl_err!(+ "Invalid identifier in ASOF USING clause. Expected single identifier, got {}",+ ObjectName(object_names)+ );+ }+ let id = object_names.swap_remove(0);+ id.as_ident()+ .ok_or_else(|| {+ plan_datafusion_err!(+ "Expected identifier in ASOF USING clause"+ )+ })+ .map(|ident| {+ Column::from_name(+ self.ident_normalizer.normalize(ident.clone()),+ )+ })+ })+ .collect::<Result<Vec<_>>>()?;+ LogicalPlanBuilder::from(left)+ .asof_join_using(right, keys, match_condition)?+ .build()+ }+ JoinConstraint::None => LogicalPlanBuilder::from(left)+ .asof_join_on(right, None, match_condition)?+ .build(),+ JoinConstraint::Natural => plan_err!("NATURAL ASOF JOIN is not supported"),+ }+ }+fn parse_cross_join(&self,left: LogicalPlan,diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rsindex e69b49e95c..18af08fc18 100644--- a/datafusion/sql/src/unparser/plan.rs+++ b/datafusion/sql/src/unparser/plan.rs@@ -49,7 +49,7 @@ use datafusion_common::{};use datafusion_expr::expr::{OUTER_REFERENCE_COLUMN_PREFIX, UNNEST_COLUMN_PREFIX};use datafusion_expr::{- Aggregate, BinaryExpr, Distinct, Expr, FetchType, JoinConstraint, JoinType,+ Aggregate, AsOfJoin, BinaryExpr, Distinct, Expr, FetchType, JoinConstraint, JoinType,LogicalPlan, LogicalPlanBuilder, Operator, Projection, SkipType, Sort, SortExpr,TableScan, Unnest, UserDefinedLogicalNode, Window, expr::Alias,};@@ -184,6 +184,7 @@ impl Unparser<'_> {| LogicalPlan::Aggregate(_)| LogicalPlan::Sort(_)| LogicalPlan::Join(_)+ | LogicalPlan::AsOfJoin(_)| LogicalPlan::Repartition(_)| LogicalPlan::Union(_)| LogicalPlan::TableScan(_)@@ -204,7 +205,6 @@ impl Unparser<'_> {| LogicalPlan::Copy(_)| LogicalPlan::DescribeTable(_)| LogicalPlan::RecursiveQuery(_)- | LogicalPlan::AsOfJoin(_)| LogicalPlan::Unnest(_) => not_impl_err!("Unsupported plan: {plan:?}"),}}@@ -1479,11 +1479,8 @@ impl Unparser<'_> {let mut right_relation = RelationBuilder::default();if already_projected- && let Some(nested_relation) = self- .qualified_passthrough_join_projection_to_nested_relation(- right_plan.as_ref(),- query,- )?+ && let Some(nested_relation) =+ self.join_input_to_nested_relation(right_plan.as_ref(), query)?{right_relation = nested_relation;} else {@@ -1617,6 +1614,9 @@ impl Unparser<'_> {Ok(())}+ LogicalPlan::AsOfJoin(join) => {+ self.asof_join_to_sql(join, query, select, relation)+ }LogicalPlan::SubqueryAlias(plan_alias) => {let (plan, mut columns) =subquery_alias_inner_query_and_columns(plan_alias);@@ -1909,6 +1909,129 @@ impl Unparser<'_> {}}+ // Keep ASOF-specific locals out of the recursive plan unparser's stack frame.+ #[inline(never)]+ fn asof_join_to_sql(+ &self,+ join: &AsOfJoin,+ query: &mut Option<QueryBuilder>,+ select: &mut SelectBuilder,+ relation: &mut RelationBuilder,+ ) -> Result<()> {+ let already_projected = select.already_projected();+ let left_plan =+ Self::unwrap_qualified_passthrough_join_projection(Arc::clone(&join.left));+ let inline_left_join = matches!(+ left_plan.as_ref(),+ LogicalPlan::Join(_) | LogicalPlan::AsOfJoin(_)+ );+ let left_projection = if already_projected {+ None+ } else if inline_left_join {+ self.select_to_sql_recursively(left_plan.as_ref(), query, select, relation)?;+ select.pop_projections();+ Some(self.derived_input_projection(join.left.as_ref(), None)?)+ } else if Self::asof_input_requires_derived(join.left.as_ref()) {+ let qualifier = self.derive_asof_input(join.left.as_ref(), relation)?;+ Some(self.derived_input_projection(join.left.as_ref(), qualifier.as_ref())?)+ } else {+ self.select_to_sql_recursively(join.left.as_ref(), query, select, relation)?;+ Some(select.pop_projections())+ };+ if already_projected {+ if inline_left_join {+ self.select_to_sql_recursively(+ left_plan.as_ref(),+ query,+ select,+ relation,+ )?;+ } else if Self::asof_input_requires_derived(join.left.as_ref()) {+ self.derive_asof_input(join.left.as_ref(), relation)?;+ } else {+ self.select_to_sql_recursively(+ join.left.as_ref(),+ query,+ select,+ relation,+ )?;+ }+ }++ let mut right_relation = RelationBuilder::default();+ let nested_right =+ self.join_input_to_nested_relation(join.right.as_ref(), query)?;+ let right_projection = if already_projected {+ if let Some(nested_right) = nested_right {+ right_relation = nested_right;+ } else if Self::asof_input_requires_derived(join.right.as_ref()) {+ self.derive_asof_input(join.right.as_ref(), &mut right_relation)?;+ } else {+ self.select_to_sql_recursively(+ join.right.as_ref(),+ query,+ select,+ &mut right_relation,+ )?;+ }+ None+ } else if let Some(nested_right) = nested_right {+ right_relation = nested_right;+ Some(self.derived_input_projection(join.right.as_ref(), None)?)+ } else if Self::asof_input_requires_derived(join.right.as_ref()) {+ let qualifier =+ self.derive_asof_input(join.right.as_ref(), &mut right_relation)?;+ Some(self.derived_input_projection(join.right.as_ref(), qualifier.as_ref())?)+ } else {+ self.select_to_sql_recursively(+ join.right.as_ref(),+ query,+ select,+ &mut right_relation,+ )?;+ Some(select.pop_projections())+ };+ let Ok(Some(relation)) = right_relation.build() else {+ return internal_err!("Failed to build ASOF right relation");+ };+ let constraint =+ self.join_constraint_to_sql(join.join_constraint, &join.on, None)?;+ let match_condition = self.expr_to_sql(&Expr::BinaryExpr(BinaryExpr::new(+ Box::new(join.match_condition.left.clone()),+ join.match_condition.op,+ Box::new(join.match_condition.right.clone()),+ )))?;+ let ast_join = ast::Join {+ relation,+ global: false,+ join_operator: ast::JoinOperator::AsOf {+ match_condition,+ constraint,+ },+ };+ let mut from = select+ .pop_from()+ .ok_or_else(|| internal_datafusion_err!("ASOF left relation is missing"))?;+ from.push_join(ast_join);+ select.push_from(from);++ if !already_projected {+ let left_projection = left_projection.ok_or_else(|| {+ internal_datafusion_err!("ASOF left projection is missing")+ })?;+ let right_projection = right_projection.ok_or_else(|| {+ internal_datafusion_err!("ASOF right projection is missing")+ })?;+ select.projection(+ left_projection+ .into_iter()+ .chain(right_projection)+ .collect(),+ );+ }+ Ok(())+ }+/// Walk through transparent nodes (SubqueryAlias) to find the inner/// Projection that feeds an Unnest node.///@@ -2207,6 +2330,74 @@ impl Unparser<'_> {)}+ fn asof_input_requires_derived(plan: &LogicalPlan) -> bool {+ let simple_scan =+ |scan: &TableScan| scan.filters.is_empty() && scan.fetch.is_none();+ match plan {+ LogicalPlan::TableScan(scan) => !simple_scan(scan),+ LogicalPlan::SubqueryAlias(alias) => {+ !matches!(alias.input.as_ref(), LogicalPlan::TableScan(scan) if simple_scan(scan))+ }+ _ => true,+ }+ }++ fn derive_asof_input(+ &self,+ plan: &LogicalPlan,+ relation: &mut RelationBuilder,+ ) -> Result<Option<TableReference>> {+ if let LogicalPlan::SubqueryAlias(alias) = plan {+ let (inner, columns) = subquery_alias_inner_query_and_columns(alias);+ let table_alias = alias.alias.clone();+ if !columns.is_empty() && !self.dialect.supports_column_alias_in_table_alias()+ {+ let rewritten =+ inject_column_aliases_into_subquery(inner.clone(), columns)?;+ self.derive(+ &rewritten,+ relation,+ Some(self.new_table_alias(table_alias.table().to_string(), vec![])),+ false,+ )?;+ } else {+ self.derive(+ inner,+ relation,+ Some(self.new_table_alias(table_alias.table().to_string(), columns)),+ false,+ )?;+ }+ return Ok(Some(table_alias));+ }++ let qualifier = plan+ .schema()+ .iter()+ .find_map(|(qualifier, _)| qualifier.cloned());+ let alias = qualifier+ .as_ref()+ .map(|qualifier| self.new_table_alias(qualifier.table().to_string(), vec![]));+ self.derive(plan, relation, alias, false)?;+ Ok(qualifier)+ }++ fn derived_input_projection(+ &self,+ plan: &LogicalPlan,+ qualifier: Option<&TableReference>,+ ) -> Result<Vec<ast::SelectItem>> {+ plan.schema()+ .iter()+ .map(|(field_qualifier, field)| {+ self.select_item_to_sql(&Expr::Column(Column::new(+ qualifier.cloned().or_else(|| field_qualifier.cloned()),+ field.name(),+ )))+ })+ .collect()+ }+fn is_qualified_passthrough_projection(projection: &Projection) -> bool {projection.expr@@ -2218,7 +2409,10 @@ impl Unparser<'_> {plan: Arc<LogicalPlan>,) -> Arc<LogicalPlan> {if let LogicalPlan::Projection(projection) = plan.as_ref()- && matches!(projection.input.as_ref(), LogicalPlan::Join(_))+ && matches!(+ projection.input.as_ref(),+ LogicalPlan::Join(_) | LogicalPlan::AsOfJoin(_)+ )&& Self::is_qualified_passthrough_projection(projection){Arc::clone(&projection.input)@@ -2227,26 +2421,30 @@ impl Unparser<'_> {}}- fn qualified_passthrough_join_projection_to_nested_relation(+ fn join_input_to_nested_relation(&self,plan: &LogicalPlan,query: &mut Option<QueryBuilder>,) -> Result<Option<RelationBuilder>> {- let LogicalPlan::Projection(projection) = plan else {- return Ok(None);+ let join_plan = match plan {+ LogicalPlan::Join(_) | LogicalPlan::AsOfJoin(_) => plan,+ LogicalPlan::Projection(projection)+ if matches!(+ projection.input.as_ref(),+ LogicalPlan::Join(_) | LogicalPlan::AsOfJoin(_)+ ) && Self::is_qualified_passthrough_projection(projection) =>+ {+ projection.input.as_ref()+ }+ _ => return Ok(None),};- if !matches!(projection.input.as_ref(), LogicalPlan::Join(_))- || !Self::is_qualified_passthrough_projection(projection)- {- return Ok(None);- }let original_query = query.clone();let mut nested_select = SelectBuilder::default();nested_select.push_from(TableWithJoinsBuilder::default());let mut nested_relation = RelationBuilder::default();self.select_to_sql_recursively(- projection.input.as_ref(),+ join_plan,query,&mut nested_select,&mut nested_relation,@@ -2257,11 +2455,11 @@ impl Unparser<'_> {}let Some(mut nested_from) = nested_select.pop_from() else {- return internal_err!("Failed to build nested join relation");+ return internal_err!("Failed to build nested join input relation");};nested_from.relation(nested_relation);let Some(table_with_joins) = nested_from.build()? else {- return internal_err!("Failed to build nested join relation");+ return internal_err!("Failed to build nested join input relation");};let mut relation = RelationBuilder::default();diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rsindex 32dfade056..191425416c 100644--- a/datafusion/sql/tests/cases/plan_to_sql.rs+++ b/datafusion/sql/tests/cases/plan_to_sql.rs@@ -2823,6 +2823,37 @@ fn test_unparse_inner_join_with_table_scan_projection() -> Result<()> {Ok(())}+#[test]+fn test_unparse_asof_join() -> Result<()> {+ let trades_schema = Schema::new(vec![+ Field::new("symbol", DataType::Utf8, false),+ Field::new("ts", DataType::Int64, false),+ Field::new("trade_id", DataType::Int32, false),+ ]);+ let prices_schema = Schema::new(vec![+ Field::new("symbol", DataType::Utf8, false),+ Field::new("ts", DataType::Int64, false),+ Field::new("price", DataType::Int32, false),+ ]);+ let trades = table_scan(Some("trades"), &trades_schema, None)?+ .alias("t")?+ .build()?;+ let prices = table_scan(Some("prices"), &prices_schema, None)?+ .alias("p")?+ .build()?;+ let plan = LogicalPlanBuilder::from(trades)+ .asof_join_on(+ prices,+ Some(col("t.symbol").eq(col("p.symbol"))),+ col("t.ts").gt_eq(col("p.ts")),+ )?+ .project(vec![col("t.trade_id"), col("p.price")])?+ .build()?;++ assert_snapshot!(plan_to_sql(&plan)?, @r#"SELECT t.trade_id, p.price FROM trades AS t ASOF JOIN prices AS p MATCH_CONDITION ((t.ts >= p.ts)) ON t.symbol = p.symbol"#);+ Ok(())+}+/// Build the three base table scans (`left_table`, `mid_table`, `right_table`)/// shared by the nested passthrough-projection join unparsing tests.fn nested_passthrough_join_tables() -> Result<(LogicalPlan, LogicalPlan, LogicalPlan)> {@@ -2933,6 +2964,65 @@ fn test_unparse_projected_join_unwraps_left_nested_passthrough_projection() -> ROk(())}+#[test]+fn test_unparse_nested_asof_join_inputs() -> Result<()> {+ let (left, mid, right) = nested_passthrough_join_tables()?;+ let nested_left = LogicalPlanBuilder::from(left)+ .asof_join_on(+ mid,+ Some(col("left_table.mid_id").eq(col("mid_table.mid_id"))),+ col("left_table.left_id").gt_eq(col("mid_table.mid_id")),+ )?+ .project(vec![+ col("left_table.left_id"),+ col("mid_table.mid_id"),+ col("mid_table.right_id"),+ ])?+ .build()?;+ let plan = LogicalPlanBuilder::from(nested_left)+ .asof_join_on(+ right,+ Some(col("mid_table.right_id").eq(col("right_table.right_id"))),+ col("mid_table.right_id").gt_eq(col("right_table.right_id")),+ )?+ .project(vec![+ col("left_table.left_id"),+ col("mid_table.mid_id"),+ col("right_table.value"),+ ])?+ .build()?;+ assert_snapshot!(plan_to_sql(&plan)?, @r#"SELECT left_table.left_id, mid_table.mid_id, right_table."value" FROM left_table ASOF JOIN mid_table MATCH_CONDITION ((left_table.left_id >= mid_table.mid_id)) ON left_table.mid_id = mid_table.mid_id ASOF JOIN right_table MATCH_CONDITION ((mid_table.right_id >= right_table.right_id)) ON mid_table.right_id = right_table.right_id"#);++ let (left, mid, right) = nested_passthrough_join_tables()?;+ let nested_right = LogicalPlanBuilder::from(mid)+ .asof_join_on(+ right,+ Some(col("mid_table.right_id").eq(col("right_table.right_id"))),+ col("mid_table.right_id").gt_eq(col("right_table.right_id")),+ )?+ .project(vec![+ col("mid_table.mid_id"),+ col("mid_table.right_id"),+ col("right_table.value"),+ ])?+ .build()?;+ let plan = LogicalPlanBuilder::from(left)+ .asof_join_on(+ nested_right,+ Some(col("left_table.mid_id").eq(col("mid_table.mid_id"))),+ col("left_table.left_id").gt_eq(col("mid_table.mid_id")),+ )?+ .project(vec![+ col("left_table.left_id"),+ col("mid_table.mid_id"),+ col("right_table.value"),+ ])?+ .build()?;+ assert_snapshot!(plan_to_sql(&plan)?, @r#"SELECT left_table.left_id, mid_table.mid_id, right_table."value" FROM left_table ASOF JOIN (mid_table ASOF JOIN right_table MATCH_CONDITION ((mid_table.right_id >= right_table.right_id)) ON mid_table.right_id = right_table.right_id) MATCH_CONDITION ((left_table.left_id >= mid_table.mid_id)) ON left_table.mid_id = mid_table.mid_id"#);++ Ok(())+}+#[test]fn test_unparse_left_semi_join_with_table_scan_projection() -> Result<()> {let schema = Schema::new(vec![diff --git a/datafusion/sqllogictest/test_files/asof_join.slt b/datafusion/sqllogictest/test_files/asof_join.sltnew file mode 100644index 0000000000..3e84cd0376--- /dev/null+++ b/datafusion/sqllogictest/test_files/asof_join.slt@@ -0,0 +1,287 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++statement ok+CREATE TABLE asof_left(id INT, grp TEXT, ts TIMESTAMP) AS VALUES+ (1, 'A', TIMESTAMP '2024-01-01 09:00:01'),+ (2, 'A', TIMESTAMP '2024-01-01 09:00:04'),+ (3, 'A', TIMESTAMP '2024-01-01 09:00:07'),+ (4, 'B', TIMESTAMP '2024-01-01 09:00:02'),+ (5, 'B', TIMESTAMP '2024-01-01 09:00:08'),+ (6, NULL, TIMESTAMP '2024-01-01 09:00:03'),+ (7, 'A', NULL);++statement ok+CREATE TABLE asof_right(grp TEXT, ts TIMESTAMP, val TEXT) AS VALUES+ ('A', TIMESTAMP '2024-01-01 09:00:02', 'a2'),+ ('A', TIMESTAMP '2024-01-01 09:00:04', 'a4'),+ ('A', TIMESTAMP '2024-01-01 09:00:06', 'a6'),+ ('B', TIMESTAMP '2024-01-01 09:00:01', 'b1'),+ ('B', TIMESTAMP '2024-01-01 09:00:06', 'b6'),+ (NULL, TIMESTAMP '2024-01-01 09:00:02', 'null-group'),+ ('A', NULL, 'null-ts');++# Inclusive predecessor per equality group. This also verifies unmatched left+# rows and NULL behavior for equality keys and ordered expressions.+query IPPT+SELECT l.id, l.ts, r.ts, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp+ORDER BY l.id;+----+1 2024-01-01T09:00:01 NULL NULL+2 2024-01-01T09:00:04 2024-01-01T09:00:04 a4+3 2024-01-01T09:00:07 2024-01-01T09:00:06 a6+4 2024-01-01T09:00:02 2024-01-01T09:00:01 b1+5 2024-01-01T09:00:08 2024-01-01T09:00:06 b6+6 2024-01-01T09:00:03 NULL NULL+7 NULL NULL NULL++# Strict predecessor per equality group.+query IPPT+SELECT l.id, l.ts, r.ts, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts > r.ts)+ON l.grp = r.grp+ORDER BY l.id;+----+1 2024-01-01T09:00:01 NULL NULL+2 2024-01-01T09:00:04 2024-01-01T09:00:02 a2+3 2024-01-01T09:00:07 2024-01-01T09:00:06 a6+4 2024-01-01T09:00:02 2024-01-01T09:00:01 b1+5 2024-01-01T09:00:08 2024-01-01T09:00:06 b6+6 2024-01-01T09:00:03 NULL NULL+7 NULL NULL NULL++# Inclusive successor per equality group.+query IPPT+SELECT l.id, l.ts, r.ts, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts <= r.ts)+ON l.grp = r.grp+ORDER BY l.id;+----+1 2024-01-01T09:00:01 2024-01-01T09:00:02 a2+2 2024-01-01T09:00:04 2024-01-01T09:00:04 a4+3 2024-01-01T09:00:07 NULL NULL+4 2024-01-01T09:00:02 2024-01-01T09:00:06 b6+5 2024-01-01T09:00:08 NULL NULL+6 2024-01-01T09:00:03 NULL NULL+7 NULL NULL NULL++# Strict successor per equality group.+query IPPT+SELECT l.id, l.ts, r.ts, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts < r.ts)+ON l.grp = r.grp+ORDER BY l.id;+----+1 2024-01-01T09:00:01 2024-01-01T09:00:02 a2+2 2024-01-01T09:00:04 2024-01-01T09:00:06 a6+3 2024-01-01T09:00:07 NULL NULL+4 2024-01-01T09:00:02 2024-01-01T09:00:06 b6+5 2024-01-01T09:00:08 NULL NULL+6 2024-01-01T09:00:03 NULL NULL+7 NULL NULL NULL++# USING exposes one unqualified equality key.+query TIPT+SELECT grp, l.id, r.ts, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+USING (grp)+ORDER BY l.id;+----+A 1 NULL NULL+A 2 2024-01-01T09:00:04 a4+A 3 2024-01-01T09:00:06 a6+B 4 2024-01-01T09:00:01 b1+B 5 2024-01-01T09:00:06 b6+NULL 6 NULL NULL+A 7 NULL NULL++# Both qualified equality keys remain addressable.+query ITT+SELECT l.id, l.grp, r.grp+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+USING (grp)+ORDER BY l.id;+----+1 A NULL+2 A A+3 A A+4 B B+5 B B+6 NULL NULL+7 A NULL++# Equality keys are optional.+query IT+SELECT l.id, r.label+FROM (VALUES (1, 1), (2, 5), (3, CAST(NULL AS INT))) AS l(id, ts)+ASOF JOIN (VALUES (2, 'r2'), (4, 'r4')) AS r(ts, label)+MATCH_CONDITION (l.ts >= r.ts)+ORDER BY l.id;+----+1 NULL+2 r4+3 NULL++# Multiple equality keys form one candidate group.+query IT+SELECT l.id, r.val+FROM (VALUES+ (1, 'X', 'A', TIMESTAMP '2024-01-01 09:00:04'),+ (2, 'Y', 'A', TIMESTAMP '2024-01-01 09:00:04')+) AS l(id, venue, grp, ts)+ASOF JOIN (VALUES+ ('X', 'A', TIMESTAMP '2024-01-01 09:00:02', 'x-a2'),+ ('Y', 'A', TIMESTAMP '2024-01-01 09:00:03', 'y-a3'),+ ('X', 'B', TIMESTAMP '2024-01-01 09:00:04', 'x-b4')+) AS r(venue, grp, ts, val)+MATCH_CONDITION (l.ts >= r.ts)+ON l.venue = r.venue AND l.grp = r.grp+ORDER BY l.id;+----+1 x-a2+2 y-a3++# Candidate selection sees the right input after subquery filtering.+query IT+SELECT l.id, r.val+FROM asof_left l+ASOF JOIN (SELECT * FROM asof_right WHERE val <> 'a6') r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp+WHERE l.id IN (3, 5)+ORDER BY l.id;+----+3 a4+5 b6++# Equality and match operands use the planner's common coercion types.+query II+SELECT l.id, r.payload+FROM (VALUES (1, CAST(5 AS SMALLINT), CAST(10 AS INT))) AS l(id, grp, ts)+ASOF JOIN (+ VALUES (CAST(5 AS BIGINT), CAST(9 AS BIGINT), 90)+) AS r(grp, ts, payload)+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp;+----+1 90++# Coercion is applied before the ASOF ordering requirements are planned.+query TT+EXPLAIN SELECT l.id, r.payload+FROM (VALUES (1, CAST(5 AS SMALLINT), CAST(10 AS INT))) AS l(id, grp, ts)+ASOF JOIN (+ VALUES (CAST(5 AS BIGINT), CAST(9 AS BIGINT), 90)+) AS r(grp, ts, payload)+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp;+----+logical_plan+01)Projection: l.id, r.payload+02)--AsOf Join: match=[CAST(l.ts AS Int64) >= r.ts], constraint=On, on=[CAST(l.grp AS Int64) = r.grp]+03)----SubqueryAlias: l+04)------Projection: column1 AS id, column2 AS grp, column3 AS ts+05)--------Values: (Int64(1), Int16(5) AS Int64(5), Int32(10) AS Int64(10))+06)----SubqueryAlias: r+07)------Projection: column1 AS grp, column2 AS ts, column3 AS payload+08)--------Values: (Int64(5), Int64(9), Int64(90))+physical_plan+01)ProjectionExec: expr=[id@0 as id, payload@5 as payload]+02)--AsOfJoinExec: on=[(CAST(grp AS Int64) = grp)], match=[CAST(ts AS Int64) >= ts]+03)----SortExec: expr=[CAST(grp@1 AS Int64) ASC, CAST(ts@2 AS Int64) ASC], preserve_partitioning=[false]+04)------ProjectionExec: expr=[column1@0 as id, column2@1 as grp, column3@2 as ts]+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)----ProjectionExec: expr=[column1@0 as grp, column2@1 as ts, column3@2 as payload]+07)------SortExec: expr=[column1@0 ASC, column2@1 ASC], preserve_partitioning=[false]+08)--------DataSourceExec: partitions=1, partition_sizes=[1]++# Equality operands can name the right input first.+query IT+SELECT l.id, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON r.grp = l.grp+WHERE l.id IN (2, 4)+ORDER BY l.id;+----+2 a4+4 b1++query TT+EXPLAIN SELECT l.id, r.val+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp = r.grp;+----+logical_plan+01)Projection: l.id, r.val+02)--AsOf Join: match=[l.ts >= r.ts], constraint=On, on=[l.grp = r.grp]+03)----SubqueryAlias: l+04)------TableScan: asof_left projection=[id, grp, ts]+05)----SubqueryAlias: r+06)------TableScan: asof_right projection=[grp, ts, val]+physical_plan+01)ProjectionExec: expr=[id@0 as id, val@5 as val]+02)--AsOfJoinExec: on=[(grp = grp)], match=[ts >= ts]+03)----SortExec: expr=[grp@1 ASC, ts@2 ASC], preserve_partitioning=[false]+04)------DataSourceExec: partitions=1, partition_sizes=[1]+05)----SortExec: expr=[grp@0 ASC, ts@1 ASC], preserve_partitioning=[false]+06)------DataSourceExec: partitions=1, partition_sizes=[1]++query error ASOF MATCH_CONDITION requires <, <=, >, or >=+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts = r.ts)+ON l.grp = r.grp;++query error ASOF MATCH_CONDITION left operand must reference only the left input+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (r.ts >= l.ts)+ON l.grp = r.grp;++query error ASOF ON accepts only equality conditions combined with AND+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts >= r.ts)+ON l.grp > r.grp;++query error ASOF MATCH_CONDITION must be a single comparison+SELECT *+FROM asof_left l+ASOF JOIN asof_right r+MATCH_CONDITION (l.ts)+ON l.grp = r.grp;diff --git a/datafusion/substrait/tests/cases/serialize.rs b/datafusion/substrait/tests/cases/serialize.rsindex 75e1fe251a..4a8413718e 100644--- a/datafusion/substrait/tests/cases/serialize.rs+++ b/datafusion/substrait/tests/cases/serialize.rs@@ -18,7 +18,7 @@#[cfg(test)]mod tests {use datafusion::datasource::provider_as_source;- use datafusion::logical_expr::{AsOfMatch, LogicalPlanBuilder, Operator};+ use datafusion::logical_expr::LogicalPlanBuilder;use datafusion_substrait::logical_plan::consumer::from_substrait_plan;use datafusion_substrait::logical_plan::producer::to_substrait_plan;use datafusion_substrait::serializer;@@ -110,10 +110,10 @@ mod tests {let left = LogicalPlanBuilder::scan("l", Arc::clone(&table), None)?.build()?;let right = LogicalPlanBuilder::scan("r", table, None)?.build()?;let plan = LogicalPlanBuilder::from(left)- .asof_join(+ .asof_join_on(right,- vec![(col("l.b"), col("r.b"))],- AsOfMatch::new(col("l.a"), Operator::GtEq, col("r.a")),+ Some(col("l.b").eq(col("r.b"))),+ col("l.a").gt_eq(col("r.a")),)?.build()?;let error = to_substrait_plan(&plan, &ctx.state())diff --git a/docs/source/user-guide/sql/select.md b/docs/source/user-guide/sql/select.mdindex af442de659..2cbe26fee3 100644--- a/docs/source/user-guide/sql/select.md+++ b/docs/source/user-guide/sql/select.md@@ -319,6 +319,7 @@ SELECT a FROM table_name WHERE a > 10;```textfrom_item [join_type] JOIN from_item [join_condition]+from_item ASOF JOIN from_item MATCH_CONDITION (condition) [join_condition]from_item CROSS JOIN from_itemfrom_item NATURAL JOIN from_itemfrom_item [join_type] JOIN LATERAL (query) AS alias [join_condition]@@ -400,6 +401,48 @@ SELECT * FROM x LEFT JOIN x AS y ON x.column_1 = y.column_2;+----------+----------+----------+----------+```+### ASOF JOIN++DataFusion follows the+[Snowflake `ASOF JOIN` syntax](https://docs.snowflake.com/en/sql-reference/constructs/asof-join).+An `ASOF JOIN` matches each left row with at most one right row according to an+ordered comparison. It preserves every left row and fills the right columns+with `NULL` when no right row matches.++```sql+SELECT t.*, p.price+FROM trades AS t+ASOF JOIN prices AS p+MATCH_CONDITION (t.ts >= p.ts)+ON t.symbol = p.symbol;+```++`MATCH_CONDITION` must compare an expression from the left input with an+expression from the right input using one of the following operators. Operand+order is significant: the left input expression must appear on the left.++| Condition | Selected right row |+| --------- | ----------------------------------------- |+| `l >= r` | Greatest `r` less than or equal to `l` |+| `l > r` | Greatest `r` strictly less than `l` |+| `l <= r` | Smallest `r` greater than or equal to `l` |+| `l < r` | Smallest `r` strictly greater than `l` |++An optional `ON` clause containing equality conditions combined with `AND`, or+a `USING` clause, divides rows into equality groups before the ordered match.+An unqualified `USING` key appears once in wildcard output, while both qualified+input keys remain addressable.++Without equality keys, all rows belong to one group. The initial execution+strategy collects one ordered right partition and shares it across every left+partition, so output partitioning follows the left input. The complete right+input must fit in memory and may be scanned once per left partition; spilling+and repartitioned ASOF execution are not yet supported.++A `NULL` in either ordered expression or in any equality key does not match.+Both inputs must be bounded. If multiple right rows have the same equality keys+and ordered value, which tied row is selected is nondeterministic.+### RIGHT OUTER JOINThe keywords `RIGHT JOIN` or `RIGHT OUTER JOIN` define a join that includes all rows from the right table even if there