Apache DataFusion SQL Query Engine
feat: add ASOF join logical semantics (#23829)
Xuanwoe1942b1446e4
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 9dde89fb1c..882956a63c 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -41,7 +41,8 @@ use crate::physical_plan::explain::ExplainExec;use crate::physical_plan::filter::FilterExecBuilder;use crate::physical_plan::joins::utils as join_utils;use crate::physical_plan::joins::{- CrossJoinExec, HashJoinExec, NestedLoopJoinExec, PartitionMode, SortMergeJoinExec,+ AsOfJoinExec, AsOfMatchExpr, CrossJoinExec, HashJoinExec, NestedLoopJoinExec,+ PartitionMode, SortMergeJoinExec,};use crate::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use crate::physical_plan::projection::{ProjectionExec, ProjectionExpr};@@ -1787,6 +1788,51 @@ impl DefaultPhysicalPlanner {join}}+ LogicalPlan::AsOfJoin(join) => {+ let [physical_left, physical_right] = children.two()?;+ let join_on = join+ .on+ .iter()+ .map(|(left, right)| {+ Ok((+ create_physical_expr(+ left,+ join.left.schema(),+ execution_props,+ planning_ctx,+ )?,+ create_physical_expr(+ right,+ join.right.schema(),+ execution_props,+ planning_ctx,+ )?,+ ))+ })+ .collect::<Result<join_utils::JoinOn>>()?;+ let match_condition = AsOfMatchExpr::new(+ create_physical_expr(+ &join.match_condition.left,+ join.left.schema(),+ execution_props,+ planning_ctx,+ )?,+ join.match_condition.op,+ create_physical_expr(+ &join.match_condition.right,+ join.right.schema(),+ execution_props,+ planning_ctx,+ )?,+ );+ Arc::new(AsOfJoinExec::try_new(+ physical_left,+ physical_right,+ join_on,+ match_condition,+ None,+ )?)+ }LogicalPlan::RecursiveQuery(RecursiveQuery {name,is_distinct,@@ -2288,6 +2334,7 @@ fn extract_dml_filters(| LogicalPlan::Sort(_)| LogicalPlan::Union(_)| LogicalPlan::Join(_)+ | LogicalPlan::AsOfJoin(_)| LogicalPlan::Repartition(_)| LogicalPlan::Aggregate(_)| LogicalPlan::Window(_)diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex ceb2c124e2..a6a254ea25 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -31,10 +31,10 @@ use crate::expr_rewriter::{rewrite_sort_cols_by_aggs,};use crate::logical_plan::{- Aggregate, Analyze, Distinct, DistinctOn, EmptyRelation, Explain, Filter, Join,- JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType, Prepare,- Projection, Repartition, Sort, SubqueryAlias, TableScanBuilder, Union, Unnest,- Values, Window,+ Aggregate, Analyze, AsOfJoin, AsOfMatch, Distinct, DistinctOn, EmptyRelation,+ Explain, Filter, Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning,+ PlanType, Prepare, Projection, Repartition, Sort, SubqueryAlias, TableScanBuilder,+ Union, Unnest, Values, Window,};use crate::select_expr::SelectExpr;use crate::utils::{@@ -1007,6 +1007,68 @@ impl LogicalPlanBuilder {)}+ /// Apply a left-preserving ASOF join using equality expressions and one+ /// ordered match condition.+ pub fn asof_join(+ self,+ right: LogicalPlan,+ on: Vec<(Expr, Expr)>,+ match_condition: AsOfMatch,+ ) -> Result<Self> {+ self.asof_join_with_constraint(right, on, match_condition, JoinConstraint::On)+ }++ /// Apply a left-preserving ASOF join using `USING` equality keys.+ pub fn asof_join_using(+ self,+ right: LogicalPlan,+ using_keys: Vec<Column>,+ match_condition: AsOfMatch,+ ) -> Result<Self> {+ let on = using_keys+ .into_iter()+ .map(|key| {+ let left = Self::normalize(&self.plan, key.clone())?;+ let right = Self::normalize(&right, key)?;+ Ok((Expr::Column(left), Expr::Column(right)))+ })+ .collect::<Result<_>>()?;+ self.asof_join_with_constraint(right, on, match_condition, JoinConstraint::Using)+ }++ fn asof_join_with_constraint(+ self,+ right: LogicalPlan,+ on: Vec<(Expr, Expr)>,+ match_condition: AsOfMatch,+ join_constraint: JoinConstraint,+ ) -> Result<Self> {+ let normalize = |expr, schema: &DFSchema| {+ normalize_col_with_schemas_and_ambiguity_check(expr, &[&[schema]], &[])+ };+ let on = on+ .into_iter()+ .map(|(left, right_expr)| {+ Ok((+ normalize(left, self.plan.schema())?,+ normalize(right_expr, right.schema())?,+ ))+ })+ .collect::<Result<_>>()?;+ let match_condition = AsOfMatch {+ left: normalize(match_condition.left, self.plan.schema())?,+ op: match_condition.op,+ right: normalize(match_condition.right, right.schema())?,+ };+ Ok(Self::new(LogicalPlan::AsOfJoin(AsOfJoin::try_new(+ self.plan,+ Arc::new(right),+ on,+ match_condition,+ join_constraint,+ )?)))+ }+pub(crate) fn normalize(plan: &LogicalPlan, column: Column) -> Result<Column> {if column.relation.is_some() {// column is already normalized@@ -1778,6 +1840,14 @@ pub fn build_join_schema(dfschema.with_functional_dependencies(func_dependencies)}+/// Creates the schema for a left-preserving ASOF join.+///+/// Both `ON` and `USING` preserve all qualified input fields. SQL wildcard+/// expansion handles the unqualified `USING` key as a single column.+pub fn build_asof_join_schema(left: &DFSchema, right: &DFSchema) -> Result<DFSchema> {+ build_join_schema(left, right, &JoinType::Left)+}+/// (Re)qualify the sides of a join if needed, i.e. if the columns from one side would otherwise/// conflict with the columns from the other./// This is especially useful for queries that come as Substrait, since Substrait doesn't currently allow specifyingdiff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rsindex 4afbb56702..3009253f53 100644--- a/datafusion/expr/src/logical_plan/display.rs+++ b/datafusion/expr/src/logical_plan/display.rs@@ -21,10 +21,10 @@ use std::collections::HashMap;use std::fmt;use crate::{- Aggregate, DescribeTable, Distinct, DistinctOn, DmlStatement, Expr, Filter, Join,- Limit, LogicalPlan, Partitioning, Projection, RecursiveQuery, Repartition, Sort,- Subquery, SubqueryAlias, TableProviderFilterPushDown, TableScan, Unnest, Values,- Window, expr_vec_fmt,+ Aggregate, AsOfJoin, DescribeTable, Distinct, DistinctOn, DmlStatement, Expr, Filter,+ Join, Limit, LogicalPlan, Partitioning, Projection, RecursiveQuery, Repartition,+ Sort, Subquery, SubqueryAlias, TableProviderFilterPushDown, TableScan, Unnest,+ Values, Window, expr_vec_fmt,};use crate::dml::CopyTo;@@ -493,6 +493,21 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {"Filter": format!("{}", filter_expr)})}+ LogicalPlan::AsOfJoin(AsOfJoin {+ on,+ match_condition,+ join_constraint,+ ..+ }) => {+ let join_expr: Vec<String> =+ on.iter().map(|(l, r)| format!("{l} = {r}")).collect();+ json!({+ "Node Type": "AsOf Join",+ "Join Constraint": format!("{join_constraint:?}"),+ "Join Keys": join_expr.join(", "),+ "Match Condition": match_condition.to_string(),+ })+ }LogicalPlan::Repartition(Repartition {partitioning_scheme,..diff --git a/datafusion/expr/src/logical_plan/mod.rs b/datafusion/expr/src/logical_plan/mod.rsindex 4766c3f333..98113d12c1 100644--- a/datafusion/expr/src/logical_plan/mod.rs+++ b/datafusion/expr/src/logical_plan/mod.rs@@ -28,8 +28,8 @@ pub mod tree_node;pub use builder::{LogicalPlanBuilder, LogicalPlanBuilderOptions, LogicalTableSource, UNNAMED_TABLE,- build_join_schema, requalify_sides_if_needed, table_scan, union,- wrap_projection_for_join_if_necessary,+ build_asof_join_schema, build_join_schema, requalify_sides_if_needed, table_scan,+ union, wrap_projection_for_join_if_necessary,};pub use ddl::{CreateCatalog, CreateCatalogSchema, CreateExternalTable, CreateFunction,@@ -41,12 +41,12 @@ pub use dml::{WriteOp,};pub use plan::{- Aggregate, Analyze, ColumnUnnestList, DescribeTable, Distinct, DistinctOn,- EmptyRelation, Explain, ExplainOption, Extension, FetchType, Filter, Join,- JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType, Projection,- RangePartitioning, RecursiveQuery, Repartition, SkipType, Sort, StringifiedPlan,- Subquery, SubqueryAlias, TableScan, TableScanBuilder, ToStringifiedPlan, Union,- Unnest, Values, Window, projection_schema,+ Aggregate, Analyze, AsOfJoin, AsOfMatch, ColumnUnnestList, DescribeTable, Distinct,+ DistinctOn, EmptyRelation, Explain, ExplainOption, Extension, FetchType, Filter,+ Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType,+ Projection, RangePartitioning, RecursiveQuery, Repartition, SkipType, Sort,+ StringifiedPlan, Subquery, SubqueryAlias, TableScan, TableScanBuilder,+ ToStringifiedPlan, Union, Unnest, Values, Window, projection_schema,};pub use statement::{Deallocate, Execute, Prepare, ResetVariable, SetVariable, Statement,diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex 4e039e3e77..74b5c42353 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -41,14 +41,15 @@ use crate::logical_plan::display::{GraphvizVisitor, IndentVisitor};use crate::logical_plan::extension::UserDefinedLogicalNode;use crate::logical_plan::{DmlStatement, Statement, WriteOp};use crate::utils::{- check_aggregate_and_window_nesting, enumerate_grouping_sets, exprlist_to_fields,- find_out_reference_exprs, grouping_set_expr_count, grouping_set_to_exprlist,- merge_schema, split_conjunction,+ check_aggregate_and_window_nesting, enumerate_grouping_sets, expr_to_columns,+ exprlist_to_fields, find_out_reference_exprs, grouping_set_expr_count,+ grouping_set_to_exprlist, merge_schema, split_conjunction,};use crate::{BinaryExpr, CreateMemoryTable, CreateView, Execute, Expr, ExprSchemable, GroupingSet,LogicalPlanBuilder, Operator, Prepare, TableProviderFilterPushDown, TableSource,- WindowFunctionDefinition, build_join_schema, expr_vec_fmt, requalify_sides_if_needed,+ WindowFunctionDefinition, build_asof_join_schema, build_join_schema, expr_vec_fmt,+ requalify_sides_if_needed,};use crate::statistics::StatisticsRequest;@@ -295,6 +296,9 @@ pub enum LogicalPlan {Unnest(Unnest),/// A variadic query (e.g. "Recursive CTEs")RecursiveQuery(RecursiveQuery),+ /// Match each left row with at most one ordered row from the right input.+ /// This is used to implement SQL `ASOF JOIN`.+ AsOfJoin(AsOfJoin),}impl Default for LogicalPlan {@@ -342,6 +346,7 @@ impl LogicalPlan {LogicalPlan::Aggregate(Aggregate { schema, .. }) => schema,LogicalPlan::Sort(Sort { input, .. }) => input.schema(),LogicalPlan::Join(Join { schema, .. }) => schema,+ LogicalPlan::AsOfJoin(AsOfJoin { schema, .. }) => schema,LogicalPlan::Repartition(Repartition { input, .. }) => input.schema(),LogicalPlan::Limit(Limit { input, .. }) => input.schema(),LogicalPlan::Statement(statement) => statement.schema(),@@ -370,7 +375,8 @@ impl LogicalPlan {| LogicalPlan::Projection(_)| LogicalPlan::Aggregate(_)| LogicalPlan::Unnest(_)- | LogicalPlan::Join(_) => self+ | LogicalPlan::Join(_)+ | LogicalPlan::AsOfJoin(_) => self.inputs().iter().map(|input| input.schema().as_ref())@@ -460,6 +466,9 @@ impl LogicalPlan {LogicalPlan::Aggregate(Aggregate { input, .. }) => vec![input],LogicalPlan::Sort(Sort { input, .. }) => vec![input],LogicalPlan::Join(Join { left, right, .. }) => vec![left, right],+ LogicalPlan::AsOfJoin(AsOfJoin { left, right, .. }) => {+ vec![left, right]+ }LogicalPlan::Limit(Limit { input, .. }) => vec![input],LogicalPlan::Subquery(Subquery { subquery, .. }) => vec![subquery],LogicalPlan::SubqueryAlias(SubqueryAlias { input, .. }) => vec![input],@@ -495,12 +504,20 @@ impl LogicalPlan {let mut using_columns: Vec<HashSet<Column>> = vec![];self.apply_with_subqueries(|plan| {- if let LogicalPlan::Join(Join {- join_constraint: JoinConstraint::Using,- on,- ..- }) = plan- {+ let on = match plan {+ LogicalPlan::Join(Join {+ join_constraint: JoinConstraint::Using,+ on,+ ..+ })+ | LogicalPlan::AsOfJoin(AsOfJoin {+ join_constraint: JoinConstraint::Using,+ on,+ ..+ }) => Some(on),+ _ => None,+ };+ if let Some(on) = on {// The join keys in using-join must be columns.let columns =on.iter().try_fold(HashSet::new(), |mut accumu, (l, r)| {@@ -568,6 +585,7 @@ impl LogicalPlan {right.head_output_expr()}},+ LogicalPlan::AsOfJoin(AsOfJoin { left, .. }) => left.head_output_expr(),LogicalPlan::RecursiveQuery(RecursiveQuery { static_term, .. }) => {static_term.head_output_expr()}@@ -691,6 +709,26 @@ impl LogicalPlan {null_aware,}))}+ LogicalPlan::AsOfJoin(AsOfJoin {+ left,+ right,+ on,+ match_condition,+ join_constraint,+ schema: _,+ }) => Ok(LogicalPlan::AsOfJoin(AsOfJoin::try_new(+ left,+ right,+ on.into_iter()+ .map(|(left, right)| (left.unalias(), right.unalias()))+ .collect(),+ AsOfMatch {+ left: match_condition.left.unalias(),+ op: match_condition.op,+ right: match_condition.right.unalias(),+ },+ join_constraint,+ )?)),LogicalPlan::Subquery(_) => Ok(self),LogicalPlan::SubqueryAlias(SubqueryAlias {input,@@ -994,6 +1032,45 @@ impl LogicalPlan {null_aware: *null_aware,}))}+ LogicalPlan::AsOfJoin(AsOfJoin {+ on,+ match_condition,+ join_constraint,+ ..+ }) => {+ let (left, right) = self.only_two_inputs(inputs)?;+ let expected = on.len() * 2 + 2;+ assert_eq_or_internal_err!(+ expected,+ expr.len(),+ "Invalid number of new ASOF join expressions: expected {}, got {}",+ expected,+ expr.len()+ );++ let mut iter = expr.into_iter();+ let mut new_on = Vec::with_capacity(on.len());+ for _ in 0..on.len() {+ let left = iter.next().expect("expression count checked").unalias();+ let right = iter.next().expect("expression count checked").unalias();+ new_on.push((left, right));+ }+ let match_left = iter.next().expect("expression count checked").unalias();+ let match_right =+ iter.next().expect("expression count checked").unalias();++ Ok(LogicalPlan::AsOfJoin(AsOfJoin::try_new(+ Arc::new(left),+ Arc::new(right),+ new_on,+ AsOfMatch {+ left: match_left,+ op: match_condition.op,+ right: match_right,+ },+ *join_constraint,+ )?))+ }LogicalPlan::Subquery(Subquery {outer_ref_columns,spans,@@ -1419,6 +1496,7 @@ impl LogicalPlan {right.max_rows()}},+ LogicalPlan::AsOfJoin(AsOfJoin { left, .. }) => left.max_rows(),LogicalPlan::Repartition(Repartition { input, .. }) => input.max_rows(),LogicalPlan::Union(Union { inputs, .. }) => {inputs.iter().try_fold(0usize, |mut acc, plan| {@@ -1494,6 +1572,7 @@ impl LogicalPlan {| JoinType::LeftAnti| JoinType::RightAnti => 0,},+ LogicalPlan::AsOfJoin(AsOfJoin { left, .. }) => left.min_rows(),LogicalPlan::Union(Union { inputs, .. }) => inputs.iter().fold(0, |rows, input| rows.saturating_add(input.min_rows())),@@ -1547,6 +1626,7 @@ impl LogicalPlan {LogicalPlan::Window(_) => Ok(None),LogicalPlan::Aggregate(_) => Ok(None),LogicalPlan::Join(_) => Ok(None),+ LogicalPlan::AsOfJoin(_) => Ok(None),LogicalPlan::Repartition(_) => Ok(None),LogicalPlan::Union(_) => Ok(None),LogicalPlan::EmptyRelation(_) => Ok(None),@@ -1585,6 +1665,7 @@ impl LogicalPlan {LogicalPlan::Window(_) => Ok(None),LogicalPlan::Aggregate(_) => Ok(None),LogicalPlan::Join(_) => Ok(None),+ LogicalPlan::AsOfJoin(_) => Ok(None),LogicalPlan::Repartition(_) => Ok(None),LogicalPlan::Union(_) => Ok(None),LogicalPlan::EmptyRelation(_) => Ok(None),@@ -2213,6 +2294,25 @@ impl LogicalPlan {}}}+ LogicalPlan::AsOfJoin(AsOfJoin {+ on,+ match_condition,+ join_constraint,+ ..+ }) => {+ let equality = on+ .iter()+ .map(|(left, right)| format!("{left} = {right}"))+ .join(", ");+ write!(+ f,+ "AsOf Join: match=[{match_condition}], constraint={join_constraint:?}"+ )?;+ if !equality.is_empty() {+ write!(f, ", on=[{equality}]")?;+ }+ Ok(())+ }LogicalPlan::Repartition(Repartition {partitioning_scheme,..@@ -4343,6 +4443,169 @@ pub struct Join {pub null_aware: bool,}+/// The ordered comparison used by an [`AsOfJoin`].+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash)]+pub struct AsOfMatch {+ /// Expression evaluated against the left input.+ pub left: Expr,+ /// One of [`Operator::Lt`], [`Operator::LtEq`], [`Operator::Gt`], or+ /// [`Operator::GtEq`].+ pub op: Operator,+ /// Expression evaluated against the right input.+ pub right: Expr,+}++impl AsOfMatch {+ /// Creates an ordered ASOF match condition.+ pub fn new(left: Expr, op: Operator, right: Expr) -> Self {+ Self { left, op, right }+ }+}++impl Display for AsOfMatch {+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {+ write!(f, "{} {} {}", self.left, self.op, self.right)+ }+}++/// Match each left row with at most one ordered row from the right input.+#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub struct AsOfJoin {+ /// Left input. Every left row is preserved exactly once.+ pub left: Arc<LogicalPlan>,+ /// Right input.+ pub right: Arc<LogicalPlan>,+ /// Equality clauses expressed as pairs of left and right expressions.+ pub on: Vec<(Expr, Expr)>,+ /// Ordered match condition.+ pub match_condition: Box<AsOfMatch>,+ /// Whether equality keys came from `ON` or `USING`.+ pub join_constraint: JoinConstraint,+ /// Output schema.+ pub schema: DFSchemaRef,+}++impl AsOfJoin {+ /// Creates an ASOF join and validates its logical contract.+ ///+ /// This is the pre-coercion boundary. The physical ASOF constructor repeats+ /// the shared operator, side-ownership, and determinism checks for direct+ /// physical-plan callers and adds execution-only constraints. Keep the+ /// shared checks aligned across both entry points.+ pub fn try_new(+ left: Arc<LogicalPlan>,+ right: Arc<LogicalPlan>,+ on: Vec<(Expr, Expr)>,+ match_condition: AsOfMatch,+ join_constraint: JoinConstraint,+ ) -> Result<Self> {+ if !matches!(+ match_condition.op,+ Operator::Lt | Operator::LtEq | Operator::Gt | Operator::GtEq+ ) {+ return plan_err!(+ "ASOF MATCH_CONDITION requires <, <=, >, or >=, found {}",+ match_condition.op+ );+ }++ Self::validate_side(&match_condition.left, left.schema(), "left match")?;+ Self::validate_side(&match_condition.right, right.schema(), "right match")?;+ if match_condition.left.is_volatile() || match_condition.right.is_volatile() {+ return plan_err!("ASOF MATCH_CONDITION must be deterministic");+ }++ let left_type = match_condition.left.get_type(left.schema())?;+ let right_type = match_condition.right.get_type(right.schema())?;+ if crate::type_coercion::binary::comparison_coercion(&left_type, &right_type)+ .is_none()+ {+ return plan_err!(+ "ASOF match expressions have incompatible types {left_type} and {right_type}"+ );+ }++ for (left_expr, right_expr) in &on {+ Self::validate_side(left_expr, left.schema(), "left equality")?;+ Self::validate_side(right_expr, right.schema(), "right equality")?;+ if left_expr.is_volatile() || right_expr.is_volatile() {+ return plan_err!("ASOF equality expressions must be deterministic");+ }+ let left_type = left_expr.get_type(left.schema())?;+ let right_type = right_expr.get_type(right.schema())?;+ let Some(common_type) = crate::type_coercion::binary::comparison_coercion(+ &left_type,+ &right_type,+ ) else {+ return plan_err!(+ "ASOF equality expressions have incompatible types {left_type} and {right_type}"+ );+ };+ if !crate::utils::can_hash(&common_type) {+ return plan_err!(+ "ASOF equality expressions have unsupported hash type {common_type}"+ );+ }+ }++ if join_constraint == JoinConstraint::Using+ && on.iter().any(|(left, right)| {+ left.get_as_join_column().is_none()+ || right.get_as_join_column().is_none()+ })+ {+ return plan_err!("ASOF USING keys must be columns");+ }++ let schema = build_asof_join_schema(left.schema(), right.schema())?;+ Ok(Self {+ left,+ right,+ on,+ match_condition: Box::new(match_condition),+ join_constraint,+ schema: Arc::new(schema),+ })+ }++ fn validate_side(expr: &Expr, schema: &DFSchema, name: &str) -> Result<()> {+ let mut columns = HashSet::new();+ expr_to_columns(expr, &mut columns)?;+ if columns.is_empty() {+ return plan_err!("ASOF {name} expression must reference its input");+ }+ if let Some(column) = columns+ .iter()+ .find(|column| !schema.is_column_from_schema(column))+ {+ return plan_err!(+ "ASOF {name} expression references column {column} outside its input"+ );+ }+ Ok(())+ }+}++impl PartialOrd for AsOfJoin {+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {+ (+ &self.left,+ &self.right,+ &self.on,+ &self.match_condition,+ &self.join_constraint,+ )+ .partial_cmp(&(+ &other.left,+ &other.right,+ &other.on,+ &other.match_condition,+ &other.join_constraint,+ ))+ .filter(|cmp| *cmp != Ordering::Equal || self == other)+ }+}+impl Join {/// Creates a new Join operator with automatically computed schema.///@@ -5915,6 +6178,15 @@ mod tests {.build()?;assert_eq!(cross_join.min_rows(), 2);+ let asof_join = LogicalPlanBuilder::from(two_rows.clone())+ .asof_join(+ one_row.clone(),+ vec![],+ AsOfMatch::new(col("l.column1"), Operator::GtEq, col("r.column1")),+ )?+ .build()?;+ assert_eq!(asof_join.min_rows(), 2);+for (join_type, expected_min_rows) in [// An inner join with a join condition may filter out every row,// while outer joins preserve the rows of the outer side(s).@@ -6840,6 +7112,32 @@ mod tests {Ok(())}+ #[test]+ fn test_asof_using_preserves_qualified_keys() -> Result<()> {+ let schema = Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("ts", DataType::Int64, false),+ ]);+ let left = Arc::new(table_scan(Some("t1"), &schema, None)?.build()?);+ let right = Arc::new(table_scan(Some("t2"), &schema, None)?.build()?);+ let join = AsOfJoin::try_new(+ left,+ right,+ vec![(col("t1.id"), col("t2.id"))],+ AsOfMatch::new(col("t1.ts"), Operator::GtEq, col("t2.ts")),+ JoinConstraint::Using,+ )?;++ assert_eq!(join.schema.fields().len(), 4);+ assert_eq!(+ join.schema+ .index_of_column(&Column::from_qualified_name("t2.id"))?,+ 2+ );+ assert!(join.schema.field(2).is_nullable());+ Ok(())+ }+#[test]fn test_join_try_new_schema_validation() -> Result<()> {let left_schema = Schema::new(vec![diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rsindex c4c1d743b5..ee43666736 100644--- a/datafusion/expr/src/logical_plan/tree_node.rs+++ b/datafusion/expr/src/logical_plan/tree_node.rs@@ -41,11 +41,12 @@ use std::sync::Arc;use crate::logical_plan::plan::RangePartitioning;use crate::{- Aggregate, Analyze, CreateMemoryTable, CreateView, DdlStatement, Distinct,- DistinctOn, DmlStatement, Execute, Explain, Expr, Extension, Filter, Join, Limit,- LogicalPlan, Partitioning, Prepare, Projection, RecursiveQuery, Repartition, Sort,- Statement, Subquery, SubqueryAlias, TableScan, Union, Unnest, UserDefinedLogicalNode,- Values, Window, WriteOp, builder::unnest_with_options, dml::CopyTo,+ Aggregate, Analyze, AsOfJoin, AsOfMatch, CreateMemoryTable, CreateView, DdlStatement,+ Distinct, DistinctOn, DmlStatement, Execute, Explain, Expr, Extension, Filter, Join,+ Limit, LogicalPlan, Partitioning, Prepare, Projection, RecursiveQuery, Repartition,+ Sort, Statement, Subquery, SubqueryAlias, TableScan, Union, Unnest,+ UserDefinedLogicalNode, Values, Window, WriteOp, builder::unnest_with_options,+ dml::CopyTo,};use datafusion_common::tree_node::TreeNodeRefContainer;@@ -150,6 +151,23 @@ impl TreeNode for LogicalPlan {null_aware,})}),+ LogicalPlan::AsOfJoin(AsOfJoin {+ left,+ right,+ on,+ match_condition,+ join_constraint,+ schema,+ }) => (left, right).map_elements(f)?.update_data(|(left, right)| {+ LogicalPlan::AsOfJoin(AsOfJoin {+ left,+ right,+ on,+ match_condition,+ join_constraint,+ schema,+ })+ }),LogicalPlan::Limit(Limit { skip, fetch, input }) => input.map_elements(f)?.update_data(|input| LogicalPlan::Limit(Limit { skip, fetch, input })),@@ -447,6 +465,13 @@ impl LogicalPlan {LogicalPlan::Join(Join { on, filter, .. }) => {(on, filter).apply_ref_elements(f)}+ LogicalPlan::AsOfJoin(AsOfJoin {+ on,+ match_condition,+ ..+ }) => on.apply_elements(&mut f)?.visit_sibling(|| {+ (&match_condition.left, &match_condition.right).apply_ref_elements(&mut f)+ }),LogicalPlan::Sort(Sort { expr, .. }) => expr.apply_elements(f),LogicalPlan::Extension(extension) => {// would be nice to avoid this copy -- maybe can@@ -614,6 +639,29 @@ impl LogicalPlan {null_aware,})}),+ LogicalPlan::AsOfJoin(AsOfJoin {+ left,+ right,+ on,+ match_condition,+ join_constraint,+ schema,+ }) => (on, (match_condition.left, match_condition.right))+ .map_elements(f)?+ .update_data(|(on, (left_match, right_match))| {+ LogicalPlan::AsOfJoin(AsOfJoin {+ left,+ right,+ on,+ match_condition: Box::new(AsOfMatch {+ left: left_match,+ op: match_condition.op,+ right: right_match,+ }),+ join_constraint,+ schema,+ })+ }),LogicalPlan::Sort(Sort { expr, input, fetch }) => expr.map_elements(f)?.update_data(|expr| LogicalPlan::Sort(Sort { expr, input, fetch })),diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex 13b9541b6a..07d3173d63 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -57,10 +57,10 @@ use datafusion_expr::type_coercion::{};use datafusion_expr::utils::merge_schema;use datafusion_expr::{- Cast, DmlStatement, Expr, ExprSchemable, Join, Limit, LogicalPlan, Operator,- Projection, Union, ValueOrLambda, WindowFrame, WindowFrameBound, WindowFrameUnits,- WriteOp, is_false, is_not_false, is_not_true, is_not_unknown, is_true, is_unknown,- lit, not,+ AsOfJoin, AsOfMatch, Cast, DmlStatement, Expr, ExprSchemable, Join, Limit,+ LogicalPlan, Operator, Projection, Union, ValueOrLambda, WindowFrame,+ WindowFrameBound, WindowFrameUnits, WriteOp, is_false, is_not_false, is_not_true,+ is_not_unknown, is_true, is_unknown, lit, not,};/// Performs type coercion by determining the schema@@ -191,6 +191,7 @@ impl<'a> TypeCoercionRewriter<'a> {pub fn coerce_plan(&mut self, plan: LogicalPlan) -> Result<LogicalPlan> {match plan {LogicalPlan::Join(join) => self.coerce_join(join),+ LogicalPlan::AsOfJoin(join) => self.coerce_asof_join(join),LogicalPlan::Union(union) => Self::coerce_union(union),LogicalPlan::Limit(limit) => Self::coerce_limit(limit),LogicalPlan::Dml(dml) => self.coerce_dml(dml),@@ -284,6 +285,36 @@ impl<'a> TypeCoercionRewriter<'a> {Ok(LogicalPlan::Join(join))}+ /// Coerce ASOF equality and ordered match expressions across input schemas.+ pub fn coerce_asof_join(&mut self, mut join: AsOfJoin) -> Result<LogicalPlan> {+ join.on = join+ .on+ .into_iter()+ .map(|(left, right)| {+ self.coerce_binary_op(+ left,+ join.left.schema(),+ Operator::Eq,+ right,+ join.right.schema(),+ )+ })+ .collect::<Result<_>>()?;+ let (left, right) = self.coerce_binary_op(+ join.match_condition.left,+ join.left.schema(),+ join.match_condition.op,+ join.match_condition.right,+ join.right.schema(),+ )?;+ join.match_condition = Box::new(AsOfMatch {+ left,+ op: join.match_condition.op,+ right,+ });+ Ok(LogicalPlan::AsOfJoin(join))+ }+/// Coerce the union’s inputs to a common schema compatible with all inputs./// This occurs after wildcard expansion and the coercion of the input expressions.pub fn coerce_union(union_plan: Union) -> Result<LogicalPlan> {diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rsindex 94d61b7487..6004ccb86e 100644--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs@@ -566,6 +566,7 @@ impl OptimizerRule for CommonSubexprEliminate {LogicalPlan::Window(window) => self.try_optimize_window(window, config)?,LogicalPlan::Aggregate(agg) => self.try_optimize_aggregate(agg, config)?,LogicalPlan::Join(_)+ | LogicalPlan::AsOfJoin(_)| LogicalPlan::Repartition(_)| LogicalPlan::Union(_)| LogicalPlan::TableScan(_)diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rsindex 9604bf2f18..3bbaf887ca 100644--- a/datafusion/optimizer/src/optimize_projections/mod.rs+++ b/datafusion/optimizer/src/optimize_projections/mod.rs@@ -407,6 +407,26 @@ fn optimize_projections(right_indices.with_projection_beneficial(),]}+ LogicalPlan::AsOfJoin(join) => {+ let left_len = join.left.schema().fields().len();+ let mut left_required = Vec::new();+ let mut right_required = Vec::new();+ for index in indices.indices() {+ if *index < left_len {+ left_required.push(*index);+ } else {+ right_required.push(*index - left_len);+ }+ }+ let left_indices = RequiredIndices::new_from_indices(left_required)+ .with_plan_exprs(&plan, join.left.schema())?;+ let right_indices = RequiredIndices::new_from_indices(right_required)+ .with_plan_exprs(&plan, join.right.schema())?;+ vec![+ left_indices.with_projection_beneficial(),+ right_indices.with_projection_beneficial(),+ ]+ }// these nodes are explicitly rewritten in the match statement aboveLogicalPlan::Projection(_)| LogicalPlan::Aggregate(_)diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rsindex 46c0118095..ca4a6688b5 100644--- a/datafusion/optimizer/src/optimizer.rs+++ b/datafusion/optimizer/src/optimizer.rs@@ -411,6 +411,11 @@ fn map_children_mut<F: FnMut(&mut LogicalPlan) -> Result<bool>>(let r = f(Arc::make_mut(right))?;l || r}+ LogicalPlan::AsOfJoin(join) => {+ let l = f(Arc::make_mut(&mut join.left))?;+ let r = f(Arc::make_mut(&mut join.right))?;+ l || r+ }LogicalPlan::Union(Union { inputs, .. }) => {let mut changed = false;for input in inputs {diff --git a/datafusion/physical-plan/src/joins/asof_join.rs b/datafusion/physical-plan/src/joins/asof_join.rsindex 7cc19e1153..3e67e929d0 100644--- a/datafusion/physical-plan/src/joins/asof_join.rs+++ b/datafusion/physical-plan/src/joins/asof_join.rs@@ -169,6 +169,10 @@ impl AsOfJoinExec {/// floating-point equality keys are not supported because Arrow sorting/// distinguishes signed zero while SQL equality does not. Projection indices/// refer to the full left-then-right join schema.+ ///+ /// The logical ASOF constructor validates the corresponding pre-coercion+ /// contract. Keep the shared operator, side-ownership, and determinism checks+ /// aligned across both public entry points.pub fn try_new(left: Arc<dyn ExecutionPlan>,right: Arc<dyn ExecutionPlan>,diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex 7fa6fd4ed5..90d48b056c 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -2225,6 +2225,9 @@ impl AsLogicalPlan for LogicalPlanNode {LogicalPlan::DescribeTable(_) => Err(proto_error("LogicalPlan serde is not yet implemented for DescribeTable",)),+ LogicalPlan::AsOfJoin(_) => Err(proto_error(+ "LogicalPlan serde is not yet implemented for AsOfJoin",+ )),LogicalPlan::RecursiveQuery(recursive) => {let static_term = LogicalPlanNode::try_from_logical_plan(recursive.static_term.as_ref(),diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rsindex 1a4a094e96..29d4d1b5b8 100644--- a/datafusion/sql/src/unparser/plan.rs+++ b/datafusion/sql/src/unparser/plan.rs@@ -204,6 +204,7 @@ impl Unparser<'_> {| LogicalPlan::Copy(_)| LogicalPlan::DescribeTable(_)| LogicalPlan::RecursiveQuery(_)+ | LogicalPlan::AsOfJoin(_)| LogicalPlan::Unnest(_) => not_impl_err!("Unsupported plan: {plan:?}"),}}diff --git a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs b/datafusion/substrait/src/logical_plan/producer/rel/mod.rsindex c3599a2635..15f59919a2 100644--- a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs+++ b/datafusion/substrait/src/logical_plan/producer/rel/mod.rs@@ -51,6 +51,9 @@ pub fn to_substrait_rel(LogicalPlan::Aggregate(plan) => producer.handle_aggregate(plan),LogicalPlan::Sort(plan) => producer.handle_sort(plan),LogicalPlan::Join(plan) => producer.handle_join(plan),+ LogicalPlan::AsOfJoin(plan) => {+ not_impl_err!("Substrait ASOF join is not supported: {plan:?}")?+ }LogicalPlan::Repartition(plan) => producer.handle_repartition(plan),LogicalPlan::Union(plan) => producer.handle_union(plan),LogicalPlan::TableScan(plan) => producer.handle_table_scan(plan),diff --git a/datafusion/substrait/tests/cases/serialize.rs b/datafusion/substrait/tests/cases/serialize.rsindex 1981ef66db..75e1fe251a 100644--- a/datafusion/substrait/tests/cases/serialize.rs+++ b/datafusion/substrait/tests/cases/serialize.rs@@ -18,7 +18,7 @@#[cfg(test)]mod tests {use datafusion::datasource::provider_as_source;- use datafusion::logical_expr::LogicalPlanBuilder;+ use datafusion::logical_expr::{AsOfMatch, LogicalPlanBuilder, Operator};use datafusion_substrait::logical_plan::consumer::from_substrait_plan;use datafusion_substrait::logical_plan::producer::to_substrait_plan;use datafusion_substrait::serializer;@@ -27,7 +27,7 @@ mod tests {use datafusion::prelude::*;use insta::assert_snapshot;- use std::fs;+ use std::{fs, sync::Arc};use substrait::proto::expression::field_reference::{ReferenceType, RootType};use substrait::proto::expression::reference_segment;use substrait::proto::expression::{ReferenceSegment, RexType};@@ -103,6 +103,29 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn asof_join_fails_closed_until_substrait_has_an_extension() -> Result<()> {+ let ctx = create_context().await?;+ let table = provider_as_source(ctx.table_provider("data").await?);+ let left = LogicalPlanBuilder::scan("l", Arc::clone(&table), None)?.build()?;+ let right = LogicalPlanBuilder::scan("r", table, None)?.build()?;+ let plan = LogicalPlanBuilder::from(left)+ .asof_join(+ right,+ vec![(col("l.b"), col("r.b"))],+ AsOfMatch::new(col("l.a"), Operator::GtEq, col("r.a")),+ )?+ .build()?;+ let error = to_substrait_plan(&plan, &ctx.state())+ .expect_err("ASOF must not be lowered to a generic Substrait join");+ assert!(+ error+ .to_string()+ .contains("Substrait ASOF join is not supported")+ );+ Ok(())+ }+#[tokio::test]async fn include_remaps_for_projects() -> Result<()> {let ctx = create_context().await?;