Apache DataFusion SQL Query Engine
feat(substrait): serialize correlated subqueries with OuterReference field references (#23488)
@clflushoptbec06e7506dd
diff --git a/datafusion/substrait/src/logical_plan/producer/expr/field_reference.rs b/datafusion/substrait/src/logical_plan/producer/expr/field_reference.rsindex aa34317a6e..12ed92c64f 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/field_reference.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/field_reference.rs@@ -15,11 +15,12 @@// specific language governing permissions and limitations// under the License.+use crate::logical_plan::producer::SubstraitProducer;use datafusion::common::{Column, DFSchemaRef, substrait_err};use datafusion::logical_expr::Expr;use substrait::proto::Expression;use substrait::proto::expression::field_reference::{- ReferenceType, RootReference, RootType,+ OuterReference, ReferenceType, RootReference, RootType,};use substrait::proto::expression::{FieldReference, ReferenceSegment, RexType, reference_segment,@@ -35,6 +36,13 @@ pub fn from_column(pub(crate) fn substrait_field_ref(index: usize,+) -> datafusion::common::Result<Expression> {+ substrait_field_ref_with_root(index, RootType::RootReference(RootReference {}))+}++fn substrait_field_ref_with_root(+ index: usize,+ root_type: RootType,) -> datafusion::common::Result<Expression> {Ok(Expression {rex_type: Some(RexType::Selection(Box::new(FieldReference {@@ -46,7 +54,7 @@ pub(crate) fn substrait_field_ref(}),)),})),- root_type: Some(RootType::RootReference(RootReference {})),+ root_type: Some(root_type),}))),})}@@ -76,26 +84,45 @@ pub(crate) fn try_to_substrait_field_reference(}}-/// Convert an outer reference column to a Substrait field reference.-/// Outer reference columns reference columns from an outer query scope in correlated subqueries.-/// We convert them the same way as regular columns since the subquery plan will be-/// reconstructed with the proper schema context during consumption.+/// Convert an outer reference column to a Substrait field reference with an+/// `OuterReference` root type.+///+/// Outer reference columns reference columns from an enclosing query scope in+/// correlated subqueries. The column is resolved against the producer's stack+/// of outer schemas (pushed at each subquery boundary), innermost first, and+/// the resulting `steps_out` records how many query boundaries the reference+/// crosses (`steps_out = 1` is the immediately enclosing query).pub fn from_outer_reference_column(+ producer: &mut impl SubstraitProducer,col: &Column,- schema: &DFSchemaRef,) -> datafusion::common::Result<Expression> {- // OuterReferenceColumn is converted similarly to a regular column reference.- // The schema provided should be the schema context in which the outer reference- // column appears. During Substrait round-trip, the consumer will reconstruct- // the outer reference based on the subquery context.- let index = schema.index_of_column(col)?;- substrait_field_ref(index)+ let mut steps_out = 1;+ while let Some(outer_schema) = producer.get_outer_schema(steps_out) {+ if let Some(index) = outer_schema.maybe_index_of_column(col) {+ return substrait_field_ref_with_root(+ index,+ RootType::OuterReference(OuterReference {+ steps_out: steps_out as u32,+ }),+ );+ }+ steps_out += 1;+ }+ substrait_err!(+ "Outer reference column '{col}' could not be resolved against any outer \+ query schema. If using a custom SubstraitProducer, ensure it maintains \+ the outer schema stack (push_outer_schema/pop_outer_schema/get_outer_schema)"+ )}#[cfg(test)]mod tests {use super::*;- use datafusion::common::Result;+ use crate::logical_plan::producer::DefaultSubstraitProducer;+ use datafusion::arrow::datatypes::{DataType, Field, Schema};+ use datafusion::common::{DFSchema, Result};+ use datafusion::execution::SessionStateBuilder;+ use std::sync::Arc;#[test]fn to_field_reference() -> Result<()> {@@ -116,4 +143,26 @@ mod tests {}Ok(())}++ #[test]+ fn unresolvable_outer_reference_column() -> Result<()> {+ let state = SessionStateBuilder::default().build();+ let mut producer = DefaultSubstraitProducer::new(&state);+ let col = Column::from_qualified_name("data.a");++ // Empty outer schema stack: nothing to resolve against.+ let err = from_outer_reference_column(&mut producer, &col).unwrap_err();+ assert!(err.to_string().contains("could not be resolved"));++ // Non-empty stack whose schemas don't contain the column.+ let outer_schema = Arc::new(DFSchema::try_from(Schema::new(vec![Field::new(+ "unrelated",+ DataType::Int64,+ true,+ )]))?);+ producer.push_outer_schema(outer_schema);+ let err = from_outer_reference_column(&mut producer, &col).unwrap_err();+ assert!(err.to_string().contains("could not be resolved"));+ Ok(())+ }}diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rsindex c728af2f14..a95e46a8c0 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs@@ -149,10 +149,8 @@ pub fn to_substrait_rex(Expr::Wildcard { .. } => not_impl_err!("Cannot convert {expr:?} to Substrait"),Expr::GroupingSet(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"),Expr::Placeholder(expr) => producer.handle_placeholder(expr, schema),- Expr::OuterReferenceColumn(_, _) => {- // OuterReferenceColumn requires tracking outer query schema context for correlated- // subqueries. This is a complex feature that is not yet implemented.- not_impl_err!("Cannot convert {expr:?} to Substrait")+ Expr::OuterReferenceColumn(field, col) => {+ producer.handle_outer_reference_column(field, col, schema)}Expr::Unnest(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"),Expr::HigherOrderFunction(expr) => {diff --git a/datafusion/substrait/src/logical_plan/producer/expr/subquery.rs b/datafusion/substrait/src/logical_plan/producer/expr/subquery.rsindex 97699c2132..4affbbaf29 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/subquery.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/subquery.rs@@ -18,11 +18,29 @@use crate::logical_plan::producer::{SubstraitProducer, negate};use datafusion::common::{DFSchemaRef, substrait_err};use datafusion::logical_expr::expr::{Exists, InSubquery, SetComparison, SetQuantifier};-use datafusion::logical_expr::{Operator, Subquery};-use substrait::proto::Expression;+use datafusion::logical_expr::{LogicalPlan, Operator, Subquery};+use std::sync::Arc;use substrait::proto::expression::RexType;use substrait::proto::expression::subquery::set_comparison::{ComparisonOp, ReductionOp};use substrait::proto::expression::subquery::{InPredicate, Scalar, SetPredicate};+use substrait::proto::{Expression, Rel};++/// Serialize a subquery plan, making the enclosing query's schema available+/// for resolving correlated column references.+///+/// Substrait represents correlated references using `OuterReference` field+/// references with a `steps_out` depth. To produce these, the producer+/// maintains a stack of outer schemas.+fn produce_subquery_rel(+ producer: &mut impl SubstraitProducer,+ plan: &LogicalPlan,+ outer_schema: &DFSchemaRef,+) -> datafusion::common::Result<Box<Rel>> {+ producer.push_outer_schema(Arc::clone(outer_schema));+ let result = producer.handle_plan(plan);+ producer.pop_outer_schema();+ result+}pub fn from_in_subquery(producer: &mut impl SubstraitProducer,@@ -36,7 +54,8 @@ pub fn from_in_subquery(} = subquery;let substrait_expr = producer.handle_expr(expr, schema)?;- let subquery_plan = producer.handle_plan(subquery.subquery.as_ref())?;+ let subquery_plan =+ produce_subquery_rel(producer, subquery.subquery.as_ref(), schema)?;let substrait_subquery = Expression {rex_type: Some(RexType::Subquery(Box::new(@@ -88,8 +107,11 @@ pub fn from_set_comparison(let comparison_op = comparison_op_to_proto(&set_comparison.op)? as i32;let reduction_op = reduction_op_to_proto(&set_comparison.quantifier)? as i32;let left = producer.handle_expr(set_comparison.expr.as_ref(), schema)?;- let subquery_plan =- producer.handle_plan(set_comparison.subquery.subquery.as_ref())?;+ let subquery_plan = produce_subquery_rel(+ producer,+ set_comparison.subquery.subquery.as_ref(),+ schema,+ )?;Ok(Expression {rex_type: Some(RexType::Subquery(Box::new(@@ -113,9 +135,10 @@ pub fn from_set_comparison(pub fn from_scalar_subquery(producer: &mut impl SubstraitProducer,subquery: &Subquery,- _schema: &DFSchemaRef,+ schema: &DFSchemaRef,) -> datafusion::common::Result<Expression> {- let subquery_plan = producer.handle_plan(subquery.subquery.as_ref())?;+ let subquery_plan =+ produce_subquery_rel(producer, subquery.subquery.as_ref(), schema)?;Ok(Expression {rex_type: Some(RexType::Subquery(Box::new(@@ -136,9 +159,10 @@ pub fn from_scalar_subquery(pub fn from_exists(producer: &mut impl SubstraitProducer,exists: &Exists,- _schema: &DFSchemaRef,+ schema: &DFSchemaRef,) -> datafusion::common::Result<Expression> {- let subquery_plan = producer.handle_plan(exists.subquery.subquery.as_ref())?;+ let subquery_plan =+ produce_subquery_rel(producer, exists.subquery.subquery.as_ref(), schema)?;let substrait_exists = Expression {rex_type: Some(RexType::Subquery(Box::new(diff --git a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rsindex 6d54d32cad..dcd8e49b9d 100644--- a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs+++ b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs@@ -21,10 +21,10 @@ use crate::logical_plan::producer::{from_case, from_cast, from_column, from_distinct, from_empty_relation, from_exists,from_filter, from_higher_order_function, from_in_list, from_in_subquery, from_join,from_lambda, from_lambda_variable, from_like, from_limit, from_literal,- from_placeholder, from_projection, from_repartition, from_scalar_function,- from_scalar_subquery, from_set_comparison, from_sort, from_subquery_alias,- from_table_scan, from_try_cast, from_unary_expr, from_union, from_values,- from_window, from_window_function, to_substrait_rel, to_substrait_rex,+ from_outer_reference_column, from_placeholder, from_projection, from_repartition,+ from_scalar_function, from_scalar_subquery, from_set_comparison, from_sort,+ from_subquery_alias, from_table_scan, from_try_cast, from_unary_expr, from_union,+ from_values, from_window, from_window_function, to_substrait_rel, to_substrait_rex,to_substrait_type_from_field,};use datafusion::arrow::datatypes::FieldRef;@@ -433,6 +433,15 @@ pub trait SubstraitProducer: Send + Sync + Sized {from_exists(self, exists, schema)}+ fn handle_outer_reference_column(+ &mut self,+ _field: &FieldRef,+ column: &Column,+ _schema: &DFSchemaRef,+ ) -> datafusion::common::Result<Expression> {+ from_outer_reference_column(self, column)+ }+fn handle_placeholder(&mut self,placeholder: &Placeholder,@@ -441,6 +450,30 @@ pub trait SubstraitProducer: Send + Sync + Sized {from_placeholder(self, placeholder)}+ // Outer Schema management API.+ //+ // These methods manage a stack of outer schemas for correlated subquery support+ // such as when entering a subquery, the enclosing query's schema is pushed onto+ // the stack.+ //+ // Serializing an Expr::OuterReferenceColumn uses these to resolve the column+ // against the correct enclosing query and emit an OuterReference field reference+ // with the corresponding `steps_out`.++ /// Push an outer schema onto the stack when entering a subquery.+ fn push_outer_schema(&mut self, _schema: DFSchemaRef) {}++ /// Pop an outer schema from the stack when leaving a subquery.+ fn pop_outer_schema(&mut self) {}++ /// Get the outer schema at the given nesting depth.+ /// `steps_out = 1` is the immediately enclosing query, `steps_out = 2`+ /// is two levels out, etc. Returns `None` if `steps_out` is 0 or+ /// exceeds the current nesting depth.+ fn get_outer_schema(&self, _steps_out: usize) -> Option<DFSchemaRef> {+ None+ }+fn handle_lambda(&mut self,lambda: &Lambda,@@ -499,6 +532,7 @@ pub struct DefaultSubstraitProducer<'a> {extensions: Extensions,serializer_registry: &'a dyn SerializerRegistry,lambda_producer: DefaultSubstraitLambdaProducer,+ outer_schemas: Vec<DFSchemaRef>,}impl<'a> DefaultSubstraitProducer<'a> {@@ -507,6 +541,7 @@ impl<'a> DefaultSubstraitProducer<'a> {extensions: Extensions::default(),serializer_registry: state.serializer_registry().as_ref(),lambda_producer: DefaultSubstraitLambdaProducer::new(),+ outer_schemas: Vec::new(),}}}@@ -562,6 +597,23 @@ impl SubstraitProducer for DefaultSubstraitProducer<'_> {}))}+ fn push_outer_schema(&mut self, schema: DFSchemaRef) {+ self.outer_schemas.push(schema);+ }++ fn pop_outer_schema(&mut self) {+ self.outer_schemas.pop();+ }++ fn get_outer_schema(&self, steps_out: usize) -> Option<DFSchemaRef> {+ // steps_out=1 → last element, steps_out=2 → second-to-last, etc.+ // Returns None for steps_out=0 or steps_out > stack depth.+ self.outer_schemas+ .len()+ .checked_sub(steps_out)+ .and_then(|idx| self.outer_schemas.get(idx).cloned())+ }+fn push_lambda_parameters(&mut self,lambda_parameters: Vec<FieldRef>,diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rsindex f084d3170e..4dc4f03d05 100644--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs@@ -751,6 +751,207 @@ async fn roundtrip_not_exists_substrait() -> Result<()> {Ok(())}+/// `(steps_out, field index)` of every `OuterReference` field reference in the+/// plan, collected by walking its JSON serialization.+fn outer_references(plan: &Plan) -> Vec<(u32, u32)> {+ fn walk(value: &serde_json::Value, out: &mut Vec<(u32, u32)>) {+ match value {+ serde_json::Value::Object(map) => {+ // A FieldReference serializes its root type and direct reference+ // as sibling keys, e.g.:+ // { "directReference": { "structField": { "field": 3 } },+ // "outerReference": { "stepsOut": 1 } }+ // proto3 JSON omits zero-valued fields, hence the unwrap_or(0)s.+ if let Some(outer) = map.get("outerReference") {+ let steps =+ outer.get("stepsOut").and_then(|v| v.as_u64()).unwrap_or(0);+ let field = map+ .get("directReference")+ .and_then(|d| d.get("structField"))+ .and_then(|s| s.get("field"))+ .and_then(|f| f.as_u64())+ .unwrap_or(0);+ out.push((steps as u32, field as u32));+ }+ map.values().for_each(|v| walk(v, out));+ }+ serde_json::Value::Array(items) => items.iter().for_each(|v| walk(v, out)),+ _ => {}+ }+ }+ let mut refs = vec![];+ walk(+ &serde_json::to_value(plan).expect("Plan serializes to JSON"),+ &mut refs,+ );+ refs.sort_unstable(); // key order in the JSON walk isn't proto field order+ refs+}++#[tokio::test]+async fn roundtrip_correlated_exists() -> Result<()> {+ let ctx = create_context().await?;+ let plan = ctx+ .sql("SELECT b FROM data WHERE EXISTS (SELECT 1 FROM data2 WHERE data2.a = data.a)")+ .await?+ .into_unoptimized_plan();++ let proto = to_substrait_plan(&plan, &ctx.state())?;+ assert_eq!(outer_references(&proto), vec![(1, 0)]);++ let plan2 = from_substrait_plan(&ctx.state(), &proto).await?;+ assert_eq!(plan.schema(), plan2.schema());+ assert_snapshot!(+ plan2,+ @r"+ Projection: data.b+ Filter: EXISTS (<subquery>)+ Subquery:+ Projection: Int64(1)+ Filter: data2.a = outer_ref(data.a)+ TableScan: data2+ TableScan: data+ "+ );+ Ok(())+}++#[tokio::test]+async fn roundtrip_correlated_in_subquery() -> Result<()> {+ let ctx = create_context().await?;+ let plan = ctx+ .sql("SELECT b FROM data WHERE a IN (SELECT data2.a FROM data2 WHERE data2.d = data.d)")+ .await?+ .into_unoptimized_plan();++ let proto = to_substrait_plan(&plan, &ctx.state())?;+ assert_eq!(outer_references(&proto), vec![(1, 3)]);++ let plan2 = from_substrait_plan(&ctx.state(), &proto).await?;+ assert_eq!(plan.schema(), plan2.schema());+ assert_snapshot!(+ plan2,+ @r"+ Projection: data.b+ Filter: data.a IN (<subquery>)+ Subquery:+ Projection: data2.a+ Filter: data2.d = outer_ref(data.d)+ TableScan: data2+ TableScan: data+ "+ );+ Ok(())+}++#[tokio::test]+async fn roundtrip_correlated_scalar_subquery() -> Result<()> {+ let ctx = create_context().await?;+ let plan = ctx+ .sql("SELECT a FROM data WHERE a < (SELECT sum(data2.a) FROM data2 WHERE data2.a = data.a)")+ .await?+ .into_unoptimized_plan();++ let proto = to_substrait_plan(&plan, &ctx.state())?;+ assert_eq!(outer_references(&proto), vec![(1, 0)]);++ let plan2 = from_substrait_plan(&ctx.state(), &proto).await?;+ assert_eq!(plan.schema(), plan2.schema());+ assert_snapshot!(+ plan2,+ @r"+ Projection: data.a+ Filter: data.a < (<subquery>)+ Subquery:+ Projection: sum(data2.a)+ Aggregate: groupBy=[[]], aggr=[[sum(data2.a)]]+ Filter: data2.a = outer_ref(data.a)+ TableScan: data2+ TableScan: data+ "+ );+ Ok(())+}++#[tokio::test]+async fn roundtrip_nested_correlated_subquery() -> Result<()> {+ let ctx = create_context().await?;+ // The innermost subquery references `data.a` from the outermost query,+ // two subquery boundaries out.+ let plan = ctx+ .sql(+ "SELECT b FROM data WHERE EXISTS (\+ SELECT 1 FROM data2 WHERE data2.a = data.a AND EXISTS (\+ SELECT 1 FROM book WHERE book.isbn = data.a))",+ )+ .await?+ .into_unoptimized_plan();++ let proto = to_substrait_plan(&plan, &ctx.state())?;+ assert_eq!(outer_references(&proto), vec![(1, 0), (2, 0)]);++ let plan2 = from_substrait_plan(&ctx.state(), &proto).await?;+ assert_eq!(plan.schema(), plan2.schema());+ assert_snapshot!(+ plan2,+ @r"+ Projection: data.b+ Filter: EXISTS (<subquery>)+ Subquery:+ Projection: Int64(1)+ Filter: data2.a = outer_ref(data.a) AND EXISTS (<subquery>)+ Subquery:+ Projection: Int64(1)+ Filter: book.isbn = outer_ref(data.a)+ TableScan: book+ TableScan: data2+ TableScan: data+ "+ );+ Ok(())+}++#[tokio::test]+async fn roundtrip_correlated_subquery_shadowed_outer_column() -> Result<()> {+ let ctx = create_context().await?;+ // Both the outermost query and the middle subquery scan `data`, so the+ // qualified name `data.a` is present in two enclosing scopes when the+ // innermost subquery references it. Following SQL scoping rules (and+ // SqlToRel's own resolution), the reference binds to the *nearest*+ // enclosing scope: the producer must emit steps_out = 1, not 2.+ let plan = ctx+ .sql(+ "SELECT b FROM data WHERE EXISTS (\+ SELECT 1 FROM data WHERE EXISTS (\+ SELECT 1 FROM book WHERE book.isbn = data.a))",+ )+ .await?+ .into_unoptimized_plan();++ let proto = to_substrait_plan(&plan, &ctx.state())?;+ assert_eq!(outer_references(&proto), vec![(1, 0)]);++ let plan2 = from_substrait_plan(&ctx.state(), &proto).await?;+ assert_eq!(plan.schema(), plan2.schema());+ assert_snapshot!(+ plan2,+ @r"+ Projection: data.b+ Filter: EXISTS (<subquery>)+ Subquery:+ Projection: Int64(1)+ Filter: EXISTS (<subquery>)+ Subquery:+ Projection: Int64(1)+ Filter: book.isbn = outer_ref(data.a)+ TableScan: book+ TableScan: data+ TableScan: data+ "+ );+ Ok(())+}+#[tokio::test]async fn roundtrip_not_exists_filter_left_anti_join() -> Result<()> {let plan = generate_plan_from_sql(