Apache DataFusion SQL Query Engine
fix(substrait): Correctly parse field references in subqueries (#20439)
Neil Conwaya79e6e6b3972
diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs b/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rsindex c17bf9c92e..50d93a4600 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs@@ -16,34 +16,48 @@// under the License.use crate::logical_plan::consumer::SubstraitConsumer;-use datafusion::common::{Column, DFSchema, not_impl_err};+use datafusion::common::{Column, DFSchema, not_impl_err, substrait_err};use datafusion::logical_expr::Expr;+use std::sync::Arc;use substrait::proto::expression::FieldReference;use substrait::proto::expression::field_reference::ReferenceType::DirectReference;+use substrait::proto::expression::field_reference::RootType;use substrait::proto::expression::reference_segment::ReferenceType::StructField;pub async fn from_field_reference(- _consumer: &impl SubstraitConsumer,+ consumer: &impl SubstraitConsumer,field_ref: &FieldReference,input_schema: &DFSchema,) -> datafusion::common::Result<Expr> {- from_substrait_field_reference(field_ref, input_schema)+ from_substrait_field_reference(consumer, field_ref, input_schema)}pub(crate) fn from_substrait_field_reference(+ consumer: &impl SubstraitConsumer,field_ref: &FieldReference,input_schema: &DFSchema,) -> datafusion::common::Result<Expr> {match &field_ref.reference_type {Some(DirectReference(direct)) => match &direct.reference_type.as_ref() {- Some(StructField(x)) => match &x.child.as_ref() {- Some(_) => not_impl_err!(- "Direct reference StructField with child is not supported"- ),- None => Ok(Expr::Column(Column::from(- input_schema.qualified_field(x.field as usize),- ))),- },+ Some(StructField(struct_field)) => {+ if struct_field.child.is_some() {+ return not_impl_err!(+ "Direct reference StructField with child is not supported"+ );+ }+ let field_idx = struct_field.field as usize;+ match &field_ref.root_type {+ Some(RootType::RootReference(_)) | None => Ok(Expr::Column(+ Column::from(input_schema.qualified_field(field_idx)),+ )),+ Some(RootType::OuterReference(outer_ref)) => {+ resolve_outer_reference(consumer, outer_ref, field_idx)+ }+ Some(RootType::Expression(_)) => not_impl_err!(+ "Expression root type in field reference is not supported"+ ),+ }+ }_ => not_impl_err!("Direct reference with types other than StructField is not supported"),@@ -51,3 +65,20 @@ pub(crate) fn from_substrait_field_reference(_ => not_impl_err!("unsupported field ref type"),}}++fn resolve_outer_reference(+ consumer: &impl SubstraitConsumer,+ outer_ref: &substrait::proto::expression::field_reference::OuterReference,+ field_idx: usize,+) -> datafusion::common::Result<Expr> {+ let steps_out = outer_ref.steps_out as usize;+ let Some(outer_schema) = consumer.get_outer_schema(steps_out) else {+ return substrait_err!(+ "OuterReference with steps_out={steps_out} \+ but no outer schema is available"+ );+ };+ let (qualifier, field) = outer_schema.qualified_field(field_idx);+ let col = Column::from((qualifier, field));+ Ok(Expr::OuterReferenceColumn(Arc::clone(field), col))+}diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/mod.rs b/datafusion/substrait/src/logical_plan/consumer/expr/mod.rsindex 71e3b9e96e..5d98850c72 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/mod.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/mod.rs@@ -117,10 +117,7 @@ pub async fn from_substrait_extended_expr(return not_impl_err!("Type variation extensions are not supported");}- let consumer = DefaultSubstraitConsumer {- extensions: &extensions,- state,- };+ let consumer = DefaultSubstraitConsumer::new(&extensions, state);let input_schema = DFSchemaRef::new(match &extended_expr.base_schema {Some(base_schema) => from_substrait_named_struct(&consumer, base_schema),diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/subquery.rs b/datafusion/substrait/src/logical_plan/consumer/expr/subquery.rsindex 61a381e9eb..83cf8400ee 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/subquery.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/subquery.rs@@ -18,13 +18,31 @@use crate::logical_plan::consumer::SubstraitConsumer;use datafusion::common::{DFSchema, Spans, substrait_datafusion_err, substrait_err};use datafusion::logical_expr::expr::{Exists, InSubquery, SetComparison, SetQuantifier};-use datafusion::logical_expr::{Expr, Operator, Subquery};+use datafusion::logical_expr::{Expr, LogicalPlan, Operator, Subquery};use std::sync::Arc;+use substrait::proto::Rel;use substrait::proto::expression as substrait_expression;use substrait::proto::expression::subquery::SubqueryType;use substrait::proto::expression::subquery::set_comparison::{ComparisonOp, ReductionOp};use substrait::proto::expression::subquery::set_predicate::PredicateOp;+/// Consume a subquery relation, making the enclosing query's schema+/// available for resolving correlated column references.+///+/// Substrait represents correlated references using `OuterReference`+/// field references with a `steps_out` depth. To resolve these,+/// the consumer maintains a stack of outer schemas.+async fn consume_subquery_rel(+ consumer: &impl SubstraitConsumer,+ rel: &Rel,+ outer_schema: &DFSchema,+) -> datafusion::common::Result<LogicalPlan> {+ consumer.push_outer_schema(Arc::new(outer_schema.clone()));+ let result = consumer.consume_rel(rel).await;+ consumer.pop_outer_schema();+ result+}+pub async fn from_subquery(consumer: &impl SubstraitConsumer,subquery: &substrait_expression::Subquery,@@ -41,7 +59,9 @@ pub async fn from_subquery(let needle_expr = &in_predicate.needles[0];let haystack_expr = &in_predicate.haystack;if let Some(haystack_expr) = haystack_expr {- let haystack_expr = consumer.consume_rel(haystack_expr).await?;+ let haystack_expr =+ consume_subquery_rel(consumer, haystack_expr, input_schema)+ .await?;let outer_refs = haystack_expr.all_out_ref_exprs();Ok(Expr::InSubquery(InSubquery {expr: Box::new(@@ -64,9 +84,12 @@ pub async fn from_subquery(}}SubqueryType::Scalar(query) => {- let plan = consumer- .consume_rel(&(query.input.clone()).unwrap_or_default())- .await?;+ let plan = consume_subquery_rel(+ consumer,+ &(query.input.clone()).unwrap_or_default(),+ input_schema,+ )+ .await?;let outer_ref_columns = plan.all_out_ref_exprs();Ok(Expr::ScalarSubquery(Subquery {subquery: Arc::new(plan),@@ -79,9 +102,12 @@ pub async fn from_subquery(// existPredicateOp::Exists => {let relation = &predicate.tuples;- let plan = consumer- .consume_rel(&relation.clone().unwrap_or_default())- .await?;+ let plan = consume_subquery_rel(+ consumer,+ &relation.clone().unwrap_or_default(),+ input_schema,+ )+ .await?;let outer_ref_columns = plan.all_out_ref_exprs();Ok(Expr::Exists(Exists::new(Subquery {@@ -131,7 +157,7 @@ pub async fn from_subquery(};let left_expr = consumer.consume_expression(left, input_schema).await?;- let plan = consumer.consume_rel(right).await?;+ let plan = consume_subquery_rel(consumer, right, input_schema).await?;let outer_ref_columns = plan.all_out_ref_exprs();Ok(Expr::SetComparison(SetComparison::new(diff --git a/datafusion/substrait/src/logical_plan/consumer/plan.rs b/datafusion/substrait/src/logical_plan/consumer/plan.rsindex d5e10fb604..407980c4a7 100644--- a/datafusion/substrait/src/logical_plan/consumer/plan.rs+++ b/datafusion/substrait/src/logical_plan/consumer/plan.rs@@ -35,10 +35,7 @@ pub async fn from_substrait_plan(return not_impl_err!("Type variation extensions are not supported");}- let consumer = DefaultSubstraitConsumer {- extensions: &extensions,- state,- };+ let consumer = DefaultSubstraitConsumer::new(&extensions, state);from_substrait_plan_with_consumer(&consumer, plan).await}diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/exchange_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/exchange_rel.rsindex a6132e047f..b275e523f5 100644--- a/datafusion/substrait/src/logical_plan/consumer/rel/exchange_rel.rs+++ b/datafusion/substrait/src/logical_plan/consumer/rel/exchange_rel.rs@@ -42,7 +42,8 @@ pub async fn from_exchange_rel(let mut partition_columns = vec![];let input_schema = input.schema();for field_ref in &scatter_fields.fields {- let column = from_substrait_field_reference(field_ref, input_schema)?;+ let column =+ from_substrait_field_reference(consumer, field_ref, input_schema)?;partition_columns.push(column);}Partitioning::Hash(partition_columns, exchange.partition_count as usize)diff --git a/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs b/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rsindex 4c19227a30..a23f1faed1 100644--- a/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs+++ b/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs@@ -31,7 +31,7 @@ use datafusion::common::{};use datafusion::execution::{FunctionRegistry, SessionState};use datafusion::logical_expr::{Expr, Extension, LogicalPlan};-use std::sync::Arc;+use std::sync::{Arc, RwLock};use substrait::proto;use substrait::proto::expression as substrait_expression;use substrait::proto::expression::{@@ -364,6 +364,26 @@ pub trait SubstraitConsumer: Send + Sync + Sized {not_impl_err!("Dynamic Parameter expression not supported")}+ // Outer Schema Stack+ // These methods manage a stack of outer schemas for correlated subquery support.+ // When entering a subquery, the enclosing query's schema is pushed onto the stack.+ // Field references with OuterReference root_type use these to resolve columns.++ /// Push an outer schema onto the stack when entering a subquery.+ fn push_outer_schema(&self, _schema: Arc<DFSchema>) {}++ /// Pop an outer schema from the stack when leaving a subquery.+ fn pop_outer_schema(&self) {}++ /// Get the outer schema at the given nesting depth.+ /// `steps_out = 1` is the immediately enclosing query, `steps_out = 2`+ /// is two levels out, etc. Returns `None` if `steps_out` is 0 or+ /// exceeds the current nesting depth (the caller should treat this as+ /// an error in the Substrait plan).+ fn get_outer_schema(&self, _steps_out: usize) -> Option<Arc<DFSchema>> {+ None+ }+// User-Defined Functionality// The details of extension relations, and how to handle them, are fully up to users to specify.@@ -437,11 +457,16 @@ pub trait SubstraitConsumer: Send + Sync + Sized {pub struct DefaultSubstraitConsumer<'a> {pub(super) extensions: &'a Extensions,pub(super) state: &'a SessionState,+ outer_schemas: RwLock<Vec<Arc<DFSchema>>>,}impl<'a> DefaultSubstraitConsumer<'a> {pub fn new(extensions: &'a Extensions, state: &'a SessionState) -> Self {- DefaultSubstraitConsumer { extensions, state }+ DefaultSubstraitConsumer {+ extensions,+ state,+ outer_schemas: RwLock::new(Vec::new()),+ }}}@@ -465,6 +490,24 @@ impl SubstraitConsumer for DefaultSubstraitConsumer<'_> {self.state}+ fn push_outer_schema(&self, schema: Arc<DFSchema>) {+ self.outer_schemas.write().unwrap().push(schema);+ }++ fn pop_outer_schema(&self) {+ self.outer_schemas.write().unwrap().pop();+ }++ fn get_outer_schema(&self, steps_out: usize) -> Option<Arc<DFSchema>> {+ let schemas = self.outer_schemas.read().unwrap();+ // steps_out=1 → last element, steps_out=2 → second-to-last, etc.+ // Returns None for steps_out=0 or steps_out > stack depth.+ schemas+ .len()+ .checked_sub(steps_out)+ .and_then(|idx| schemas.get(idx).cloned())+ }+async fn consume_extension_leaf(&self,rel: &ExtensionLeafRel,@@ -520,3 +563,79 @@ impl SubstraitConsumer for DefaultSubstraitConsumer<'_> {Ok(LogicalPlan::Extension(Extension { node: plan }))}}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::logical_plan::consumer::utils::tests::test_consumer;+ use datafusion::arrow::datatypes::{DataType, Field, Schema};++ fn make_schema(fields: &[(&str, DataType)]) -> Arc<DFSchema> {+ let arrow_fields: Vec<Field> = fields+ .iter()+ .map(|(name, dt)| Field::new(*name, dt.clone(), true))+ .collect();+ Arc::new(+ DFSchema::try_from(Schema::new(arrow_fields))+ .expect("failed to create schema"),+ )+ }++ #[test]+ fn test_get_outer_schema_empty_stack() {+ let consumer = test_consumer();++ // No schemas pushed — any steps_out should return None+ assert!(consumer.get_outer_schema(0).is_none());+ assert!(consumer.get_outer_schema(1).is_none());+ assert!(consumer.get_outer_schema(2).is_none());+ }++ #[test]+ fn test_get_outer_schema_single_level() {+ let consumer = test_consumer();++ let schema_a = make_schema(&[("a", DataType::Int64)]);+ consumer.push_outer_schema(Arc::clone(&schema_a));++ // steps_out=1 returns the one pushed schema+ let result = consumer.get_outer_schema(1).unwrap();+ assert_eq!(result.fields().len(), 1);+ assert_eq!(result.fields()[0].name(), "a");++ // steps_out=0 and steps_out=2 are out of range+ assert!(consumer.get_outer_schema(0).is_none());+ assert!(consumer.get_outer_schema(2).is_none());++ consumer.pop_outer_schema();+ assert!(consumer.get_outer_schema(1).is_none());+ }++ #[test]+ fn test_get_outer_schema_nested() {+ let consumer = test_consumer();++ let schema_a = make_schema(&[("a", DataType::Int64)]);+ let schema_b = make_schema(&[("b", DataType::Utf8)]);++ consumer.push_outer_schema(Arc::clone(&schema_a));+ consumer.push_outer_schema(Arc::clone(&schema_b));++ // steps_out=1 returns the most recent (schema_b)+ let result = consumer.get_outer_schema(1).unwrap();+ assert_eq!(result.fields()[0].name(), "b");++ // steps_out=2 returns the grandparent (schema_a)+ let result = consumer.get_outer_schema(2).unwrap();+ assert_eq!(result.fields()[0].name(), "a");++ // steps_out=3 exceeds depth+ assert!(consumer.get_outer_schema(3).is_none());++ // Pop one level — now steps_out=1 returns schema_a+ consumer.pop_outer_schema();+ let result = consumer.get_outer_schema(1).unwrap();+ assert_eq!(result.fields()[0].name(), "a");+ assert!(consumer.get_outer_schema(2).is_none());+ }+}diff --git a/datafusion/substrait/tests/cases/consumer_integration.rs b/datafusion/substrait/tests/cases/consumer_integration.rsindex 149317248e..88c4eb6678 100644--- a/datafusion/substrait/tests/cases/consumer_integration.rs+++ b/datafusion/substrait/tests/cases/consumer_integration.rs@@ -77,7 +77,7 @@ mod tests {Subquery:Aggregate: groupBy=[[]], aggr=[[min(PARTSUPP.PS_SUPPLYCOST)]]Projection: PARTSUPP.PS_SUPPLYCOST- Filter: PARTSUPP.PS_PARTKEY = PARTSUPP.PS_PARTKEY AND SUPPLIER.S_SUPPKEY = PARTSUPP.PS_SUPPKEY AND SUPPLIER.S_NATIONKEY = NATION.N_NATIONKEY AND NATION.N_REGIONKEY = REGION.R_REGIONKEY AND REGION.R_NAME = Utf8("EUROPE")+ Filter: outer_ref(PART.P_PARTKEY) = PARTSUPP.PS_PARTKEY AND SUPPLIER.S_SUPPKEY = PARTSUPP.PS_SUPPKEY AND SUPPLIER.S_NATIONKEY = NATION.N_NATIONKEY AND NATION.N_REGIONKEY = REGION.R_REGIONKEY AND REGION.R_NAME = Utf8("EUROPE")Cross Join:Cross Join:Cross Join:@@ -134,7 +134,7 @@ mod tests {Projection: ORDERS.O_ORDERPRIORITYFilter: ORDERS.O_ORDERDATE >= CAST(Utf8("1993-07-01") AS Date32) AND ORDERS.O_ORDERDATE < CAST(Utf8("1993-10-01") AS Date32) AND EXISTS (<subquery>)Subquery:- Filter: LINEITEM.L_ORDERKEY = LINEITEM.L_ORDERKEY AND LINEITEM.L_COMMITDATE < LINEITEM.L_RECEIPTDATE+ Filter: LINEITEM.L_ORDERKEY = outer_ref(ORDERS.O_ORDERKEY) AND LINEITEM.L_COMMITDATE < LINEITEM.L_RECEIPTDATETableScan: LINEITEMTableScan: ORDERS"#@@ -353,11 +353,27 @@ mod tests {Ok(())}- #[ignore]#[tokio::test]async fn tpch_test_17() -> Result<()> {let plan_str = tpch_plan_to_string(17).await?;- assert_snapshot!(plan_str, "panics due to out of bounds field access");+ assert_snapshot!(+ plan_str,+ @r#"+ Projection: sum(LINEITEM.L_EXTENDEDPRICE) / Decimal128(Some(70),2,1) AS AVG_YEARLY+ Aggregate: groupBy=[[]], aggr=[[sum(LINEITEM.L_EXTENDEDPRICE)]]+ Projection: LINEITEM.L_EXTENDEDPRICE+ Filter: PART.P_PARTKEY = LINEITEM.L_PARTKEY AND PART.P_BRAND = Utf8("Brand#23") AND PART.P_CONTAINER = Utf8("MED BOX") AND LINEITEM.L_QUANTITY < (<subquery>)+ Subquery:+ Projection: Decimal128(Some(2),2,1) * avg(LINEITEM.L_QUANTITY)+ Aggregate: groupBy=[[]], aggr=[[avg(LINEITEM.L_QUANTITY)]]+ Projection: LINEITEM.L_QUANTITY+ Filter: LINEITEM.L_PARTKEY = outer_ref(PART.P_PARTKEY)+ TableScan: LINEITEM+ Cross Join:+ TableScan: LINEITEM+ TableScan: PART+ "#+ );Ok(())}@@ -425,7 +441,7 @@ mod tests {Projection: Decimal128(Some(5),2,1) * sum(LINEITEM.L_QUANTITY)Aggregate: groupBy=[[]], aggr=[[sum(LINEITEM.L_QUANTITY)]]Projection: LINEITEM.L_QUANTITY- Filter: LINEITEM.L_PARTKEY = LINEITEM.L_ORDERKEY AND LINEITEM.L_SUPPKEY = LINEITEM.L_PARTKEY AND LINEITEM.L_SHIPDATE >= CAST(Utf8("1994-01-01") AS Date32) AND LINEITEM.L_SHIPDATE < CAST(Utf8("1995-01-01") AS Date32)+ Filter: LINEITEM.L_PARTKEY = outer_ref(PARTSUPP.PS_PARTKEY) AND LINEITEM.L_SUPPKEY = outer_ref(PARTSUPP.PS_SUPPKEY) AND LINEITEM.L_SHIPDATE >= CAST(Utf8("1994-01-01") AS Date32) AND LINEITEM.L_SHIPDATE < CAST(Utf8("1995-01-01") AS Date32)TableScan: LINEITEMTableScan: PARTSUPPCross Join:@@ -449,10 +465,10 @@ mod tests {Projection: SUPPLIER.S_NAMEFilter: SUPPLIER.S_SUPPKEY = LINEITEM.L_SUPPKEY AND ORDERS.O_ORDERKEY = LINEITEM.L_ORDERKEY AND ORDERS.O_ORDERSTATUS = Utf8("F") AND LINEITEM.L_RECEIPTDATE > LINEITEM.L_COMMITDATE AND EXISTS (<subquery>) AND NOT EXISTS (<subquery>) AND SUPPLIER.S_NATIONKEY = NATION.N_NATIONKEY AND NATION.N_NAME = Utf8("SAUDI ARABIA")Subquery:- Filter: LINEITEM.L_ORDERKEY = LINEITEM.L_TAX AND LINEITEM.L_SUPPKEY != LINEITEM.L_LINESTATUS+ Filter: LINEITEM.L_ORDERKEY = outer_ref(LINEITEM.L_ORDERKEY) AND LINEITEM.L_SUPPKEY != outer_ref(LINEITEM.L_SUPPKEY)TableScan: LINEITEMSubquery:- Filter: LINEITEM.L_ORDERKEY = LINEITEM.L_TAX AND LINEITEM.L_SUPPKEY != LINEITEM.L_LINESTATUS AND LINEITEM.L_RECEIPTDATE > LINEITEM.L_COMMITDATE+ Filter: LINEITEM.L_ORDERKEY = outer_ref(LINEITEM.L_ORDERKEY) AND LINEITEM.L_SUPPKEY != outer_ref(LINEITEM.L_SUPPKEY) AND LINEITEM.L_RECEIPTDATE > LINEITEM.L_COMMITDATETableScan: LINEITEMCross Join:Cross Join:@@ -483,7 +499,7 @@ mod tests {Filter: CUSTOMER.C_ACCTBAL > Decimal128(Some(0),3,2) AND (substr(CUSTOMER.C_PHONE, Int32(1), Int32(2)) = CAST(Utf8("13") AS Utf8) OR substr(CUSTOMER.C_PHONE, Int32(1), Int32(2)) = CAST(Utf8("31") AS Utf8) OR substr(CUSTOMER.C_PHONE, Int32(1), Int32(2)) = CAST(Utf8("23") AS Utf8) OR substr(CUSTOMER.C_PHONE, Int32(1), Int32(2)) = CAST(Utf8("29") AS Utf8) OR substr(CUSTOMER.C_PHONE, Int32(1), Int32(2)) = CAST(Utf8("30") AS Utf8) OR substr(CUSTOMER.C_PHONE, Int32(1), Int32(2)) = CAST(Utf8("18") AS Utf8) OR substr(CUSTOMER.C_PHONE, Int32(1), Int32(2)) = CAST(Utf8("17") AS Utf8))TableScan: CUSTOMERSubquery:- Filter: ORDERS.O_CUSTKEY = ORDERS.O_ORDERKEY+ Filter: ORDERS.O_CUSTKEY = outer_ref(CUSTOMER.C_CUSTKEY)TableScan: ORDERSTableScan: CUSTOMER"#@@ -491,6 +507,52 @@ mod tests {Ok(())}+ /// Tests nested correlated subqueries where the innermost subquery+ /// references the outermost query (steps_out=2).+ ///+ /// This tests the outer schema stack with depth > 1.+ /// The plan represents:+ /// ```sql+ /// SELECT * FROM A+ /// WHERE EXISTS (+ /// SELECT * FROM B+ /// WHERE B.b1 = A.a1 -- steps_out=1 (references immediate parent)+ /// AND EXISTS (+ /// SELECT * FROM C+ /// WHERE C.c1 = A.a1 -- steps_out=2 (references grandparent)+ /// AND C.c2 = B.b2 -- steps_out=1 (references immediate parent)+ /// )+ /// )+ /// ```+ ///+ #[tokio::test]+ async fn test_nested_correlated_subquery() -> Result<()> {+ let path = "tests/testdata/test_plans/nested_correlated_subquery.substrait.json";+ let proto = serde_json::from_reader::<_, Plan>(BufReader::new(+ File::open(path).expect("file not found"),+ ))+ .expect("failed to parse json");++ let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto)?;+ let plan = from_substrait_plan(&ctx.state(), &proto).await?;+ let plan_str = format!("{plan}");++ assert_snapshot!(+ plan_str,+ @r#"+ Filter: EXISTS (<subquery>)+ Subquery:+ Filter: B.b1 = outer_ref(A.a1) AND EXISTS (<subquery>)+ Subquery:+ Filter: C.c1 = outer_ref(A.a1) AND C.c2 = outer_ref(B.b2)+ TableScan: C+ TableScan: B+ TableScan: A+ "#+ );+ Ok(())+ }+async fn test_plan_to_string(name: &str) -> Result<String> {let path = format!("tests/testdata/test_plans/{name}");let proto = serde_json::from_reader::<_, Plan>(BufReader::new(diff --git a/datafusion/substrait/tests/testdata/test_plans/nested_correlated_subquery.substrait.json b/datafusion/substrait/tests/testdata/test_plans/nested_correlated_subquery.substrait.jsonnew file mode 100644index 0000000000..6c565a0f94--- /dev/null+++ b/datafusion/substrait/tests/testdata/test_plans/nested_correlated_subquery.substrait.json@@ -0,0 +1,265 @@+{+ "extensionUris": [{+ "extensionUriAnchor": 1,+ "uri": "/functions_boolean.yaml"+ }, {+ "extensionUriAnchor": 2,+ "uri": "/functions_comparison.yaml"+ }],+ "extensions": [{+ "extensionFunction": {+ "extensionUriReference": 1,+ "name": "and:bool"+ }+ }, {+ "extensionFunction": {+ "extensionUriReference": 2,+ "functionAnchor": 1,+ "name": "equal:any_any"+ }+ }],+ "relations": [{+ "root": {+ "input": {+ "filter": {+ "common": {+ "direct": {}+ },+ "input": {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": ["a1", "a2"],+ "struct": {+ "types": [{+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ }, {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ }],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": ["A"]+ }+ }+ },+ "condition": {+ "subquery": {+ "setPredicate": {+ "predicateOp": "PREDICATE_OP_EXISTS",+ "tuples": {+ "filter": {+ "common": {+ "direct": {}+ },+ "input": {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": ["b1", "b2"],+ "struct": {+ "types": [{+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ }, {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ }],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": ["B"]+ }+ }+ },+ "condition": {+ "scalarFunction": {+ "outputType": {+ "bool": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "arguments": [{+ "value": {+ "scalarFunction": {+ "functionReference": 1,+ "outputType": {+ "bool": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "arguments": [{+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 0+ }+ },+ "rootReference": {}+ }+ }+ }, {+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 0+ }+ },+ "outerReference": {+ "stepsOut": 1+ }+ }+ }+ }]+ }+ }+ }, {+ "value": {+ "subquery": {+ "setPredicate": {+ "predicateOp": "PREDICATE_OP_EXISTS",+ "tuples": {+ "filter": {+ "common": {+ "direct": {}+ },+ "input": {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": ["c1", "c2"],+ "struct": {+ "types": [{+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ }, {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ }],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": ["C"]+ }+ }+ },+ "condition": {+ "scalarFunction": {+ "outputType": {+ "bool": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "arguments": [{+ "value": {+ "scalarFunction": {+ "functionReference": 1,+ "outputType": {+ "bool": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "arguments": [{+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 0+ }+ },+ "rootReference": {}+ }+ }+ }, {+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 0+ }+ },+ "outerReference": {+ "stepsOut": 2+ }+ }+ }+ }]+ }+ }+ }, {+ "value": {+ "scalarFunction": {+ "functionReference": 1,+ "outputType": {+ "bool": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "arguments": [{+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 1+ }+ },+ "rootReference": {}+ }+ }+ }, {+ "value": {+ "selection": {+ "directReference": {+ "structField": {+ "field": 1+ }+ },+ "outerReference": {+ "stepsOut": 1+ }+ }+ }+ }]+ }+ }+ }]+ }+ }+ }+ }+ }+ }+ }+ }]+ }+ }+ }+ }+ }+ }+ }+ }+ },+ "names": ["a1", "a2"]+ }+ }]+}