Apache DataFusion SQL Query Engine
fix: Derive Substrait intersection nullability from every input (#25091)
namanjain24-sudod040501c2e6f
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rsindex 36bf8dbae4..10e78a98ee 100644--- a/datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rs+++ b/datafusion/substrait/src/logical_plan/consumer/rel/set_rel.rs@@ -16,8 +16,12 @@// under the License.use crate::logical_plan::consumer::SubstraitConsumer;-use datafusion::common::{not_impl_err, substrait_err};-use datafusion::logical_expr::{LogicalPlan, LogicalPlanBuilder};+use datafusion::arrow::datatypes::Field;+use datafusion::common::{JoinType, NullEquality, not_impl_err, substrait_err};+use datafusion::logical_expr::{+ Cast, Expr, LogicalPlan, LogicalPlanBuilder, requalify_sides_if_needed,+};+use std::sync::Arc;use substrait::proto::set_rel::SetOp;use substrait::proto::{Rel, SetRel};@@ -31,7 +35,7 @@ pub async fn from_set_rel(match set.op() {SetOp::UnionAll => union_rels(consumer, &set.inputs, true).await,SetOp::UnionDistinct => union_rels(consumer, &set.inputs, false).await,- SetOp::IntersectionPrimary => LogicalPlanBuilder::intersect(+ SetOp::IntersectionPrimary => intersect_rel(consumer.consume_rel(&set.inputs[0]).await?,union_rels(consumer, &set.inputs[1..], true).await?,false,@@ -77,16 +81,137 @@ async fn intersect_rels(let mut rel = consumer.consume_rel(&rels[0]).await?;for input in &rels[1..] {- rel = LogicalPlanBuilder::intersect(- rel,- consumer.consume_rel(input).await?,- is_all,- )?;+ rel = intersect_rel(rel, consumer.consume_rel(input).await?, is_all)?;}Ok(rel)}+/// Intersects two relations, giving the result the nullability the Substrait+/// [Set Operation rules] prescribe.+///+/// [`LogicalPlanBuilder::intersect`] compiles an intersection into a left semi+/// join, so on its own the result keeps the left input's nullability. The join+/// matches nulls with nulls, so a left row holding a null in some field only+/// survives when the right input holds a null there too. A field is therefore+/// nullable in the result only when it is nullable in *both* inputs.+///+/// Applied to each step of a chain, that gives the spec's rule for the multiset+/// intersections - a field is required when any input requires it. For+/// `INTERSECTION_PRIMARY` the right side is the union of the secondary inputs,+/// whose field is nullable exactly when some secondary input makes it nullable,+/// so the same rule yields "nullable in the primary input and in at least one+/// secondary input".+///+/// When the right input requires a field the left input leaves nullable, the+/// intersection is built as an inner join against the distinct right rows+/// instead, and that field is read from the right side. Matched rows hold equal+/// values, so the result is unchanged, and the field is non-nullable because+/// its source is: the logical and the physical planner both derive that from+/// the input schema, so the plan, the physical plan and the batches agree.+/// Joining against distinct right rows keeps each left row at most once, as the+/// semi join does.+///+/// Differing metadata does not change which path is taken. A column read from+/// the right is cast to an explicit target field carrying the left field's+/// metadata; an explicit-field cast target's metadata is used exactly as+/// given rather than merged into the source's, so this drops a key only the+/// right field carried and keeps the left field's metadata exactly. A+/// `Field`'s own metadata travels with it through any later rewrite, so this+/// holds for the plan this function returns and for any rewrite of it.+///+/// Schema-level (as opposed to per-field) metadata gets no such treatment:+/// [`build_join_schema`](datafusion::logical_expr::build_join_schema) merges+/// the two inputs' schema metadata with the left's value winning a+/// conflicting key, and that merged map is what the projection built on top+/// of the join reports - the same as the semi join+/// [`LogicalPlanBuilder::intersect`] takes for the cases this function+/// doesn't rewrite. A key only the right input's schema carries can+/// therefore still appear in the result's schema-level metadata.+///+/// [Set Operation rules]: https://substrait.io/relations/logical_relations/#set-operation+fn intersect_rel(+ left: LogicalPlan,+ right: LogicalPlan,+ is_all: bool,+) -> datafusion::common::Result<LogicalPlan> {+ let left_fields = left.schema().fields();+ let right_fields = right.schema().fields();+ // A field is read from the right side when the left leaves it nullable and+ // the right requires it. Its metadata does not matter here: it is read from+ // the right with the left field's metadata substituted for its own.+ let from_right: Vec<bool> = left_fields+ .iter()+ .zip(right_fields.iter())+ .map(|(left, right)| {+ left.is_nullable()+ && !right.is_nullable()+ && left.data_type() == right.data_type()+ })+ .collect();++ // `intersect` also reports inputs of different widths.+ if left_fields.len() != right_fields.len() || !from_right.contains(&true) {+ return LogicalPlanBuilder::intersect(left, right, is_all);+ }++ let (left, right, _) = requalify_sides_if_needed(+ LogicalPlanBuilder::from(left),+ LogicalPlanBuilder::from(right),+ )?;+ let left = if is_all { left } else { left.distinct()? };+ let right = right.distinct()?.build()?;++ let left_columns = left.schema().columns();+ let right_columns = right.schema().columns();+ let exprs = left+ .schema()+ .fields()+ .iter()+ .zip(&left_columns)+ .zip(&right_columns)+ .zip(&from_right)+ .map(|(((field, left), right), from_right)| {+ if *from_right {+ // `alias_qualified_with_metadata` would not do: `Expr::Alias`'s+ // field derivation extends the aliased expression's own+ // metadata with the alias's, so a key only the right column+ // carries would survive alongside the left field's metadata.+ // An explicit-field `Cast` target's metadata is instead used+ // exactly as given, in both the logical and the physical+ // plan, so casting to the left field's type (already proven+ // equal to the right's) and metadata drops the right's own+ // metadata outright. The qualifier and name still need+ // `alias_qualified` on top, since a `Cast`'s own field is not+ // renamed to its target field's name.+ let target_field = Arc::new(+ Field::new(&left.name, field.data_type().clone(), false)+ .with_metadata(field.metadata().clone()),+ );+ Expr::Cast(Cast::new_from_field(+ Box::new(Expr::Column(right.clone())),+ target_field,+ ))+ .alias_qualified(left.relation.clone(), &left.name)+ } else {+ Expr::Column(left.clone())+ }+ })+ .collect::<Vec<_>>();++ let joined = left+ .join_detailed(+ right,+ JoinType::Inner,+ (left_columns, right_columns),+ None,+ NullEquality::NullEqualsNull,+ )?+ .build()?;++ LogicalPlanBuilder::from(joined).project(exprs)?.build()+}+async fn except_rels(consumer: &impl SubstraitConsumer,rels: &[Rel],diff --git a/datafusion/substrait/tests/cases/logical_plans.rs b/datafusion/substrait/tests/cases/logical_plans.rsindex d1c930d297..9d89067125 100644--- a/datafusion/substrait/tests/cases/logical_plans.rs+++ b/datafusion/substrait/tests/cases/logical_plans.rs@@ -21,8 +21,13 @@mod tests {use crate::cases::roundtrip_logical_plan::higher_order_function_ctx;use crate::utils::test::{add_plan_schemas_to_ctx, read_json};+ use datafusion::arrow::array::{ArrayRef, Int64Array, RecordBatch};+ use datafusion::arrow::datatypes::{Field, Schema, SchemaRef};+ use datafusion::assert_batches_sorted_eq;use datafusion::common::test_util::format_batches;- use std::collections::HashSet;+ use datafusion::datasource::MemTable;+ use std::collections::{HashMap, HashSet};+ use std::sync::Arc;use datafusion::common::Result;use datafusion::dataframe::DataFrame;@@ -229,6 +234,228 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn intersect_nullability() -> Result<()> {+ // Substrait's set operation rules derive an intersection's nullability from+ // every input, not only the primary one. Each plan below intersects three+ // tables carrying the same six columns, with these nullabilities (`?` marks+ // a nullable column, `~` a column with unspecified nullability, which the+ // consumer reads as nullable):+ //+ // primary a? b? c? d? e? f?+ // secondary a b c? d? e~ f~+ // secondary a b? c d? e? f+ let rows: [(&str, &[[Option<i64>; 6]]); 3] = [+ (+ "data",+ &[+ [Some(1), Some(1), Some(1), None, None, Some(1)],+ [Some(2), None, Some(2), Some(2), Some(2), Some(2)],+ [Some(3), Some(3), None, Some(3), Some(3), Some(3)],+ [None, Some(4), Some(4), Some(4), Some(4), Some(4)],+ ],+ ),+ (+ "data2",+ &[+ [Some(1), Some(1), Some(1), None, None, Some(1)],+ [Some(3), Some(3), None, Some(3), Some(3), Some(3)],+ ],+ ),+ (+ "data3",+ &[+ [Some(1), Some(1), Some(1), None, None, Some(1)],+ [Some(2), None, Some(2), Some(2), Some(2), Some(2)],+ ],+ ),+ ];++ // Schema and field metadata are no part of an input's nullability, so the+ // result must not depend on whether the tables carry any. With metadata,+ // every table describes itself and the secondary tables add keys the+ // primary one lacks: the result has to keep the primary table's metadata+ // where they disagree, including on the columns read from a secondary+ // input, and the physical plan and the batches have to agree with it. The+ // secondary tables share their metadata, as `INTERSECTION_PRIMARY` unions+ // them, and a union of inputs with differing field metadata reports+ // different metadata in its logical and in its physical schema.+ let with_metadata = |schema: &SchemaRef, table: &str| -> SchemaRef {+ let role = if table == "data" { "data" } else { "secondary" };+ let tag = |mut metadata: HashMap<String, String>| {+ if role == "secondary" {+ metadata.insert("only_in_secondary".to_string(), "yes".to_string());+ }+ metadata+ };+ let fields: Vec<Field> = schema+ .fields()+ .iter()+ .map(|field| {+ let metadata = HashMap::from([(+ "column".to_string(),+ format!("{role}.{}", field.name()),+ )]);+ field.as_ref().clone().with_metadata(tag(metadata))+ })+ .collect();+ let metadata = HashMap::from([("table".to_string(), role.to_string())]);+ Arc::new(Schema::new_with_metadata(fields, tag(metadata)))+ };++ let cases = [+ // Nullable in the primary input and in at least one secondary input.+ (+ "intersect_primary_mixed_nullability",+ "a, b?, c?, d?, e?, f?",+ &[+ "+---+---+---+---+---+---+",+ "| a | b | c | d | e | f |",+ "+---+---+---+---+---+---+",+ "| 1 | 1 | 1 | | | 1 |",+ "| 2 | | 2 | 2 | 2 | 2 |",+ "| 3 | 3 | | 3 | 3 | 3 |",+ "+---+---+---+---+---+---+",+ ][..],+ ),+ // Required as soon as any input requires it.+ (+ "intersect_multiset_mixed_nullability",+ "a, b, c, d?, e?, f",+ &[+ "+---+---+---+---+---+---+",+ "| a | b | c | d | e | f |",+ "+---+---+---+---+---+---+",+ "| 1 | 1 | 1 | | | 1 |",+ "+---+---+---+---+---+---+",+ ][..],+ ),+ (+ "intersect_multiset_all_mixed_nullability",+ "a, b, c, d?, e?, f",+ &[+ "+---+---+---+---+---+---+",+ "| a | b | c | d | e | f |",+ "+---+---+---+---+---+---+",+ "| 1 | 1 | 1 | | | 1 |",+ "+---+---+---+---+---+---+",+ ][..],+ ),+ ];++ for ((file, expected_nullability, expected_rows), tagged) in cases+ .into_iter()+ .flat_map(|case| [(case, false), (case, true)])+ {+ let proto_plan =+ read_json(&format!("tests/testdata/test_plans/{file}.substrait.json"));+ let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;+ // Give each table rows, so the batch schemas below come from real batches+ for (table, rows) in rows {+ let schema = ctx.table_provider(table).await?.schema();+ let schema = if tagged {+ with_metadata(&schema, table)+ } else {+ schema+ };+ let columns = (0..schema.fields().len())+ .map(|i| {+ Arc::new(rows.iter().map(|row| row[i]).collect::<Int64Array>())+ as ArrayRef+ })+ .collect();+ let batch = RecordBatch::try_new(Arc::clone(&schema), columns)?;+ ctx.deregister_table(table)?;+ ctx.register_table(+ table,+ Arc::new(MemTable::try_new(schema, vec![vec![batch]])?),+ )?;+ }+ let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;++ let nullability = plan+ .schema()+ .fields()+ .iter()+ .map(|field| {+ format!(+ "{}{}",+ field.name(),+ if field.is_nullable() { "?" } else { "" }+ )+ })+ .collect::<Vec<_>>()+ .join(", ");+ assert_eq!(+ nullability, expected_nullability,+ "nullability of {file} (tagged: {tagged})"+ );++ if tagged {+ // Schema-level metadata is a join of both inputs' maps, per+ // `intersect_rel`'s doc comment, so a secondary-only key+ // ("only_in_secondary") can appear alongside the primary+ // table's own keys; only that a conflicting key resolves to+ // the primary table's value is asserted here. Per-field+ // metadata has no such leak and is checked exactly below,+ // and again on the optimized, physical and batch schemas.+ assert_eq!(+ plan.schema().metadata().get("table"),+ Some(&"data".to_string()),+ "schema metadata of {file}"+ );+ for field in plan.schema().fields() {+ let expected_metadata = HashMap::from([(+ "column".to_string(),+ format!("data.{}", field.name()),+ )]);+ assert_eq!(+ field.metadata(),+ &expected_metadata,+ "metadata of column {} of {file}",+ field.name()+ );+ }+ }++ // The physical plan and the batches it produces must carry the same+ // schema as the *optimized* logical plan, since physical planning+ // runs on the optimizer's output, not on `plan` as+ // `from_substrait_plan` returned it.+ let optimized = ctx.state().optimize(&plan)?;+ let logical_schema = Arc::clone(optimized.schema().inner());+ if tagged {+ for field in logical_schema.fields() {+ let expected_metadata = HashMap::from([(+ "column".to_string(),+ format!("data.{}", field.name()),+ )]);+ assert_eq!(+ field.metadata(),+ &expected_metadata,+ "optimized metadata of column {} of {file}",+ field.name()+ );+ }+ }+ let df = DataFrame::new(ctx.state(), optimized);+ let physical_plan = df.clone().create_physical_plan().await?;+ assert_eq!(+ physical_plan.schema(),+ logical_schema,+ "physical schema of {file}"+ );+ let batches = df.collect().await?;+ assert!(!batches.is_empty(), "no batches for {file}");+ for batch in &batches {+ assert_eq!(batch.schema(), logical_schema, "batch schema of {file}");+ }+ assert_batches_sorted_eq!(expected_rows, &batches);+ }++ Ok(())+ }+#[tokio::test]async fn multilayer_aggregate() -> Result<()> {let proto_plan =diff --git a/datafusion/substrait/tests/testdata/test_plans/intersect_multiset_all_mixed_nullability.substrait.json b/datafusion/substrait/tests/testdata/test_plans/intersect_multiset_all_mixed_nullability.substrait.jsonnew file mode 100644index 0000000000..a99e7dd572--- /dev/null+++ b/datafusion/substrait/tests/testdata/test_plans/intersect_multiset_all_mixed_nullability.substrait.json@@ -0,0 +1,198 @@+{+ "relations": [+ {+ "root": {+ "input": {+ "set": {+ "inputs": [+ {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": [+ "data"+ ]+ }+ }+ },+ {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_UNSPECIFIED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_UNSPECIFIED"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": [+ "data2"+ ]+ }+ }+ },+ {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": [+ "data3"+ ]+ }+ }+ }+ ],+ "op": "SET_OP_INTERSECTION_MULTISET_ALL"+ }+ },+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ]+ }+ }+ ],+ "version": {+ "minorNumber": 54,+ "producer": "datafusion-test"+ }+}diff --git a/datafusion/substrait/tests/testdata/test_plans/intersect_multiset_mixed_nullability.substrait.json b/datafusion/substrait/tests/testdata/test_plans/intersect_multiset_mixed_nullability.substrait.jsonnew file mode 100644index 0000000000..e896015c8a--- /dev/null+++ b/datafusion/substrait/tests/testdata/test_plans/intersect_multiset_mixed_nullability.substrait.json@@ -0,0 +1,198 @@+{+ "relations": [+ {+ "root": {+ "input": {+ "set": {+ "inputs": [+ {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": [+ "data"+ ]+ }+ }+ },+ {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_UNSPECIFIED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_UNSPECIFIED"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": [+ "data2"+ ]+ }+ }+ },+ {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": [+ "data3"+ ]+ }+ }+ }+ ],+ "op": "SET_OP_INTERSECTION_MULTISET"+ }+ },+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ]+ }+ }+ ],+ "version": {+ "minorNumber": 54,+ "producer": "datafusion-test"+ }+}diff --git a/datafusion/substrait/tests/testdata/test_plans/intersect_primary_mixed_nullability.substrait.json b/datafusion/substrait/tests/testdata/test_plans/intersect_primary_mixed_nullability.substrait.jsonnew file mode 100644index 0000000000..77ac9d50da--- /dev/null+++ b/datafusion/substrait/tests/testdata/test_plans/intersect_primary_mixed_nullability.substrait.json@@ -0,0 +1,198 @@+{+ "relations": [+ {+ "root": {+ "input": {+ "set": {+ "inputs": [+ {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": [+ "data"+ ]+ }+ }+ },+ {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_UNSPECIFIED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_UNSPECIFIED"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": [+ "data2"+ ]+ }+ }+ },+ {+ "read": {+ "common": {+ "direct": {}+ },+ "baseSchema": {+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ],+ "struct": {+ "types": [+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ {+ "i64": {+ "nullability": "NULLABILITY_REQUIRED"+ }+ }+ ],+ "nullability": "NULLABILITY_REQUIRED"+ }+ },+ "namedTable": {+ "names": [+ "data3"+ ]+ }+ }+ }+ ],+ "op": "SET_OP_INTERSECTION_PRIMARY"+ }+ },+ "names": [+ "a",+ "b",+ "c",+ "d",+ "e",+ "f"+ ]+ }+ }+ ],+ "version": {+ "minorNumber": 54,+ "producer": "datafusion-test"+ }+}