Skip to content

Apache DataFusion SQL Query Engine

feat: switch VirtualTable producer to use expressions field instead of deprecated values (#23672)

eliot14809051efdb87cd
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
index 832110e111..78951a3aff 100644
--- a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
@@ -148,19 +148,48 @@ pub async fn from_read_rel(
let values = if !vt.expressions.is_empty() {
let mut exprs = vec![];
for row in &vt.expressions {
+ if row.fields.len() != substrait_schema.fields().len() {
+ return substrait_err!(
+ "Field count mismatch: expected {} fields but found {} in virtual table row",
+ substrait_schema.fields().len(),
+ row.fields.len()
+ );
+ }
+
let mut row_exprs = vec![];
+ let mut name_idx = 0;
for expression in &row.fields {
- let expr = consumer
- .consume_expression(expression, &substrait_schema)
- .await?;
+ // Top-level names are provided through schema
+ // Each expression consumes at least one name, and Literals may consume additional names.
+ name_idx += 1;
+ let expr = match expression.rex_type.as_ref() {
+ Some(substrait::proto::expression::RexType::Literal(lit)) => {
+ // Values literals need 'named_struct.names' so nested struct fields keep their names from the ReadRel base schema.
+ // This is important for nested struct fields to retain their names.
+ Expr::Literal(
+ from_substrait_literal(
+ consumer,
+ lit,
+ &named_struct.names,
+ &mut name_idx,
+ )?,
+ None,
+ )
+ }
+ _ => {
+ consumer
+ .consume_expression(expression, &substrait_schema)
+ .await?
+ }
+ };
row_exprs.push(expr);
}
- // For expressions, validate against top-level schema fields, not nested names
- if row_exprs.len() != substrait_schema.fields().len() {
+
+ if name_idx != named_struct.names.len() {
return substrait_err!(
- "Field count mismatch: expected {} fields but found {} in virtual table row",
- substrait_schema.fields().len(),
- row_exprs.len()
+ "Names list must match exactly to nested schema, but found {} uses for {} names",
+ name_idx,
+ named_struct.names.len()
);
}
exprs.push(row_exprs);
diff --git a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
index 8dfbb36d37..900273bf8e 100644
--- a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
+++ b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
@@ -15,55 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-use crate::logical_plan::producer::{
- SubstraitProducer, to_substrait_literal, to_substrait_named_struct,
-};
+use crate::logical_plan::producer::{SubstraitProducer, to_substrait_named_struct};
use datafusion::common::{DFSchema, ToDFSchema, substrait_datafusion_err};
use datafusion::logical_expr::utils::conjunction;
use datafusion::logical_expr::{EmptyRelation, Expr, TableScan, Values};
use datafusion::scalar::ScalarValue;
use std::sync::Arc;
use substrait::proto::expression::MaskExpression;
-use substrait::proto::expression::literal::Struct as LiteralStruct;
use substrait::proto::expression::mask_expression::{StructItem, StructSelect};
use substrait::proto::expression::nested::Struct as NestedStruct;
use substrait::proto::read_rel::{NamedTable, ReadType, VirtualTable};
use substrait::proto::rel::RelType;
use substrait::proto::{ReadRel, Rel};
-/// Converts rows of literal expressions into Substrait literal structs.
-///
-/// Each row is expected to contain only `Expr::Literal` or `Expr::Alias` wrapping literals.
-/// Aliases are unwrapped and the underlying literal is converted.
-fn convert_literal_rows(
- producer: &mut impl SubstraitProducer,
- rows: &[Vec<Expr>],
-) -> datafusion::common::Result<Vec<LiteralStruct>> {
- rows.iter()
- .map(|row| {
- let fields = row
- .iter()
- .map(|expr| match expr {
- Expr::Literal(sv, _) => to_substrait_literal(producer, sv),
- Expr::Alias(alias) => match alias.expr.as_ref() {
- // The schema gives us the names, so we can skip aliases
- Expr::Literal(sv, _) => to_substrait_literal(producer, sv),
- _ => Err(substrait_datafusion_err!(
- "Only literal types can be aliased in Virtual Tables, got: {}",
- alias.expr.variant_name()
- )),
- },
- _ => Err(substrait_datafusion_err!(
- "Only literal types and aliases are supported in Virtual Tables, got: {}",
- expr.variant_name()
- )),
- })
- .collect::<datafusion::common::Result<_>>()?;
- Ok(LiteralStruct { fields })
- })
- .collect()
-}
-
/// Converts rows of arbitrary expressions into Substrait nested structs.
///
/// Validates that each row has the expected schema length and converts each expression
@@ -163,6 +127,7 @@ pub fn from_empty_relation(
let base_schema = to_substrait_named_struct(producer, &e.schema)?;
let read_type = if e.produce_one_row {
+ let empty_schema = Arc::new(DFSchema::empty());
// Create one row with default scalar values for each field in the schema.
// For example, an Int32 field gets Int32(NULL), a Utf8 field gets Utf8(NULL), etc.
// This represents the "phantom row" that provides a context for evaluating
@@ -173,25 +138,16 @@ pub fn from_empty_relation(
.iter()
.map(|f| {
let scalar = ScalarValue::try_from(f.data_type())?;
- to_substrait_literal(producer, &scalar)
+ producer.handle_expr(&Expr::Literal(scalar, None), &empty_schema)
})
.collect::<datafusion::common::Result<_>>()?;
ReadType::VirtualTable(VirtualTable {
- // Use deprecated 'values' field instead of 'expressions' because the consumer's
- // nested expression support (RexType::Nested) is not yet implemented.
- // The 'values' field uses literal::Struct which the consumer can properly
- // deserialize with field name preservation.
- #[expect(deprecated)]
- values: vec![LiteralStruct { fields }],
- expressions: vec![],
+ expressions: vec![NestedStruct { fields }],
+ ..Default::default()
})
} else {
- ReadType::VirtualTable(VirtualTable {
- #[expect(deprecated)]
- values: vec![],
- expressions: vec![],
- })
+ ReadType::VirtualTable(VirtualTable::default())
};
Ok(Box::new(Rel {
rel_type: Some(RelType::Read(Box::new(ReadRel {
@@ -212,23 +168,8 @@ pub fn from_values(
) -> datafusion::common::Result<Box<Rel>> {
let schema_len = v.schema.fields().len();
let empty_schema = Arc::new(DFSchema::empty());
-
- let use_literals = v.values.iter().all(|row| {
- row.iter().all(|expr| match expr {
- Expr::Literal(_, _) => true,
- Expr::Alias(alias) => matches!(alias.expr.as_ref(), Expr::Literal(_, _)),
- _ => false,
- })
- });
-
- let (values, expressions) = if use_literals {
- let values = convert_literal_rows(producer, &v.values)?;
- (values, vec![])
- } else {
- let expressions =
- convert_expression_rows(producer, &v.values, schema_len, &empty_schema)?;
- (vec![], expressions)
- };
+ let expressions =
+ convert_expression_rows(producer, &v.values, schema_len, &empty_schema)?;
Ok(Box::new(Rel {
rel_type: Some(RelType::Read(Box::new(ReadRel {
common: None,
@@ -237,10 +178,9 @@ pub fn from_values(
best_effort_filter: None,
projection: None,
advanced_extension: None,
- #[expect(deprecated)]
read_type: Some(ReadType::VirtualTable(VirtualTable {
- values,
expressions,
+ ..Default::default()
})),
}))),
}))
diff --git a/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json b/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json
index 73fa06eea5..8a81a9a0c7 100644
--- a/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json
+++ b/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json
@@ -100,29 +100,52 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "string": "aaa",
- "nullable": true
- }, {
- "string": "host-a",
- "nullable": true
- }, {
- "i64": "128",
- "nullable": true
- }]
- }, {
- "fields": [{
- "string": "bbb",
- "nullable": true
- }, {
- "string": "host-b",
- "nullable": true
- }, {
- "i64": "256",
- "nullable": true
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "aaa",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "host-a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "128",
+ "nullable": true
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "bbb",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "host-b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "256",
+ "nullable": true
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -293,23 +316,40 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "string": "host-a",
- "nullable": true
- }, {
- "i64": "107",
- "nullable": true
- }]
- }, {
- "fields": [{
- "string": "host-b",
- "nullable": true
- }, {
- "i64": "214",
- "nullable": true
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "host-a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "107",
+ "nullable": true
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "host-b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "214",
+ "nullable": true
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -365,29 +405,52 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "string": "aaa",
- "nullable": true
- }, {
- "string": "host-a",
- "nullable": true
- }, {
- "i64": "128",
- "nullable": true
- }]
- }, {
- "fields": [{
- "string": "bbb",
- "nullable": true
- }, {
- "string": "host-b",
- "nullable": true
- }, {
- "i64": "256",
- "nullable": true
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "aaa",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "host-a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "128",
+ "nullable": true
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "bbb",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "host-b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "i64": "256",
+ "nullable": true
+ }
+ }
+ ]
+ }
+ ]
}
}
},
diff --git a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json
index 642256c562..13c1e5899d 100644
--- a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json
+++ b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json
@@ -24,13 +24,147 @@
}
},
"virtualTable": {
- "values": [
- { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },
- { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },
- { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] },
- { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] },
- { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },
- { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "1",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c1",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "2",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c2",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "3",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c3",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "4",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c4",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "5",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "e",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c5",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "6",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "f",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c6",
+ "nullable": false
+ }
+ }
+ ]
+ }
]
}
}
@@ -50,13 +184,147 @@
}
},
"virtualTable": {
- "values": [
- { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },
- { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },
- { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] },
- { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] },
- { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },
- { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "1",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c1",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "2",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c2",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "3",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c3",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "4",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c4",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "5",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "e",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c5",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "6",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "f",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c6",
+ "nullable": false
+ }
+ }
+ ]
+ }
]
}
}
diff --git a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json
index f16672947e..481bba44d8 100644
--- a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json
+++ b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json
@@ -24,13 +24,147 @@
}
},
"virtualTable": {
- "values": [
- { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },
- { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },
- { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] },
- { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] },
- { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },
- { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "1",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c1",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "2",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c2",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "3",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c3",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "4",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c4",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "5",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "e",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c5",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "6",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "f",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c6",
+ "nullable": false
+ }
+ }
+ ]
+ }
]
}
}
@@ -50,13 +184,147 @@
}
},
"virtualTable": {
- "values": [
- { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },
- { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },
- { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] },
- { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] },
- { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },
- { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "1",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "a",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c1",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "2",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "b",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c2",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "3",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c3",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "4",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "null": {
+ "string": {
+ "nullability": "NULLABILITY_NULLABLE"
+ }
+ },
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c4",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "5",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "e",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c5",
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "string": "6",
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "string": "f",
+ "nullable": true
+ }
+ },
+ {
+ "literal": {
+ "string": "c6",
+ "nullable": false
+ }
+ }
+ ]
+ }
]
}
}
diff --git a/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json b/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json
index e88cce648d..15c0313b43 100644
--- a/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json
+++ b/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json
@@ -72,19 +72,30 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i64": "1",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }, {
- "fields": [{
- "i64": "2",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "1",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "2",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -153,27 +164,44 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i64": "1",
- "nullable": true,
- "typeVariationReference": 0
- }, {
- "string": "info",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }, {
- "fields": [{
- "i64": "2",
- "nullable": true,
- "typeVariationReference": 0
- }, {
- "string": "low",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "1",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ },
+ {
+ "literal": {
+ "string": "info",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "2",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ },
+ {
+ "literal": {
+ "string": "low",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -272,19 +300,30 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i64": "1",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }, {
- "fields": [{
- "i64": "2",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "1",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "2",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ }
+ ]
}
}
},
@@ -389,19 +428,30 @@
}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i64": "1",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }, {
- "fields": [{
- "i64": "2",
- "nullable": true,
- "typeVariationReference": 0
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "1",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i64": "2",
+ "nullable": true,
+ "typeVariationReference": 0
+ }
+ }
+ ]
+ }
+ ]
}
}
},
diff --git a/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json b/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json
index e1c5574f8b..e29c000ee6 100644
--- a/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json
+++ b/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json
@@ -34,26 +34,28 @@
}
},
"virtualTable": {
- "values": [
+ "expressions": [
{
"fields": [
{
- "list": {
- "values": [
- {
- "i32": 1,
- "nullable": false,
- "typeVariationReference": 0
- },
- {
- "i32": 2,
- "nullable": false,
- "typeVariationReference": 0
- }
- ]
- },
- "nullable": false,
- "typeVariationReference": 0
+ "literal": {
+ "list": {
+ "values": [
+ {
+ "i32": 1,
+ "nullable": false,
+ "typeVariationReference": 0
+ },
+ {
+ "i32": 2,
+ "nullable": false,
+ "typeVariationReference": 0
+ }
+ ]
+ },
+ "nullable": false,
+ "typeVariationReference": 0
+ }
}
]
}
diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json
index eeaf5a3dd8..d5209a683d 100644
--- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json
+++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json
@@ -85,23 +85,40 @@
"direct": {}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "fp32": 1.0,
- "nullable": false
- }, {
- "fp32": 10.0,
- "nullable": false
- }]
- }, {
- "fields": [{
- "fp32": 100.0,
- "nullable": false
- }, {
- "fp32": 10.0,
- "nullable": false
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "fp32": 1.0,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "fp32": 10.0,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "fp32": 100.0,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "fp32": 10.0,
+ "nullable": false
+ }
+ }
+ ]
+ }
+ ]
}
}
}
diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json
index 6749a301b1..f609d26138 100644
--- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json
+++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json
@@ -106,29 +106,52 @@
"direct": {}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "i8": 2,
- "nullable": false
- }, {
- "i8": 1,
- "nullable": false
- }, {
- "i8": 3,
- "nullable": false
- }]
- }, {
- "fields": [{
- "i8": 4,
- "nullable": false
- }, {
- "i8": 1,
- "nullable": false
- }, {
- "i8": 2,
- "nullable": false
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "i8": 2,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "i8": 1,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "i8": 3,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "i8": 4,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "i8": 1,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "i8": 2,
+ "nullable": false
+ }
+ }
+ ]
+ }
+ ]
}
}
}
diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json
index 8365b1edfe..5d91342257 100644
--- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json
+++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json
@@ -85,39 +85,72 @@
"direct": {}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "boolean": true,
- "nullable": false
- }, {
- "boolean": true,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": true,
- "nullable": false
- }, {
- "boolean": false,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": false,
- "nullable": false
- }, {
- "boolean": true,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": false,
- "nullable": false
- }, {
- "boolean": false,
- "nullable": false
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ }
+ ]
+ }
+ ]
}
}
}
diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json
index cfd760de89..2514c0afc9 100644
--- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json
+++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json
@@ -85,39 +85,72 @@
"direct": {}
},
"virtualTable": {
- "values": [{
- "fields": [{
- "boolean": true,
- "nullable": false
- }, {
- "boolean": true,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": true,
- "nullable": false
- }, {
- "boolean": false,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": false,
- "nullable": false
- }, {
- "boolean": true,
- "nullable": false
- }]
- }, {
- "fields": [{
- "boolean": false,
- "nullable": false
- }, {
- "boolean": false,
- "nullable": false
- }]
- }]
+ "expressions": [
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": true,
+ "nullable": false
+ }
+ }
+ ]
+ },
+ {
+ "fields": [
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ },
+ {
+ "literal": {
+ "boolean": false,
+ "nullable": false
+ }
+ }
+ ]
+ }
+ ]
}
}
}
diff --git a/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json b/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json
index e9f6795880..b0d4ba4813 100644
--- a/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json
+++ b/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json
@@ -43,12 +43,14 @@
}
},
"virtualTable": {
- "values": [
+ "expressions": [
{
"fields": [
{
- "i64": "0",
- "nullable": false
+ "literal": {
+ "i64": "0",
+ "nullable": false
+ }
}
]
}