Apache DataFusion SQL Query Engine
feat: switch VirtualTable producer to use expressions field instead of deprecated values (#23672)
eliot14809051efdb87cd
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rsindex 832110e111..78951a3aff 100644--- a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs+++ b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs@@ -148,19 +148,48 @@ pub async fn from_read_rel(let values = if !vt.expressions.is_empty() {let mut exprs = vec![];for row in &vt.expressions {+ if row.fields.len() != substrait_schema.fields().len() {+ return substrait_err!(+ "Field count mismatch: expected {} fields but found {} in virtual table row",+ substrait_schema.fields().len(),+ row.fields.len()+ );+ }+let mut row_exprs = vec![];+ let mut name_idx = 0;for expression in &row.fields {- let expr = consumer- .consume_expression(expression, &substrait_schema)- .await?;+ // Top-level names are provided through schema+ // Each expression consumes at least one name, and Literals may consume additional names.+ name_idx += 1;+ let expr = match expression.rex_type.as_ref() {+ Some(substrait::proto::expression::RexType::Literal(lit)) => {+ // Values literals need 'named_struct.names' so nested struct fields keep their names from the ReadRel base schema.+ // This is important for nested struct fields to retain their names.+ Expr::Literal(+ from_substrait_literal(+ consumer,+ lit,+ &named_struct.names,+ &mut name_idx,+ )?,+ None,+ )+ }+ _ => {+ consumer+ .consume_expression(expression, &substrait_schema)+ .await?+ }+ };row_exprs.push(expr);}- // For expressions, validate against top-level schema fields, not nested names- if row_exprs.len() != substrait_schema.fields().len() {++ if name_idx != named_struct.names.len() {return substrait_err!(- "Field count mismatch: expected {} fields but found {} in virtual table row",- substrait_schema.fields().len(),- row_exprs.len()+ "Names list must match exactly to nested schema, but found {} uses for {} names",+ name_idx,+ named_struct.names.len());}exprs.push(row_exprs);diff --git a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rsindex 8dfbb36d37..900273bf8e 100644--- a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs+++ b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs@@ -15,55 +15,19 @@// specific language governing permissions and limitations// under the License.-use crate::logical_plan::producer::{- SubstraitProducer, to_substrait_literal, to_substrait_named_struct,-};+use crate::logical_plan::producer::{SubstraitProducer, to_substrait_named_struct};use datafusion::common::{DFSchema, ToDFSchema, substrait_datafusion_err};use datafusion::logical_expr::utils::conjunction;use datafusion::logical_expr::{EmptyRelation, Expr, TableScan, Values};use datafusion::scalar::ScalarValue;use std::sync::Arc;use substrait::proto::expression::MaskExpression;-use substrait::proto::expression::literal::Struct as LiteralStruct;use substrait::proto::expression::mask_expression::{StructItem, StructSelect};use substrait::proto::expression::nested::Struct as NestedStruct;use substrait::proto::read_rel::{NamedTable, ReadType, VirtualTable};use substrait::proto::rel::RelType;use substrait::proto::{ReadRel, Rel};-/// Converts rows of literal expressions into Substrait literal structs.-///-/// Each row is expected to contain only `Expr::Literal` or `Expr::Alias` wrapping literals.-/// Aliases are unwrapped and the underlying literal is converted.-fn convert_literal_rows(- producer: &mut impl SubstraitProducer,- rows: &[Vec<Expr>],-) -> datafusion::common::Result<Vec<LiteralStruct>> {- rows.iter()- .map(|row| {- let fields = row- .iter()- .map(|expr| match expr {- Expr::Literal(sv, _) => to_substrait_literal(producer, sv),- Expr::Alias(alias) => match alias.expr.as_ref() {- // The schema gives us the names, so we can skip aliases- Expr::Literal(sv, _) => to_substrait_literal(producer, sv),- _ => Err(substrait_datafusion_err!(- "Only literal types can be aliased in Virtual Tables, got: {}",- alias.expr.variant_name()- )),- },- _ => Err(substrait_datafusion_err!(- "Only literal types and aliases are supported in Virtual Tables, got: {}",- expr.variant_name()- )),- })- .collect::<datafusion::common::Result<_>>()?;- Ok(LiteralStruct { fields })- })- .collect()-}-/// Converts rows of arbitrary expressions into Substrait nested structs.////// Validates that each row has the expected schema length and converts each expression@@ -163,6 +127,7 @@ pub fn from_empty_relation(let base_schema = to_substrait_named_struct(producer, &e.schema)?;let read_type = if e.produce_one_row {+ let empty_schema = Arc::new(DFSchema::empty());// Create one row with default scalar values for each field in the schema.// For example, an Int32 field gets Int32(NULL), a Utf8 field gets Utf8(NULL), etc.// This represents the "phantom row" that provides a context for evaluating@@ -173,25 +138,16 @@ pub fn from_empty_relation(.iter().map(|f| {let scalar = ScalarValue::try_from(f.data_type())?;- to_substrait_literal(producer, &scalar)+ producer.handle_expr(&Expr::Literal(scalar, None), &empty_schema)}).collect::<datafusion::common::Result<_>>()?;ReadType::VirtualTable(VirtualTable {- // Use deprecated 'values' field instead of 'expressions' because the consumer's- // nested expression support (RexType::Nested) is not yet implemented.- // The 'values' field uses literal::Struct which the consumer can properly- // deserialize with field name preservation.- #[expect(deprecated)]- values: vec![LiteralStruct { fields }],- expressions: vec![],+ expressions: vec![NestedStruct { fields }],+ ..Default::default()})} else {- ReadType::VirtualTable(VirtualTable {- #[expect(deprecated)]- values: vec![],- expressions: vec![],- })+ ReadType::VirtualTable(VirtualTable::default())};Ok(Box::new(Rel {rel_type: Some(RelType::Read(Box::new(ReadRel {@@ -212,23 +168,8 @@ pub fn from_values() -> datafusion::common::Result<Box<Rel>> {let schema_len = v.schema.fields().len();let empty_schema = Arc::new(DFSchema::empty());-- let use_literals = v.values.iter().all(|row| {- row.iter().all(|expr| match expr {- Expr::Literal(_, _) => true,- Expr::Alias(alias) => matches!(alias.expr.as_ref(), Expr::Literal(_, _)),- _ => false,- })- });-- let (values, expressions) = if use_literals {- let values = convert_literal_rows(producer, &v.values)?;- (values, vec![])- } else {- let expressions =- convert_expression_rows(producer, &v.values, schema_len, &empty_schema)?;- (vec![], expressions)- };+ let expressions =+ convert_expression_rows(producer, &v.values, schema_len, &empty_schema)?;Ok(Box::new(Rel {rel_type: Some(RelType::Read(Box::new(ReadRel {common: None,@@ -237,10 +178,9 @@ pub fn from_values(best_effort_filter: None,projection: None,advanced_extension: None,- #[expect(deprecated)]read_type: Some(ReadType::VirtualTable(VirtualTable {- values,expressions,+ ..Default::default()})),}))),}))diff --git a/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json b/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.jsonindex 73fa06eea5..8a81a9a0c7 100644--- a/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json+++ b/datafusion/substrait/tests/testdata/test_plans/join_with_expression_key.json@@ -100,29 +100,52 @@}},"virtualTable": {- "values": [{- "fields": [{- "string": "aaa",- "nullable": true- }, {- "string": "host-a",- "nullable": true- }, {- "i64": "128",- "nullable": true- }]- }, {- "fields": [{- "string": "bbb",- "nullable": true- }, {- "string": "host-b",- "nullable": true- }, {- "i64": "256",- "nullable": true- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "string": "aaa",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "host-a",+ "nullable": true+ }+ },+ {+ "literal": {+ "i64": "128",+ "nullable": true+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "bbb",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "host-b",+ "nullable": true+ }+ },+ {+ "literal": {+ "i64": "256",+ "nullable": true+ }+ }+ ]+ }+ ]}}},@@ -293,23 +316,40 @@}},"virtualTable": {- "values": [{- "fields": [{- "string": "host-a",- "nullable": true- }, {- "i64": "107",- "nullable": true- }]- }, {- "fields": [{- "string": "host-b",- "nullable": true- }, {- "i64": "214",- "nullable": true- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "string": "host-a",+ "nullable": true+ }+ },+ {+ "literal": {+ "i64": "107",+ "nullable": true+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "host-b",+ "nullable": true+ }+ },+ {+ "literal": {+ "i64": "214",+ "nullable": true+ }+ }+ ]+ }+ ]}}},@@ -365,29 +405,52 @@}},"virtualTable": {- "values": [{- "fields": [{- "string": "aaa",- "nullable": true- }, {- "string": "host-a",- "nullable": true- }, {- "i64": "128",- "nullable": true- }]- }, {- "fields": [{- "string": "bbb",- "nullable": true- }, {- "string": "host-b",- "nullable": true- }, {- "i64": "256",- "nullable": true- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "string": "aaa",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "host-a",+ "nullable": true+ }+ },+ {+ "literal": {+ "i64": "128",+ "nullable": true+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "bbb",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "host-b",+ "nullable": true+ }+ },+ {+ "literal": {+ "i64": "256",+ "nullable": true+ }+ }+ ]+ }+ ]}}},diff --git a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.jsonindex 642256c562..13c1e5899d 100644--- a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json+++ b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct.json@@ -24,13 +24,147 @@}},"virtualTable": {- "values": [- { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },- { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },- { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] },- { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] },- { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },- { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "string": "1",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "a",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c1",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "2",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "b",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c2",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "3",+ "nullable": false+ }+ },+ {+ "literal": {+ "null": {+ "string": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c3",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "4",+ "nullable": false+ }+ },+ {+ "literal": {+ "null": {+ "string": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c4",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "5",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "e",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c5",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "6",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "f",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c6",+ "nullable": false+ }+ }+ ]+ }]}}@@ -50,13 +184,147 @@}},"virtualTable": {- "values": [- { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },- { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },- { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] },- { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] },- { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },- { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "string": "1",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "a",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c1",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "2",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "b",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c2",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "3",+ "nullable": false+ }+ },+ {+ "literal": {+ "null": {+ "string": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c3",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "4",+ "nullable": false+ }+ },+ {+ "literal": {+ "null": {+ "string": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c4",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "5",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "e",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c5",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "6",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "f",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c6",+ "nullable": false+ }+ }+ ]+ }]}}diff --git a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.jsonindex f16672947e..481bba44d8 100644--- a/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json+++ b/datafusion/substrait/tests/testdata/test_plans/mixed_join_equal_and_indistinct_left.json@@ -24,13 +24,147 @@}},"virtualTable": {- "values": [- { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },- { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },- { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] },- { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] },- { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },- { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "string": "1",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "a",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c1",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "2",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "b",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c2",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "3",+ "nullable": false+ }+ },+ {+ "literal": {+ "null": {+ "string": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c3",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "4",+ "nullable": false+ }+ },+ {+ "literal": {+ "null": {+ "string": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c4",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "5",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "e",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c5",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "6",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "f",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c6",+ "nullable": false+ }+ }+ ]+ }]}}@@ -50,13 +184,147 @@}},"virtualTable": {- "values": [- { "fields": [{ "string": "1", "nullable": false }, { "string": "a", "nullable": true }, { "string": "c1", "nullable": false }] },- { "fields": [{ "string": "2", "nullable": false }, { "string": "b", "nullable": true }, { "string": "c2", "nullable": false }] },- { "fields": [{ "string": "3", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c3", "nullable": false }] },- { "fields": [{ "string": "4", "nullable": false }, { "null": { "string": { "nullability": "NULLABILITY_NULLABLE" } }, "nullable": true }, { "string": "c4", "nullable": false }] },- { "fields": [{ "string": "5", "nullable": false }, { "string": "e", "nullable": true }, { "string": "c5", "nullable": false }] },- { "fields": [{ "string": "6", "nullable": false }, { "string": "f", "nullable": true }, { "string": "c6", "nullable": false }] }+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "string": "1",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "a",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c1",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "2",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "b",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c2",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "3",+ "nullable": false+ }+ },+ {+ "literal": {+ "null": {+ "string": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c3",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "4",+ "nullable": false+ }+ },+ {+ "literal": {+ "null": {+ "string": {+ "nullability": "NULLABILITY_NULLABLE"+ }+ },+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c4",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "5",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "e",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c5",+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "string": "6",+ "nullable": false+ }+ },+ {+ "literal": {+ "string": "f",+ "nullable": true+ }+ },+ {+ "literal": {+ "string": "c6",+ "nullable": false+ }+ }+ ]+ }]}}diff --git a/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json b/datafusion/substrait/tests/testdata/test_plans/multiple_joins.jsonindex e88cce648d..15c0313b43 100644--- a/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json+++ b/datafusion/substrait/tests/testdata/test_plans/multiple_joins.json@@ -72,19 +72,30 @@}},"virtualTable": {- "values": [{- "fields": [{- "i64": "1",- "nullable": true,- "typeVariationReference": 0- }]- }, {- "fields": [{- "i64": "2",- "nullable": true,- "typeVariationReference": 0- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "i64": "1",+ "nullable": true,+ "typeVariationReference": 0+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "i64": "2",+ "nullable": true,+ "typeVariationReference": 0+ }+ }+ ]+ }+ ]}}},@@ -153,27 +164,44 @@}},"virtualTable": {- "values": [{- "fields": [{- "i64": "1",- "nullable": true,- "typeVariationReference": 0- }, {- "string": "info",- "nullable": true,- "typeVariationReference": 0- }]- }, {- "fields": [{- "i64": "2",- "nullable": true,- "typeVariationReference": 0- }, {- "string": "low",- "nullable": true,- "typeVariationReference": 0- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "i64": "1",+ "nullable": true,+ "typeVariationReference": 0+ }+ },+ {+ "literal": {+ "string": "info",+ "nullable": true,+ "typeVariationReference": 0+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "i64": "2",+ "nullable": true,+ "typeVariationReference": 0+ }+ },+ {+ "literal": {+ "string": "low",+ "nullable": true,+ "typeVariationReference": 0+ }+ }+ ]+ }+ ]}}},@@ -272,19 +300,30 @@}},"virtualTable": {- "values": [{- "fields": [{- "i64": "1",- "nullable": true,- "typeVariationReference": 0- }]- }, {- "fields": [{- "i64": "2",- "nullable": true,- "typeVariationReference": 0- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "i64": "1",+ "nullable": true,+ "typeVariationReference": 0+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "i64": "2",+ "nullable": true,+ "typeVariationReference": 0+ }+ }+ ]+ }+ ]}}},@@ -389,19 +428,30 @@}},"virtualTable": {- "values": [{- "fields": [{- "i64": "1",- "nullable": true,- "typeVariationReference": 0- }]- }, {- "fields": [{- "i64": "2",- "nullable": true,- "typeVariationReference": 0- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "i64": "1",+ "nullable": true,+ "typeVariationReference": 0+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "i64": "2",+ "nullable": true,+ "typeVariationReference": 0+ }+ }+ ]+ }+ ]}}},diff --git a/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json b/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.jsonindex e1c5574f8b..e29c000ee6 100644--- a/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json+++ b/datafusion/substrait/tests/testdata/test_plans/non_nullable_lists.substrait.json@@ -34,26 +34,28 @@}},"virtualTable": {- "values": [+ "expressions": [{"fields": [{- "list": {- "values": [- {- "i32": 1,- "nullable": false,- "typeVariationReference": 0- },- {- "i32": 2,- "nullable": false,- "typeVariationReference": 0- }- ]- },- "nullable": false,- "typeVariationReference": 0+ "literal": {+ "list": {+ "values": [+ {+ "i32": 1,+ "nullable": false,+ "typeVariationReference": 0+ },+ {+ "i32": 2,+ "nullable": false,+ "typeVariationReference": 0+ }+ ]+ },+ "nullable": false,+ "typeVariationReference": 0+ }}]}diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.jsonindex eeaf5a3dd8..d5209a683d 100644--- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json+++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_logb_expr.substrait.json@@ -85,23 +85,40 @@"direct": {}},"virtualTable": {- "values": [{- "fields": [{- "fp32": 1.0,- "nullable": false- }, {- "fp32": 10.0,- "nullable": false- }]- }, {- "fields": [{- "fp32": 100.0,- "nullable": false- }, {- "fp32": 10.0,- "nullable": false- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "fp32": 1.0,+ "nullable": false+ }+ },+ {+ "literal": {+ "fp32": 10.0,+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "fp32": 100.0,+ "nullable": false+ }+ },+ {+ "literal": {+ "fp32": 10.0,+ "nullable": false+ }+ }+ ]+ }+ ]}}}diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.jsonindex 6749a301b1..f609d26138 100644--- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json+++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_between_expr.substrait.json@@ -106,29 +106,52 @@"direct": {}},"virtualTable": {- "values": [{- "fields": [{- "i8": 2,- "nullable": false- }, {- "i8": 1,- "nullable": false- }, {- "i8": 3,- "nullable": false- }]- }, {- "fields": [{- "i8": 4,- "nullable": false- }, {- "i8": 1,- "nullable": false- }, {- "i8": 2,- "nullable": false- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "i8": 2,+ "nullable": false+ }+ },+ {+ "literal": {+ "i8": 1,+ "nullable": false+ }+ },+ {+ "literal": {+ "i8": 3,+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "i8": 4,+ "nullable": false+ }+ },+ {+ "literal": {+ "i8": 1,+ "nullable": false+ }+ },+ {+ "literal": {+ "i8": 2,+ "nullable": false+ }+ }+ ]+ }+ ]}}}diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.jsonindex 8365b1edfe..5d91342257 100644--- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json+++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_and_not.substrait.json@@ -85,39 +85,72 @@"direct": {}},"virtualTable": {- "values": [{- "fields": [{- "boolean": true,- "nullable": false- }, {- "boolean": true,- "nullable": false- }]- }, {- "fields": [{- "boolean": true,- "nullable": false- }, {- "boolean": false,- "nullable": false- }]- }, {- "fields": [{- "boolean": false,- "nullable": false- }, {- "boolean": true,- "nullable": false- }]- }, {- "fields": [{- "boolean": false,- "nullable": false- }, {- "boolean": false,- "nullable": false- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "boolean": true,+ "nullable": false+ }+ },+ {+ "literal": {+ "boolean": true,+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "boolean": true,+ "nullable": false+ }+ },+ {+ "literal": {+ "boolean": false,+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "boolean": false,+ "nullable": false+ }+ },+ {+ "literal": {+ "boolean": true,+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "boolean": false,+ "nullable": false+ }+ },+ {+ "literal": {+ "boolean": false,+ "nullable": false+ }+ }+ ]+ }+ ]}}}diff --git a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.jsonindex cfd760de89..2514c0afc9 100644--- a/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json+++ b/datafusion/substrait/tests/testdata/test_plans/scalar_fn_to_built_in_binary_expr_xor.substrait.json@@ -85,39 +85,72 @@"direct": {}},"virtualTable": {- "values": [{- "fields": [{- "boolean": true,- "nullable": false- }, {- "boolean": true,- "nullable": false- }]- }, {- "fields": [{- "boolean": true,- "nullable": false- }, {- "boolean": false,- "nullable": false- }]- }, {- "fields": [{- "boolean": false,- "nullable": false- }, {- "boolean": true,- "nullable": false- }]- }, {- "fields": [{- "boolean": false,- "nullable": false- }, {- "boolean": false,- "nullable": false- }]- }]+ "expressions": [+ {+ "fields": [+ {+ "literal": {+ "boolean": true,+ "nullable": false+ }+ },+ {+ "literal": {+ "boolean": true,+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "boolean": true,+ "nullable": false+ }+ },+ {+ "literal": {+ "boolean": false,+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "boolean": false,+ "nullable": false+ }+ },+ {+ "literal": {+ "boolean": true,+ "nullable": false+ }+ }+ ]+ },+ {+ "fields": [+ {+ "literal": {+ "boolean": false,+ "nullable": false+ }+ },+ {+ "literal": {+ "boolean": false,+ "nullable": false+ }+ }+ ]+ }+ ]}}}diff --git a/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json b/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.jsonindex e9f6795880..b0d4ba4813 100644--- a/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json+++ b/datafusion/substrait/tests/testdata/test_plans/select_count_from_select_1.substrait.json@@ -43,12 +43,14 @@}},"virtualTable": {- "values": [+ "expressions": [{"fields": [{- "i64": "0",- "nullable": false+ "literal": {+ "i64": "0",+ "nullable": false+ }}]}