Skip to content

Apache DataFusion SQL Query Engine

change: fully deprecated the acceptance of value field in VirtualTables (#24346)

eliot1480c98fe99d3335
diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
index 2cad144080..865707a11d 100644
--- a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs
@@ -35,7 +35,6 @@ use substrait::proto::read_rel::local_files::file_or_files::PathType::UriFile;
use substrait::proto::{Expression, ReadRel};
use url::Url;
-#[expect(deprecated)]
pub async fn from_read_rel(
consumer: &impl SubstraitConsumer,
read: &ReadRel,
@@ -114,14 +113,14 @@ pub async fn from_read_rel(
.await
}
Some(ReadType::VirtualTable(vt)) => {
- if vt.values.is_empty() && vt.expressions.is_empty() {
+ if vt.expressions.is_empty() {
return Ok(LogicalPlan::EmptyRelation(EmptyRelation {
produce_one_row: false,
schema: DFSchemaRef::new(substrait_schema),
}));
}
- // Check for produce_one_row pattern in both old (values) and new (expressions) formats.
+ // Check for produce_one_row pattern
// A VirtualTable with exactly one row containing only empty/default fields represents
// an EmptyRelation with produce_one_row=true. This pattern is used for queries without
// a FROM clause (e.g., "SELECT 1 AS one") where a single phantom row is needed to
@@ -129,14 +128,9 @@ pub async fn from_read_rel(
// the SQL "DUAL" table (see: https://en.wikipedia.org/wiki/DUAL_table) which some
// databases provide as a single-row source for selecting constant expressions when no
// real table is present.
- let is_produce_one_row = (vt.values.len() == 1
- && vt.expressions.is_empty()
+ let is_produce_one_row = vt.expressions.len() == 1
&& substrait_schema.fields().is_empty()
- && vt.values[0].fields.is_empty())
- || (vt.expressions.len() == 1
- && vt.values.is_empty()
- && substrait_schema.fields().is_empty()
- && vt.expressions[0].fields.is_empty());
+ && vt.expressions[0].fields.is_empty();
if is_produce_one_row {
return Ok(LogicalPlan::EmptyRelation(EmptyRelation {
@@ -145,63 +139,58 @@ pub async fn from_read_rel(
}));
}
- let values = if !vt.expressions.is_empty() {
- let mut exprs = vec![];
- for row in &vt.expressions {
- if row.fields.len() != substrait_schema.fields().len() {
- return substrait_err!(
- "Field count mismatch: expected {} fields but found {} in virtual table row",
- substrait_schema.fields().len(),
- row.fields.len()
- );
- }
-
- let mut row_exprs = vec![];
- let mut name_idx = 0;
- for expression in &row.fields {
- // Top-level names are provided through schema
- // Each expression consumes at least one name, and Literals may consume additional names.
- name_idx += 1;
- let expr = match expression.rex_type.as_ref() {
- Some(substrait::proto::expression::RexType::Literal(lit)) => {
- // Values literals need 'named_struct.names' so nested struct fields keep their names from the ReadRel base schema.
- // This is important for nested struct fields to retain their names.
- Expr::Literal(
- from_substrait_literal(
- consumer,
- lit,
- &named_struct.names,
- &mut name_idx,
- )?,
- None,
- )
- }
- _ => {
- consumer
- .consume_expression(expression, &substrait_schema)
- .await?
- }
- };
- row_exprs.push(expr);
- }
-
- if name_idx != named_struct.names.len() {
- return substrait_err!(
- "Names list must match exactly to nested schema, but found {} uses for {} names",
- name_idx,
- named_struct.names.len()
- );
- }
- exprs.push(row_exprs);
+ let mut rows = vec![];
+ for row in &vt.expressions {
+ if row.fields.len() != substrait_schema.fields().len() {
+ return substrait_err!(
+ "Field count mismatch: expected {} fields but found {} in virtual table row",
+ substrait_schema.fields().len(),
+ row.fields.len()
+ );
}
- exprs
- } else {
- convert_literal_rows(consumer, vt, named_struct)?
- };
+
+ let mut row_exprs = vec![];
+ let mut name_idx = 0;
+ for expression in &row.fields {
+ // Top-level names are provided through schema
+ // Each expression consumes at least one name, and Literals may consume additional names.
+ name_idx += 1;
+ let expr = match expression.rex_type.as_ref() {
+ Some(substrait::proto::expression::RexType::Literal(lit)) => {
+ // Literal expressions need 'named_struct.names' so nested struct fields
+ // keep their names from the ReadRel base schema.
+ Expr::Literal(
+ from_substrait_literal(
+ consumer,
+ lit,
+ &named_struct.names,
+ &mut name_idx,
+ )?,
+ None,
+ )
+ }
+ _ => {
+ consumer
+ .consume_expression(expression, &substrait_schema)
+ .await?
+ }
+ };
+ row_exprs.push(expr);
+ }
+
+ if name_idx != named_struct.names.len() {
+ return substrait_err!(
+ "Names list must match exactly to nested schema, but found {} uses for {} names",
+ name_idx,
+ named_struct.names.len()
+ );
+ }
+ rows.push(row_exprs);
+ }
Ok(LogicalPlan::Values(Values {
schema: DFSchemaRef::new(substrait_schema),
- values,
+ values: rows,
}))
}
Some(ReadType::LocalFiles(lf)) => {
@@ -251,46 +240,6 @@ pub async fn from_read_rel(
}
}
-/// Converts Substrait literal rows from a VirtualTable into DataFusion expressions.
-///
-/// This function processes the deprecated `values` field of VirtualTable, converting
-/// each literal value into a `Expr::Literal` while tracking and validating the name
-/// indices against the provided named struct schema.
-fn convert_literal_rows(
- consumer: &impl SubstraitConsumer,
- vt: &substrait::proto::read_rel::VirtualTable,
- named_struct: &substrait::proto::NamedStruct,
-) -> datafusion::common::Result<Vec<Vec<Expr>>> {
- #[expect(deprecated)]
- vt.values
- .iter()
- .map(|row| {
- let mut name_idx = 0;
- let lits = row
- .fields
- .iter()
- .map(|lit| {
- name_idx += 1; // top-level names are provided through schema
- Ok(Expr::Literal(from_substrait_literal(
- consumer,
- lit,
- &named_struct.names,
- &mut name_idx,
- )?, None))
- })
- .collect::<datafusion::common::Result<_>>()?;
- if name_idx != named_struct.names.len() {
- return substrait_err!(
- "Names list must match exactly to nested schema, but found {} uses for {} names",
- name_idx,
- named_struct.names.len()
- );
- }
- Ok(lits)
- })
- .collect::<datafusion::common::Result<_>>()
-}
-
pub fn apply_masking(
schema: DFSchema,
mask_expression: Option<&MaskExpression>,