Apache DataFusion SQL Query Engine
chore(deps): bump substrait from 0.62 to 0.63.0 (#20876)
Ben Bellick981b5c37bfd1
diff --git a/Cargo.lock b/Cargo.lockindex f0d8568502..bc97507d64 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2750,7 +2750,7 @@ dependencies = ["libc","option-ext","redox_users",- "windows-sys 0.59.0",+ "windows-sys 0.61.2",][[package]]@@ -2894,7 +2894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"dependencies = ["libc",- "windows-sys 0.59.0",+ "windows-sys 0.61.2",][[package]]@@ -4155,7 +4155,7 @@ version = "0.50.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"dependencies = [- "windows-sys 0.59.0",+ "windows-sys 0.61.2",][[package]]@@ -4773,7 +4773,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"dependencies = ["heck",- "itertools 0.13.0",+ "itertools 0.14.0","log","multimap","petgraph",@@ -4792,7 +4792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"dependencies = ["anyhow",- "itertools 0.13.0",+ "itertools 0.14.0","proc-macro2","quote","syn 2.0.117",@@ -4894,7 +4894,7 @@ dependencies = ["once_cell","socket2","tracing",- "windows-sys 0.59.0",+ "windows-sys 0.60.2",][[package]]@@ -5261,7 +5261,7 @@ dependencies = ["errno","libc","linux-raw-sys",- "windows-sys 0.59.0",+ "windows-sys 0.61.2",][[package]]@@ -5516,6 +5516,7 @@ version = "1.0.149"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"dependencies = [+ "indexmap 2.13.0","itoa","memchr","serde",@@ -5873,11 +5874,12 @@ dependencies = [[[package]]name = "substrait"-version = "0.62.2"+version = "0.63.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"dependencies = ["heck",+ "indexmap 2.13.0","pbjson 0.8.0","pbjson-build 0.8.0","pbjson-types",@@ -5969,7 +5971,7 @@ dependencies = ["getrandom 0.3.4","once_cell","rustix",- "windows-sys 0.59.0",+ "windows-sys 0.61.2",][[package]]@@ -6927,7 +6929,7 @@ version = "0.1.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"dependencies = [- "windows-sys 0.59.0",+ "windows-sys 0.61.2",][[package]]diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex 1c901f6d4a..dc4daf1ab7 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -4456,49 +4456,49 @@ mod tests {[{"Plan": {+ "Node Type": "Projection","Expressions": ["employee_csv.id"],- "Node Type": "Projection",- "Output": [- "id"- ],"Plans": [{- "Condition": "employee_csv.state IN (<subquery>)","Node Type": "Filter",- "Output": [- "id",- "state"- ],+ "Condition": "employee_csv.state IN (<subquery>)","Plans": [{"Node Type": "Subquery",- "Output": [- "state"- ],"Plans": [{"Node Type": "TableScan",+ "Relation Name": "employee_csv",+ "Plans": [],"Output": ["state"- ],- "Plans": [],- "Relation Name": "employee_csv"+ ]}+ ],+ "Output": [+ "state"]},{"Node Type": "TableScan",+ "Relation Name": "employee_csv",+ "Plans": [],"Output": ["id","state"- ],- "Plans": [],- "Relation Name": "employee_csv"+ ]}+ ],+ "Output": [+ "id",+ "state"]}+ ],+ "Output": [+ "id"]}}diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.sltindex 717b455193..9916892058 100644--- a/datafusion/sqllogictest/test_files/explain.slt+++ b/datafusion/sqllogictest/test_files/explain.slt@@ -646,11 +646,11 @@ logical_plan02)--{03)----"Plan": {04)------"Node Type": "Values",-05)------"Output": [-06)--------"column1"-07)------],-08)------"Plans": [],-09)------"Values": "(Int64(1))"+05)------"Values": "(Int64(1))",+06)------"Plans": [],+07)------"Output": [+08)--------"column1"+09)------]10)----}11)--}12)]diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.tomlindex 927c326b88..a0f203cec8 100644--- a/datafusion/substrait/Cargo.toml+++ b/datafusion/substrait/Cargo.toml@@ -44,7 +44,7 @@ object_store = { workspace = true }# We need to match the version in substrait, so we don't use the workspace version herepbjson-types = { version = "0.8.0" }prost = { workspace = true }-substrait = { version = "0.62", features = ["serde"] }+substrait = { version = "0.63.0", features = ["serde"] }url = { workspace = true }tokio = { workspace = true, features = ["fs"] }diff --git a/datafusion/substrait/src/extensions.rs b/datafusion/substrait/src/extensions.rsindex 0f848270ba..78c357f3b8 100644--- a/datafusion/substrait/src/extensions.rs+++ b/datafusion/substrait/src/extensions.rs@@ -113,14 +113,10 @@ impl TryFrom<&Vec<SimpleExtensionDeclaration>> for Extensions {}impl From<Extensions> for Vec<SimpleExtensionDeclaration> {- // Silence deprecation warnings for `extension_uri_reference` during the uri -> urn migration- // See: https://github.com/substrait-io/substrait/issues/856- #[expect(deprecated)]fn from(val: Extensions) -> Vec<SimpleExtensionDeclaration> {let mut extensions = vec![];for (f_anchor, f_name) in val.functions {let function_extension = ExtensionFunction {- extension_uri_reference: u32::MAX,extension_urn_reference: u32::MAX,function_anchor: f_anchor,name: f_name,@@ -133,7 +129,6 @@ impl From<Extensions> for Vec<SimpleExtensionDeclaration> {for (t_anchor, t_name) in val.types {let type_extension = ExtensionType {- extension_uri_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545extension_urn_reference: u32::MAX, // https://github.com/apache/datafusion/issues/11545type_anchor: t_anchor,name: t_name,@@ -146,8 +141,7 @@ impl From<Extensions> for Vec<SimpleExtensionDeclaration> {for (tv_anchor, tv_name) in val.type_variations {let type_variation_extension = ExtensionTypeVariation {- extension_uri_reference: u32::MAX, // We don't register proper extension URIs yet- extension_urn_reference: u32::MAX, // We don't register proper extension URIs yet+ extension_urn_reference: u32::MAX, // We don't register proper extension URNs yettype_variation_anchor: tv_anchor,name: tv_name,};diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs b/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rsindex 50d93a4600..dae6c625ef 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/field_reference.rs@@ -56,6 +56,9 @@ pub(crate) fn from_substrait_field_reference(Some(RootType::Expression(_)) => not_impl_err!("Expression root type in field reference is not supported"),+ Some(RootType::LambdaParameterReference(_)) => not_impl_err!(+ "Lambda parameter reference in field reference is not yet supported"+ ),}}_ => not_impl_err!(diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rsindex ad38b6adde..d7d7a69581 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs@@ -43,7 +43,7 @@ use prost::Message;use std::sync::Arc;use substrait::proto;use substrait::proto::expression::Literal;-use substrait::proto::expression::literal::user_defined::Val;+use substrait::proto::expression::literal::user_defined::{TypeAnchorType, Val};use substrait::proto::expression::literal::{IntervalCompound, IntervalDayToSecond, IntervalYearToMonth, LiteralType,interval_day_to_second,@@ -474,11 +474,17 @@ pub(crate) fn from_substrait_literal()))};- if let Some(name) = consumer- .get_extensions()- .types- .get(&user_defined.type_reference)- {+ let type_ref = match user_defined.type_anchor_type {+ Some(TypeAnchorType::TypeReference(ref_val)) => ref_val,+ Some(TypeAnchorType::TypeAliasReference(_)) => {+ return not_impl_err!(+ "Type alias references in user-defined literals are not yet supported"+ );+ }+ None => 0,+ };++ if let Some(name) = consumer.get_extensions().types.get(&type_ref) {match name.as_ref() {FLOAT_16_TYPE_NAME => {// Rules for encoding fp16 Substrait literals are defined as part of Arrow here:@@ -518,14 +524,14 @@ pub(crate) fn from_substrait_literal(_ => {return not_impl_err!("Unsupported Substrait user defined type with ref {} and name {}",- user_defined.type_reference,+ type_ref,name);}}} else {#[expect(deprecated)]- match user_defined.type_reference {+ match type_ref {// Kept for backwards compatibility, producers should useIntervalYearToMonth insteadINTERVAL_YEAR_MONTH_TYPE_REF => {let Some(Val::Value(raw_val)) = user_defined.val.as_ref() else {@@ -568,7 +574,7 @@ pub(crate) fn from_substrait_literal(_ => {return not_impl_err!("Unsupported Substrait user defined type literal with ref {}",- user_defined.type_reference+ type_ref);}}diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/mod.rs b/datafusion/substrait/src/logical_plan/consumer/expr/mod.rsindex 5d98850c72..a0468dbd45 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/mod.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/mod.rs@@ -93,6 +93,9 @@ pub async fn from_substrait_rex(RexType::DynamicParameter(expr) => {consumer.consume_dynamic_parameter(expr, input_schema).await}+ RexType::Lambda(_) | RexType::LambdaInvocation(_) => {+ not_impl_err!("Lambda expressions are not yet supported")+ }},None => substrait_err!("Expression must set rex_type: {expression:?}"),}diff --git a/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs b/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rsindex a23f1faed1..730ceab8cc 100644--- a/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs+++ b/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs@@ -141,7 +141,15 @@ use substrait::proto::{////// // and user-defined literals/// fn consume_user_defined_literal(&self, literal: &proto::expression::literal::UserDefined) -> Result<ScalarValue> {-/// let type_string = self.extensions.types.get(&literal.type_reference).unwrap();+/// // extract type_reference from the new TypeAnchorType oneof+/// let type_ref = match literal.type_anchor_type {+/// Some(proto::expression::literal::user_defined::TypeAnchorType::TypeReference(r)) => r,+/// Some(proto::expression::literal::user_defined::TypeAnchorType::TypeAliasReference(_)) => {+/// return not_impl_err!("Type alias references are not yet supported")+/// }+/// None => 0,+/// };+/// let type_string = self.extensions.types.get(&type_ref).unwrap();/// match type_string.as_str() {/// "u!foo" => not_impl_err!("handle foo conversion"),/// "u!bar" => not_impl_err!("handle bar conversion"),@@ -444,10 +452,22 @@ pub trait SubstraitConsumer: Send + Sync + Sized {&self,user_defined_literal: &proto::expression::literal::UserDefined,) -> datafusion::common::Result<ScalarValue> {- substrait_err!(- "Missing handler for user-defined literals {}",- user_defined_literal.type_reference- )+ let type_ref = match user_defined_literal.type_anchor_type {+ Some(+ proto::expression::literal::user_defined::TypeAnchorType::TypeReference(+ ref_val,+ ),+ ) => ref_val,+ Some(+ proto::expression::literal::user_defined::TypeAnchorType::TypeAliasReference(_),+ ) => {+ return not_impl_err!(+ "Type alias references in user-defined literals are not yet supported"+ )+ }+ None => 0,+ };+ substrait_err!("Missing handler for user-defined literals {}", type_ref)}}diff --git a/datafusion/substrait/src/logical_plan/producer/expr/literal.rs b/datafusion/substrait/src/logical_plan/producer/expr/literal.rsindex 8882c992dc..bbed7ee9be 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/literal.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/literal.rs@@ -117,7 +117,7 @@ pub(crate) fn to_substrait_literal((LiteralType::UserDefined(substrait::proto::expression::literal::UserDefined {- type_reference: type_anchor,+ type_anchor_type: Some(substrait::proto::expression::literal::user_defined::TypeAnchorType::TypeReference(type_anchor)),type_parameters: vec![],val: Some(substrait::proto::expression::literal::user_defined::Val::Value(pbjson_types::Any {diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rsindex 3aa8aa2b68..d130961596 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs@@ -60,9 +60,6 @@ use substrait::version;////// Substrait also requires the input schema of the expressions to be included in the/// message. The field names of the input schema will be serialized.-// Silence deprecation warnings for `extension_uris` during the uri -> urn migration-// See: https://github.com/substrait-io/substrait/issues/856-#[expect(deprecated)]pub fn to_substrait_extended_expr(exprs: &[(&Expr, &Field)],schema: &DFSchemaRef,@@ -87,7 +84,6 @@ pub fn to_substrait_extended_expr(Ok(Box::new(ExtendedExpression {advanced_extensions: None,expected_type_urls: vec![],- extension_uris: vec![],extension_urns: vec![],extensions: extensions.into(),version: Some(version::version_with_producer("datafusion")),diff --git a/datafusion/substrait/src/logical_plan/producer/plan.rs b/datafusion/substrait/src/logical_plan/producer/plan.rsindex 9396329f8d..3b58720dba 100644--- a/datafusion/substrait/src/logical_plan/producer/plan.rs+++ b/datafusion/substrait/src/logical_plan/producer/plan.rs@@ -24,9 +24,6 @@ use substrait::proto::{Plan, PlanRel, Rel, RelRoot, plan_rel};use substrait::version;/// Convert DataFusion LogicalPlan to Substrait Plan-// Silence deprecation warnings for `extension_uris` during the uri -> urn migration-// See: https://github.com/substrait-io/substrait/issues/856-#[expect(deprecated)]pub fn to_substrait_plan(plan: &LogicalPlan,state: &SessionState,@@ -47,7 +44,6 @@ pub fn to_substrait_plan(let extensions = producer.get_extensions();Ok(Box::new(Plan {version: Some(version::version_with_producer("datafusion")),- extension_uris: vec![],extension_urns: vec![],extensions: extensions.into(),relations: plan_rels,diff --git a/datafusion/substrait/tests/utils.rs b/datafusion/substrait/tests/utils.rsindex 6a6824579b..4d9b5ca83e 100644--- a/datafusion/substrait/tests/utils.rs+++ b/datafusion/substrait/tests/utils.rs@@ -486,6 +486,7 @@ pub mod test {// Enum is deprecated#[expect(deprecated)]RexType::Enum(_) => {}+ RexType::Lambda(_) | RexType::LambdaInvocation(_) => {}}Ok(())}