Apache DataFusion SQL Query Engine
Split proto serialization to encapsulate private state (#21835) (#21929)
Adrian Garcia Badaracco077f08a9a663
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex 5ff1f6467b..bb2075ee01 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -605,7 +605,6 @@ jobs:rust-version: stable- name: Runrun: |- echo '' > datafusion/proto/src/generated/datafusion.rsci/scripts/rust_fmt.sh# Coverage job disabled due todiff --git a/Cargo.lock b/Cargo.lockindex 4d5b15075e..63d57b2e69 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2344,6 +2344,7 @@ dependencies = ["datafusion-functions","datafusion-functions-aggregate-common","datafusion-physical-expr-common",+ "datafusion-proto-models","half","hashbrown 0.17.1","indexmap 2.14.0",@@ -2379,6 +2380,7 @@ dependencies = ["criterion","datafusion-common","datafusion-expr-common",+ "datafusion-proto-models","hashbrown 0.17.1","indexmap 2.14.0","itertools 0.14.0",@@ -2473,12 +2475,11 @@ dependencies = ["datafusion-physical-expr-common","datafusion-physical-plan","datafusion-proto-common",+ "datafusion-proto-models","doc-comment","object_store",- "pbjson 0.9.0","pretty_assertions","prost",- "serde","serde_json","tokio",]@@ -2495,6 +2496,16 @@ dependencies = ["serde",]+[[package]]+name = "datafusion-proto-models"+version = "53.1.0"+dependencies = [+ "datafusion-proto-common",+ "pbjson 0.9.0",+ "prost",+ "serde",+]+[[package]]name = "datafusion-pruning"version = "53.1.0"@@ -6201,9 +6212,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"[[package]]name = "tokio"-version = "1.52.3"+version = "1.52.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"+checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386"dependencies = ["bytes","libc",@@ -6341,9 +6352,9 @@ checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db"[[package]]name = "tonic"-version = "0.14.6"+version = "0.14.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"+checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec"dependencies = ["async-trait","axum",diff --git a/Cargo.toml b/Cargo.tomlindex 78c271d524..2862a7a97b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -47,9 +47,10 @@ members = ["datafusion/pruning","datafusion/physical-plan","datafusion/proto",- "datafusion/proto/gen","datafusion/proto-common","datafusion/proto-common/gen",+ "datafusion/proto-models",+ "datafusion/proto-models/gen","datafusion/session","datafusion/spark","datafusion/sql",@@ -152,6 +153,7 @@ datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", versiodatafusion-physical-plan = { path = "datafusion/physical-plan", version = "53.1.0" }datafusion-proto = { path = "datafusion/proto", version = "53.1.0" }datafusion-proto-common = { path = "datafusion/proto-common", version = "53.1.0" }+datafusion-proto-models = { path = "datafusion/proto-models", version = "53.1.0" }datafusion-pruning = { path = "datafusion/pruning", version = "53.1.0" }datafusion-session = { path = "datafusion/session", version = "53.1.0" }datafusion-spark = { path = "datafusion/spark", version = "53.1.0" }diff --git a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs b/datafusion-examples/examples/custom_data_source/adapter_serialization.rsindex a956348279..f18b888f3e 100644--- a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs+++ b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs@@ -62,7 +62,8 @@ use datafusion_proto::bytes::{use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converter;use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;use datafusion_proto::physical_plan::{- PhysicalExtensionCodec, PhysicalPlanDecodeContext, PhysicalProtoConverterExtension,+ PhysicalExtensionCodec, PhysicalPlanDecodeContext, PhysicalPlanNodeExt,+ PhysicalProtoConverterExtension,};use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;use datafusion_proto::protobuf::{diff --git a/datafusion-examples/examples/proto/expression_deduplication.rs b/datafusion-examples/examples/proto/expression_deduplication.rsindex ae4a76e79f..31bb234e28 100644--- a/datafusion-examples/examples/proto/expression_deduplication.rs+++ b/datafusion-examples/examples/proto/expression_deduplication.rs@@ -52,7 +52,7 @@ use datafusion_proto::physical_plan::from_proto::parse_physical_expr_with_converuse datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;use datafusion_proto::physical_plan::{DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalPlanDecodeContext,- PhysicalProtoConverterExtension,+ PhysicalPlanNodeExt, PhysicalProtoConverterExtension,};use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};use prost::Message;diff --git a/datafusion/expr-common/src/operator.rs b/datafusion/expr-common/src/operator.rsindex a078a27f2a..7b10d9bfae 100644--- a/datafusion/expr-common/src/operator.rs+++ b/datafusion/expr-common/src/operator.rs@@ -390,6 +390,50 @@ impl Operator {| Operator::StringConcat => false,}}++ /// Parse an `Operator` from the string name `datafusion-proto` uses on the+ /// wire (the `Debug` name of the variant, e.g. `"Eq"`).+ ///+ /// Returns `None` for names with no binary-operator counterpart. This is+ /// the canonical proto-string mapping, shared by `datafusion-proto`+ /// (logical plans) and `PhysicalExpr` decoders such as `BinaryExpr`, so the+ /// mapping is not duplicated across crates.+ pub fn from_proto_name(name: &str) -> Option<Operator> {+ Some(match name {+ "And" => Operator::And,+ "Or" => Operator::Or,+ "Eq" => Operator::Eq,+ "NotEq" => Operator::NotEq,+ "LtEq" => Operator::LtEq,+ "Lt" => Operator::Lt,+ "Gt" => Operator::Gt,+ "GtEq" => Operator::GtEq,+ "Plus" => Operator::Plus,+ "Minus" => Operator::Minus,+ "Multiply" => Operator::Multiply,+ "Divide" => Operator::Divide,+ "Modulo" => Operator::Modulo,+ "IsDistinctFrom" => Operator::IsDistinctFrom,+ "IsNotDistinctFrom" => Operator::IsNotDistinctFrom,+ "BitwiseAnd" => Operator::BitwiseAnd,+ "BitwiseOr" => Operator::BitwiseOr,+ "BitwiseXor" => Operator::BitwiseXor,+ "BitwiseShiftLeft" => Operator::BitwiseShiftLeft,+ "BitwiseShiftRight" => Operator::BitwiseShiftRight,+ "RegexIMatch" => Operator::RegexIMatch,+ "RegexMatch" => Operator::RegexMatch,+ "RegexNotIMatch" => Operator::RegexNotIMatch,+ "RegexNotMatch" => Operator::RegexNotMatch,+ "LikeMatch" => Operator::LikeMatch,+ "ILikeMatch" => Operator::ILikeMatch,+ "NotLikeMatch" => Operator::NotLikeMatch,+ "NotILikeMatch" => Operator::NotILikeMatch,+ "StringConcat" => Operator::StringConcat,+ "AtArrow" => Operator::AtArrow,+ "ArrowAt" => Operator::ArrowAt,+ _ => return None,+ })+ }}impl fmt::Display for Operator {diff --git a/datafusion/physical-expr-common/Cargo.toml b/datafusion/physical-expr-common/Cargo.tomlindex 0e4748b81d..d1ee7feb29 100644--- a/datafusion/physical-expr-common/Cargo.toml+++ b/datafusion/physical-expr-common/Cargo.toml@@ -40,11 +40,18 @@ workspace = true[lib]name = "datafusion_physical_expr_common"+[features]+default = []+# Enables the `PhysicalExpr::to_proto` hook used by `datafusion-proto`.+# Off by default so crates that never serialize plans pay nothing.+proto = ["dep:datafusion-proto-models"]+[dependencies]arrow = { workspace = true }chrono = { workspace = true }datafusion-common = { workspace = true }datafusion-expr-common = { workspace = true }+datafusion-proto-models = { workspace = true, optional = true }hashbrown = { workspace = true }indexmap = { workspace = true }itertools = { workspace = true }diff --git a/datafusion/physical-expr-common/src/physical_expr.rs b/datafusion/physical-expr-common/src/physical_expr.rsindex 7b3f7dcc76..887ed73745 100644--- a/datafusion/physical-expr-common/src/physical_expr.rs+++ b/datafusion/physical-expr-common/src/physical_expr.rs@@ -477,6 +477,180 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {fn expression_id(&self) -> Option<u64> {None}++ /// Serialize this expression to a [`PhysicalExprNode`] proto message.+ ///+ /// Returning `Ok(None)` means "this expression does not know how to+ /// serialize itself"; the caller (typically `datafusion-proto`) will fall+ /// back to its existing codec / extension paths. This matches today's+ /// behavior for expressions that aren't built into `datafusion-proto`.+ ///+ /// Returning `Ok(Some(node))` means the expression has serialized itself+ /// fully; the caller should not try any further fallback path.+ ///+ /// Returning `Err(_)` means a real serialization failure (e.g. the+ /// expression knows it should serialize but a child failed).+ ///+ /// The motivating use case is letting expressions with private state+ /// (e.g. `DynamicFilterPhysicalExpr`'s `RwLock`-protected inner fields)+ /// reach into their own internals for `try_to_proto`/`try_from_proto`+ /// without having to expose `pub` accessors to `datafusion-proto`. See+ /// <https://github.com/apache/datafusion/issues/21835>.+ ///+ /// The `try_` prefix matches the fallible `try_from_proto` decode+ /// constructors (and the `TryFromProto` trait in `datafusion-proto`);+ /// both sides of the round-trip are fallible and named consistently.+ ///+ /// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode+ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ _ctx: &proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {+ Ok(None)+ }+}++/// Encode-side context for [`PhysicalExpr::try_to_proto`].+///+/// Expression authors only ever see [`proto_encode::PhysicalExprEncodeCtx`]:+/// a concrete struct with stable methods. Internally it dispatches to a+/// [`proto_encode::PhysicalExprEncode`] implementor that lives in+/// `datafusion-proto`, which is what lets `physical-expr-common` stay free+/// of `datafusion-proto` as a dep.+///+/// More specialized helpers (e.g. encoding UDFs/UDAFs/UDWFs through the+/// extension codec) can be added to the context as expressions migrate;+/// today they're not required because the encoder forwards to the existing+/// codec via the proto converter.+#[cfg(feature = "proto")]+pub mod proto_encode {+ use std::sync::Arc;++ use datafusion_common::Result;+ use datafusion_proto_models::protobuf::PhysicalExprNode;++ use super::PhysicalExpr;++ /// Encoder context handed to [`super::PhysicalExpr::try_to_proto`].+ ///+ /// Wraps an internal [`PhysicalExprEncode`] trait object so callers see a+ /// stable concrete type while implementations can evolve in+ /// `datafusion-proto`.+ pub struct PhysicalExprEncodeCtx<'a> {+ encoder: &'a dyn PhysicalExprEncode,+ }++ impl<'a> PhysicalExprEncodeCtx<'a> {+ /// Construct a new encode context. Typically called by+ /// `datafusion-proto`; expression authors receive `&PhysicalExprEncodeCtx`.+ pub fn new(encoder: &'a dyn PhysicalExprEncode) -> Self {+ Self { encoder }+ }++ /// Encode a child expression. Routes through the configured encoder+ /// so dedup-aware encoding is preserved.+ pub fn encode_child(+ &self,+ expr: &Arc<dyn PhysicalExpr>,+ ) -> Result<PhysicalExprNode> {+ self.encoder.encode(expr)+ }+ }++ /// Internal dispatch trait. Implementors live in `datafusion-proto` and+ /// wrap the existing `PhysicalExtensionCodec` ++ /// `PhysicalProtoConverterExtension` plumbing. Expression authors should+ /// use [`PhysicalExprEncodeCtx`] instead of calling this directly.+ pub trait PhysicalExprEncode {+ /// Encode an expression to a protobuf node.+ fn encode(&self, expr: &Arc<dyn PhysicalExpr>) -> Result<PhysicalExprNode>;+ }+}++/// Decode-side counterpart to [`proto_encode`].+///+/// Expression authors implement an associated `try_from_proto` on their+/// concrete type, with the signature+///+/// ```ignore+/// fn try_from_proto(+/// node: &PhysicalExprNode,+/// ctx: &PhysicalExprDecodeCtx<'_>,+/// ) -> Result<Arc<dyn PhysicalExpr>>+/// ```+///+/// It takes the whole [`PhysicalExprNode`] — the exact inverse of what+/// [`PhysicalExpr::try_to_proto`] returns — so the constructor can also see+/// outer-node fields such as `expr_id`. The central match in+/// `datafusion-proto` dispatches `ExprType` variants to these constructors.+///+/// As with the encode side, the public surface is a struct (not a `&dyn`+/// trait) so future fields/helpers (registries for third-party expressions,+/// schema-resolution caches, etc.) can be added without changing the+/// signature every expression depends on.+///+/// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode+#[cfg(feature = "proto")]+pub mod proto_decode {+ use std::sync::Arc;++ use arrow::datatypes::Schema;+ use datafusion_common::Result;+ use datafusion_proto_models::protobuf::PhysicalExprNode;++ use super::PhysicalExpr;++ /// Decoder context handed to per-expression `try_from_proto` constructors.+ ///+ /// Wraps an internal [`PhysicalExprDecode`] trait object plus a borrowed+ /// schema. The trait stays an implementation detail of `datafusion-proto`;+ /// expression authors only see this struct.+ pub struct PhysicalExprDecodeCtx<'a> {+ schema: &'a Schema,+ decoder: &'a dyn PhysicalExprDecode,+ }++ impl<'a> PhysicalExprDecodeCtx<'a> {+ /// Construct a new decode context. Typically called by+ /// `datafusion-proto`; expression authors receive+ /// `&PhysicalExprDecodeCtx`.+ pub fn new(schema: &'a Schema, decoder: &'a dyn PhysicalExprDecode) -> Self {+ Self { schema, decoder }+ }++ /// The schema bound to this decode context. Use it for column lookups,+ /// data-type resolution, etc.+ pub fn schema(&self) -> &Schema {+ self.schema+ }++ /// Decode an expression node, recursing into child sub-expressions.+ ///+ /// Routes built-in `ExprType` variants through `datafusion-proto`'s+ /// central match and forwards extension nodes to the registered codec+ /// (today via [`PhysicalExtensionCodec::try_decode_expr`]; later via+ /// a per-type registry — see #21835).+ ///+ /// [`PhysicalExtensionCodec::try_decode_expr`]: https://docs.rs/datafusion-proto/latest/datafusion_proto/physical_plan/trait.PhysicalExtensionCodec.html#method.try_decode_expr+ pub fn decode(&self, node: &PhysicalExprNode) -> Result<Arc<dyn PhysicalExpr>> {+ self.decoder.decode(node, self.schema)+ }+ }++ /// Internal dispatch trait. Implementors live in `datafusion-proto`.+ /// Expression authors should use [`PhysicalExprDecodeCtx`] instead of+ /// calling this directly.+ pub trait PhysicalExprDecode {+ /// Decode a proto node into a concrete `PhysicalExpr`. The schema is+ /// passed alongside so implementations can support recursive children+ /// and rebind the context per call (e.g. for nested plans).+ fn decode(+ &self,+ node: &PhysicalExprNode,+ schema: &Schema,+ ) -> Result<Arc<dyn PhysicalExpr>>;+ }}#[deprecated(diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.tomlindex b755353d75..65ef2a3ceb 100644--- a/datafusion/physical-expr/Cargo.toml+++ b/datafusion/physical-expr/Cargo.toml@@ -42,6 +42,12 @@ name = "datafusion_physical_expr"[features]recursive_protection = ["dep:recursive"]+# Forwards the `proto` feature to `datafusion-physical-expr-common`, exposing+# `PhysicalExpr::to_proto` and letting expressions in this crate implement it.+proto = [+ "dep:datafusion-proto-models",+ "datafusion-physical-expr-common/proto",+][dependencies]arrow = { workspace = true }@@ -50,6 +56,7 @@ datafusion-expr = { workspace = true }datafusion-expr-common = { workspace = true }datafusion-functions-aggregate-common = { workspace = true }datafusion-physical-expr-common = { workspace = true }+datafusion-proto-models = { workspace = true, optional = true }hashbrown = { workspace = true }indexmap = { workspace = true }itertools = { workspace = true, features = ["use_std"] }diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rsindex b92668fe9b..712f8f58f3 100644--- a/datafusion/physical-expr/src/expressions/binary.rs+++ b/datafusion/physical-expr/src/expressions/binary.rs@@ -610,6 +610,124 @@ impl PhysicalExpr for BinaryExpr {write!(f, " {} ", self.op)?;write_child(f, self.right.as_ref(), precedence)}++ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {+ use datafusion_proto_models::protobuf;++ // Linearize a nested binary expression tree of the same operator+ // into a flat vector of operands to avoid deep recursion in proto.+ let op = self.op;+ let mut operand_refs: Vec<&Arc<dyn PhysicalExpr>> = vec![&self.right];+ let mut current_expr: &BinaryExpr = self;+ loop {+ match current_expr.left.downcast_ref::<BinaryExpr>() {+ Some(bin) if bin.op == op => {+ operand_refs.push(&bin.right);+ current_expr = bin;+ }+ _ => {+ operand_refs.push(¤t_expr.left);+ break;+ }+ }+ }+ // Reverse so operands are ordered from left innermost to right outermost.+ operand_refs.reverse();++ let operands = operand_refs+ .iter()+ .map(|e| ctx.encode_child(e))+ .collect::<Result<Vec<_>>>()?;++ Ok(Some(protobuf::PhysicalExprNode {+ expr_id: None,+ expr_type: Some(protobuf::physical_expr_node::ExprType::BinaryExpr(+ Box::new(protobuf::PhysicalBinaryExprNode {+ l: None,+ r: None,+ op: format!("{op:?}"),+ operands,+ }),+ )),+ }))+ }+}++#[cfg(feature = "proto")]+impl BinaryExpr {+ /// Reconstruct a [`BinaryExpr`] (or a left-deep tree of them when the proto+ /// uses the linearized `operands` form) from its protobuf representation.+ ///+ /// Takes the whole [`PhysicalExprNode`] — the exact inverse of what+ /// [`PhysicalExpr::try_to_proto`] produces — so every expression's+ /// `try_from_proto` shares one signature. The operator string is parsed+ /// via the canonical [`Operator::from_proto_name`] mapping, so no `op`+ /// argument needs to be threaded in by the caller.+ ///+ /// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode+ /// [`PhysicalExpr::try_to_proto`]: datafusion_physical_expr_common::physical_expr::PhysicalExpr::try_to_proto+ /// [`PhysicalExprDecodeCtx::decode`]: datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx::decode+ pub fn try_from_proto(+ node: &datafusion_proto_models::protobuf::PhysicalExprNode,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ use datafusion_proto_models::protobuf;+ let node = match &node.expr_type {+ Some(protobuf::physical_expr_node::ExprType::BinaryExpr(b)) => b.as_ref(),+ _ => return internal_err!("PhysicalExprNode is not a BinaryExpr"),+ };+ let op = Operator::from_proto_name(&node.op).ok_or_else(|| {+ datafusion_common::DataFusionError::Internal(format!(+ "Unsupported binary operator '{}'",+ node.op+ ))+ })?;++ if !node.operands.is_empty() {+ // New linearized format: reduce the flat operands list back into+ // a nested binary expression tree.+ let operands = node+ .operands+ .iter()+ .map(|e| ctx.decode(e))+ .collect::<Result<Vec<_>>>()?;++ if operands.len() < 2 {+ return Err(datafusion_common::DataFusionError::Internal(+ "A binary expression must always have at least 2 operands"+ .to_string(),+ ));+ }++ Ok(operands+ .into_iter()+ .reduce(|left, right| {+ Arc::new(BinaryExpr::new(left, op, right)) as Arc<dyn PhysicalExpr>+ })+ .expect("Binary expression could not be reduced to a single expression."))+ } else {+ // Legacy format with l/r fields.+ let left = node.l.as_deref().ok_or_else(|| {+ datafusion_common::DataFusionError::Internal(+ "BinaryExpr is missing required field 'left'".to_string(),+ )+ })?;+ let right = node.r.as_deref().ok_or_else(|| {+ datafusion_common::DataFusionError::Internal(+ "BinaryExpr is missing required field 'right'".to_string(),+ )+ })?;+ Ok(Arc::new(BinaryExpr::new(+ ctx.decode(left)?,+ op,+ ctx.decode(right)?,+ )))+ }+ }}/// Casts dictionary array to result type for binary numerical operators. Such operatorsdiff --git a/datafusion/physical-expr/src/expressions/column.rs b/datafusion/physical-expr/src/expressions/column.rsindex 7d4b0e7e2f..2b1de870e7 100644--- a/datafusion/physical-expr/src/expressions/column.rs+++ b/datafusion/physical-expr/src/expressions/column.rs@@ -146,6 +146,49 @@ impl PhysicalExpr for Column {fn placement(&self) -> ExpressionPlacement {ExpressionPlacement::Column}++ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ _ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {+ use datafusion_proto_models::protobuf;+ Ok(Some(protobuf::PhysicalExprNode {+ expr_id: None,+ expr_type: Some(protobuf::physical_expr_node::ExprType::Column(+ protobuf::PhysicalColumn {+ name: self.name.clone(),+ index: self.index as u32,+ },+ )),+ }))+ }+}++#[cfg(feature = "proto")]+impl Column {+ /// Reconstruct a [`Column`] from its protobuf representation.+ ///+ /// Takes the whole [`PhysicalExprNode`] — the exact inverse of what+ /// [`PhysicalExpr::try_to_proto`] produces — so every expression's+ /// `try_from_proto` shares one signature. The decode context is currently+ /// unused, but is threaded through so that future expressions with child+ /// sub-expressions can recurse via [`PhysicalExprDecodeCtx::decode`].+ ///+ /// [`PhysicalExprNode`]: datafusion_proto_models::protobuf::PhysicalExprNode+ /// [`PhysicalExpr::try_to_proto`]: datafusion_physical_expr_common::physical_expr::PhysicalExpr::try_to_proto+ /// [`PhysicalExprDecodeCtx::decode`]: datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx::decode+ pub fn try_from_proto(+ node: &datafusion_proto_models::protobuf::PhysicalExprNode,+ _ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ use datafusion_proto_models::protobuf;+ let protobuf::PhysicalColumn { name, index } = match &node.expr_type {+ Some(protobuf::physical_expr_node::ExprType::Column(c)) => c,+ _ => return internal_err!("PhysicalExprNode is not a Column"),+ };+ Ok(Arc::new(Column::new(name, *index as usize)))+ }}impl Column {diff --git a/datafusion/proto-models/Cargo.toml b/datafusion/proto-models/Cargo.tomlnew file mode 100644index 0000000000..e37c4a2dba--- /dev/null+++ b/datafusion/proto-models/Cargo.toml@@ -0,0 +1,45 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++[package]+name = "datafusion-proto-models"+description = "Protobuf-generated model types for DataFusion logical and physical plans"+keywords = ["arrow", "query", "sql"]+readme = "README.md"+version = { workspace = true }+edition = { workspace = true }+homepage = { workspace = true }+repository = { workspace = true }+license = { workspace = true }+authors = { workspace = true }+rust-version = { workspace = true }++[package.metadata.docs.rs]+all-features = true++[lib]+name = "datafusion_proto_models"++[features]+default = []+json = ["serde", "pbjson", "datafusion-proto-common/json"]++[dependencies]+datafusion-proto-common = { workspace = true }+pbjson = { workspace = true, optional = true }+prost = { workspace = true }+serde = { version = "1.0", optional = true }diff --git a/datafusion/proto-models/LICENSE.txt b/datafusion/proto-models/LICENSE.txtnew file mode 100644index 0000000000..d74c6b599d--- /dev/null+++ b/datafusion/proto-models/LICENSE.txt@@ -0,0 +1,212 @@++ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+++This project includes code from Apache Aurora.++* dev/release/{release,changelog,release-candidate} are based on the scripts from+ Apache Aurora++Copyright: 2016 The Apache Software Foundation.+Home page: https://aurora.apache.org/+License: http://www.apache.org/licenses/LICENSE-2.0diff --git a/datafusion/proto-models/NOTICE.txt b/datafusion/proto-models/NOTICE.txtnew file mode 100644index 0000000000..0bd2d52368--- /dev/null+++ b/datafusion/proto-models/NOTICE.txt@@ -0,0 +1,5 @@+Apache DataFusion+Copyright 2019-2026 The Apache Software Foundation++This product includes software developed at+The Apache Software Foundation (http://www.apache.org/).diff --git a/datafusion/proto-models/README.md b/datafusion/proto-models/README.mdnew file mode 100644index 0000000000..34adae9c1a--- /dev/null+++ b/datafusion/proto-models/README.md@@ -0,0 +1,43 @@+<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# Apache DataFusion Protobuf Models++[Apache DataFusion] is an extensible query execution framework, written in Rust, that uses [Apache Arrow] as its in-memory format.++This crate contains the [prost]-generated Rust types for DataFusion's logical+and physical plan protobuf schemas. It is intentionally kept narrow: it has no+DataFusion dependencies beyond [`datafusion-proto-common`] and exposes only the+generated structs (and optional [pbjson]/[serde] support).++This crate is consumed by [`datafusion-proto`] and may also be depended on+directly by other DataFusion crates that need to refer to the proto schema+types without pulling in the full [`datafusion-proto`] surface.++Most projects should use the [`datafusion-proto`] crate directly, which+re-exports this module. If you are already using the [`datafusion-proto`]+crate, there is no reason to use this crate directly in your project as well.++[apache arrow]: https://arrow.apache.org/+[apache datafusion]: https://datafusion.apache.org/+[prost]: https://docs.rs/prost/latest/prost/+[pbjson]: https://docs.rs/pbjson/latest/pbjson/+[serde]: https://serde.rs/+[`datafusion-proto`]: https://crates.io/crates/datafusion-proto+[`datafusion-proto-common`]: https://crates.io/crates/datafusion-proto-commondiff --git a/datafusion/proto/gen/Cargo.toml b/datafusion/proto-models/gen/Cargo.tomlsimilarity index 100%rename from datafusion/proto/gen/Cargo.tomlrename to datafusion/proto-models/gen/Cargo.tomldiff --git a/datafusion/proto/gen/src/main.rs b/datafusion/proto-models/gen/src/main.rssimilarity index 92%rename from datafusion/proto/gen/src/main.rsrename to datafusion/proto-models/gen/src/main.rsindex 7f16316203..4da674c43c 100644--- a/datafusion/proto/gen/src/main.rs+++ b/datafusion/proto-models/gen/src/main.rs@@ -18,9 +18,9 @@use std::path::Path;fn main() -> Result<(), String> {- let proto_dir = Path::new("datafusion/proto");- let proto_path = Path::new("datafusion/proto/proto/datafusion.proto");- let out_dir = Path::new("datafusion/proto/src");+ let proto_dir = Path::new("datafusion/proto-models");+ let proto_path = Path::new("datafusion/proto-models/proto/datafusion.proto");+ let out_dir = Path::new("datafusion/proto-models/src");// proto definitions has to be therelet descriptor_path = proto_dir.join("proto/proto_descriptor.bin");diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protosimilarity index 100%rename from datafusion/proto/proto/datafusion.protorename to datafusion/proto-models/proto/datafusion.protodiff --git a/datafusion/proto-models/regen.sh b/datafusion/proto-models/regen.shnew file mode 100755index 0000000000..4bb07a1f32--- /dev/null+++ b/datafusion/proto-models/regen.sh@@ -0,0 +1,21 @@+#!/usr/bin/env bash++# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++repo_root=$(git rev-parse --show-toplevel)+cd "$repo_root" && cargo run --manifest-path datafusion/proto-models/gen/Cargo.tomldiff --git a/datafusion/proto/src/generated/datafusion_proto_common.rs b/datafusion/proto-models/src/generated/datafusion_proto_common.rssimilarity index 100%rename from datafusion/proto/src/generated/datafusion_proto_common.rsrename to datafusion/proto-models/src/generated/datafusion_proto_common.rsdiff --git a/datafusion/proto/src/generated/mod.rs b/datafusion/proto-models/src/generated/mod.rssimilarity index 100%rename from datafusion/proto/src/generated/mod.rsrename to datafusion/proto-models/src/generated/mod.rsdiff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rssimilarity index 100%rename from datafusion/proto/src/generated/pbjson.rsrename to datafusion/proto-models/src/generated/pbjson.rsdiff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rssimilarity index 100%rename from datafusion/proto/src/generated/prost.rsrename to datafusion/proto-models/src/generated/prost.rsdiff --git a/datafusion/proto-models/src/lib.rs b/datafusion/proto-models/src/lib.rsnew file mode 100644index 0000000000..8f845a8a99--- /dev/null+++ b/datafusion/proto-models/src/lib.rs@@ -0,0 +1,54 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++#![doc(+ html_logo_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg",+ html_favicon_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg"+)]+#![cfg_attr(docsrs, feature(doc_cfg))]+// Make sure fast / cheap clones on Arc are explicit:+// https://github.com/apache/datafusion/issues/11143+#![deny(clippy::clone_on_ref_ptr)]++//! `prost`-generated DataFusion protobuf model types.+//!+//! This crate contains only the generated structs for DataFusion's logical and+//! physical plan protobuf schemas (see `proto/datafusion.proto`). It is the+//! schema source of truth for [`datafusion-proto`] and intentionally has no+//! DataFusion dependencies beyond [`datafusion-proto-common`].+//!+//! Most users should depend on [`datafusion-proto`] instead, which re-exports+//! these types under [`datafusion_proto::protobuf`].+//!+//! [`datafusion-proto`]: https://crates.io/crates/datafusion-proto+//! [`datafusion-proto-common`]: https://crates.io/crates/datafusion-proto-common+//! [`datafusion_proto::protobuf`]: https://docs.rs/datafusion-proto/latest/datafusion_proto/protobuf/index.html++pub mod generated;++/// All DataFusion protobuf model types.+///+/// Includes both the types declared in `datafusion.proto` and the+/// `datafusion_proto_common` types it imports, in a single flat namespace+/// so consumers can `use datafusion_proto_models::protobuf::*;`.+pub mod protobuf {+ pub use crate::generated::datafusion::*;+}++/// Re-export of the `datafusion_proto_common` types as exposed through this+/// crate's generated module, for callers that want the common-only namespace.+pub use generated::datafusion_common;diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex 4484846813..53e7cd78dc 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -36,7 +36,11 @@ name = "datafusion_proto"[features]default = ["parquet"]-json = ["pbjson", "serde", "serde_json", "datafusion-proto-common/json"]+json = [+ "serde_json",+ "datafusion-proto-common/json",+ "datafusion-proto-models/json",+]parquet = ["datafusion-datasource-parquet", "datafusion-common/parquet", "datafusion/parquet"]avro = ["datafusion-datasource-avro"]@@ -59,14 +63,13 @@ datafusion-datasource-parquet = { workspace = true, optional = true }datafusion-execution = { workspace = true }datafusion-expr = { workspace = true }datafusion-functions-table = { workspace = true }-datafusion-physical-expr = { workspace = true }-datafusion-physical-expr-common = { workspace = true }+datafusion-physical-expr = { workspace = true, features = ["proto"] }+datafusion-physical-expr-common = { workspace = true, features = ["proto"] }datafusion-physical-plan = { workspace = true }datafusion-proto-common = { workspace = true }+datafusion-proto-models = { workspace = true }object_store = { workspace = true }-pbjson = { workspace = true, optional = true }prost = { workspace = true }-serde = { version = "1.0", optional = true }serde_json = { workspace = true, optional = true }[dev-dependencies]diff --git a/datafusion/proto/regen.sh b/datafusion/proto/regen.shindex 02970a90ad..c4bcea9ff5 100755--- a/datafusion/proto/regen.sh+++ b/datafusion/proto/regen.sh@@ -17,5 +17,7 @@# specific language governing permissions and limitations# under the License.+# The proto schema and code generation now live in `datafusion-proto-models`.+# This script is kept as a convenience wrapper.repo_root=$(git rev-parse --show-toplevel)-cd "$repo_root" && cargo run --manifest-path datafusion/proto/gen/Cargo.toml+exec "$repo_root/datafusion/proto-models/regen.sh"diff --git a/datafusion/proto/src/common.rs b/datafusion/proto/src/common.rsindex 22ded708d8..bff017edbc 100644--- a/datafusion/proto/src/common.rs+++ b/datafusion/proto/src/common.rs@@ -47,6 +47,24 @@ macro_rules! convert_required {}};}+/// Like [`convert_required`] but for types whose proto conversion goes through+/// the [`TryFromProto`](crate::convert::TryFromProto) trait instead of+/// [`TryFrom`]. Required because some prost-generated types now live in a+/// separate crate, so `TryFrom`/`From` cannot be implemented on foreign-foreign+/// pairs from `datafusion-proto` directly.+#[macro_export]+macro_rules! convert_required_proto {+ ($T:ty, $PB:expr) => {{+ if let Some(field) = $PB.as_ref() {+ Ok::<$T, _>(<$T as $crate::convert::TryFromProto<_>>::try_from_proto(+ field,+ )?)+ } else {+ Err(proto_error("Missing required field in protobuf"))+ }+ }};+}+#[macro_export]macro_rules! into_required {($PB:expr) => {{diff --git a/datafusion/proto/src/convert.rs b/datafusion/proto/src/convert.rsnew file mode 100644index 0000000000..cb5c5bd7f8--- /dev/null+++ b/datafusion/proto/src/convert.rs@@ -0,0 +1,44 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Conversion traits between proto-generated types and DataFusion types.+//!+//! The `prost`-generated structs now live in `datafusion-proto-models`, while+//! their counterparts (`StringifiedPlan`, `JoinType`, `WindowFrame`, ...) live+//! in `datafusion-common` / `datafusion-expr` / `datafusion-datasource` etc.+//! Both sides are foreign to `datafusion-proto`, which means the orphan rule+//! forbids a direct `impl From<&protobuf::X> for Y` written here.+//!+//! To keep the conversion logic colocated with serialization while satisfying+//! the orphan rule, we route those conversions through the `FromProto` /+//! `TryFromProto` traits defined in this module. Their signatures mirror the+//! standard library's `From` / `TryFrom`, so callers spell the conversion+//! `Y::from_proto(&p)` / `Y::try_from_proto(&p)?` instead of+//! `(&p).into()` / `(&p).try_into()?`.++/// Infallible conversion from a proto value into a DataFusion value (or vice+/// versa). Mirrors [`From`].+pub trait FromProto<T>: Sized {+ fn from_proto(value: T) -> Self;+}++/// Fallible conversion from a proto value into a DataFusion value (or vice+/// versa). Mirrors [`TryFrom`].+pub trait TryFromProto<T>: Sized {+ type Error;+ fn try_from_proto(value: T) -> std::result::Result<Self, Self::Error>;+}diff --git a/datafusion/proto/src/generated/datafusion.rs b/datafusion/proto/src/generated/datafusion.rsdeleted file mode 100644index 8b13789179..0000000000--- a/datafusion/proto/src/generated/datafusion.rs+++ /dev/null@@ -1 +0,0 @@-diff --git a/datafusion/proto/src/lib.rs b/datafusion/proto/src/lib.rsindex 7ddc930fa2..0e63bcf5f5 100644--- a/datafusion/proto/src/lib.rs+++ b/datafusion/proto/src/lib.rs@@ -123,12 +123,13 @@//! ```pub mod bytes;pub mod common;-pub mod generated;+pub mod convert;pub mod logical_plan;pub mod physical_plan;+pub use convert::{FromProto, TryFromProto};+pub mod protobuf {- pub use crate::generated::datafusion::*;pub use datafusion_proto_common::common::proto_error;pub use datafusion_proto_common::protobuf_common::{ArrowFormat, ArrowOptions, ArrowType, AvroFormat, AvroOptions, CsvFormat,@@ -136,6 +137,40 @@ pub mod protobuf {ScalarValue, Schema,};pub use datafusion_proto_common::{FromProtoError, ToProtoError};+ // Re-export every type from `datafusion-proto-models`'s generated module+ // so the existing `datafusion_proto::protobuf::Foo` paths keep resolving.+ // Going through the deeper `generated::datafusion` path (rather than+ // `datafusion_proto_models::protobuf`, which is itself a `pub use ::*`)+ // avoids a double wildcard re-export that some tools (cargo-semver-checks)+ // don't follow.+ pub use datafusion_proto_models::generated::datafusion::*;+}++/// Backwards-compatible re-export of the moved generated types.+///+/// The prost-generated structs now live in `datafusion-proto-models`;+/// this module preserves the legacy `datafusion_proto::generated::*` paths+/// for downstream callers. Prefer the [`protobuf`] module (or+/// [`datafusion_proto_models`] directly) in new code.+#[deprecated(+ since = "53.1.0",+ note = "use `datafusion_proto::protobuf` (or `datafusion_proto_models::protobuf`) instead"+)]+pub mod generated {+ /// Re-export of the prost-generated types defined in `datafusion.proto`.+ #[deprecated(+ since = "53.1.0",+ note = "use `datafusion_proto::protobuf` (or `datafusion_proto_models::protobuf`) instead"+ )]+ pub use datafusion_proto_models::generated::datafusion;++ /// Re-export of the prost-generated common types defined in+ /// `datafusion_common.proto`.+ #[deprecated(+ since = "53.1.0",+ note = "use `datafusion_proto_common::protobuf_common` instead"+ )]+ pub use datafusion_proto_models::generated::datafusion_common;}#[cfg(doctest)]diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rsindex 683b6a612a..d5af7be485 100644--- a/datafusion/proto/src/logical_plan/file_formats.rs+++ b/datafusion/proto/src/logical_plan/file_formats.rs@@ -18,6 +18,7 @@use std::sync::Arc;use super::LogicalExtensionCodec;+use crate::convert::FromProto;use crate::protobuf::{CsvOptions as CsvOptionsProto, CsvQuoteStyle as CsvQuoteStyleProto,JsonOptions as JsonOptionsProto,@@ -37,8 +38,8 @@ use prost::Message;#[derive(Debug)]pub struct CsvLogicalExtensionCodec;-impl CsvOptionsProto {- fn from_factory(factory: &CsvFormatFactory) -> Self {+impl FromProto<&CsvFormatFactory> for CsvOptionsProto {+ fn from_proto(factory: &CsvFormatFactory) -> Self {if let Some(options) = &factory.options {CsvOptionsProto {has_header: options.has_header.map_or(vec![], |v| vec![v as u8]),@@ -79,8 +80,8 @@ impl CsvOptionsProto {}}-impl From<&CsvOptionsProto> for CsvOptions {- fn from(proto: &CsvOptionsProto) -> Self {+impl FromProto<&CsvOptionsProto> for CsvOptions {+ fn from_proto(proto: &CsvOptionsProto) -> Self {CsvOptions {has_header: if !proto.has_header.is_empty() {Some(proto.has_header[0] != 0)@@ -230,7 +231,7 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {let proto = CsvOptionsProto::decode(buf).map_err(|e| {exec_datafusion_err!("Failed to decode CsvOptionsProto: {e:?}")})?;- let options: CsvOptions = (&proto).into();+ let options = CsvOptions::from_proto(&proto);Ok(Arc::new(CsvFormatFactory {options: Some(options),}))@@ -247,7 +248,7 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {return exec_err!("{}", "Unsupported FileFormatFactory type".to_string());};- let proto = CsvOptionsProto::from_factory(&CsvFormatFactory {+ let proto = CsvOptionsProto::from_proto(&CsvFormatFactory {options: Some(options),});@@ -259,8 +260,8 @@ impl LogicalExtensionCodec for CsvLogicalExtensionCodec {}}-impl JsonOptionsProto {- fn from_factory(factory: &JsonFormatFactory) -> Self {+impl FromProto<&JsonFormatFactory> for JsonOptionsProto {+ fn from_proto(factory: &JsonFormatFactory) -> Self {if let Some(options) = &factory.options {JsonOptionsProto {compression: options.compression as i32,@@ -274,8 +275,8 @@ impl JsonOptionsProto {}}-impl From<&JsonOptionsProto> for JsonOptions {- fn from(proto: &JsonOptionsProto) -> Self {+impl FromProto<&JsonOptionsProto> for JsonOptions {+ fn from_proto(proto: &JsonOptionsProto) -> Self {JsonOptions {compression: match proto.compression {0 => CompressionTypeVariant::GZIP,@@ -340,7 +341,7 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {let proto = JsonOptionsProto::decode(buf).map_err(|e| {exec_datafusion_err!("Failed to decode JsonOptionsProto: {e:?}")})?;- let options: JsonOptions = (&proto).into();+ let options = JsonOptions::from_proto(&proto);Ok(Arc::new(JsonFormatFactory {options: Some(options),}))@@ -358,7 +359,7 @@ impl LogicalExtensionCodec for JsonLogicalExtensionCodec {return exec_err!("Unsupported FileFormatFactory type");};- let proto = JsonOptionsProto::from_factory(&JsonFormatFactory {+ let proto = JsonOptionsProto::from_proto(&JsonFormatFactory {options: Some(options),});@@ -385,8 +386,8 @@ mod parquet {};use datafusion_datasource_parquet::file_format::ParquetFormatFactory;- impl TableParquetOptionsProto {- fn from_factory(factory: &ParquetFormatFactory) -> Self {+ impl FromProto<&ParquetFormatFactory> for TableParquetOptionsProto {+ fn from_proto(factory: &ParquetFormatFactory) -> Self {let global_options = if let Some(ref options) = factory.options {options.clone()} else {@@ -499,8 +500,8 @@ mod parquet {}}- impl From<&ParquetOptionsProto> for ParquetOptions {- fn from(proto: &ParquetOptionsProto) -> Self {+ impl FromProto<&ParquetOptionsProto> for ParquetOptions {+ fn from_proto(proto: &ParquetOptionsProto) -> Self {ParquetOptions {enable_page_index: proto.enable_page_index,pruning: proto.pruning,@@ -577,8 +578,8 @@ mod parquet {}}- impl From<ParquetColumnOptionsProto> for ParquetColumnOptions {- fn from(proto: ParquetColumnOptionsProto) -> Self {+ impl FromProto<ParquetColumnOptionsProto> for ParquetColumnOptions {+ fn from_proto(proto: ParquetColumnOptionsProto) -> Self {ParquetColumnOptions {bloom_filter_enabled: proto.bloom_filter_enabled_opt.map(|parquet_column_options::BloomFilterEnabledOpt::BloomFilterEnabled(v)| v,@@ -605,13 +606,13 @@ mod parquet {}}- impl From<&TableParquetOptionsProto> for TableParquetOptions {- fn from(proto: &TableParquetOptionsProto) -> Self {+ impl FromProto<&TableParquetOptionsProto> for TableParquetOptions {+ fn from_proto(proto: &TableParquetOptionsProto) -> Self {TableParquetOptions {global: proto.global.as_ref()- .map(ParquetOptions::from)+ .map(ParquetOptions::from_proto).unwrap_or_default(),column_specific_options: proto.column_specific_options@@ -619,7 +620,7 @@ mod parquet {.map(|parquet_column_options| {(parquet_column_options.column_name.clone(),- ParquetColumnOptions::from(+ ParquetColumnOptions::from_proto(parquet_column_options.options.clone()@@ -688,7 +689,7 @@ mod parquet {let proto = TableParquetOptionsProto::decode(buf).map_err(|e| {exec_datafusion_err!("Failed to decode TableParquetOptionsProto: {e:?}")})?;- let options: TableParquetOptions = (&proto).into();+ let options = TableParquetOptions::from_proto(&proto);Ok(Arc::new(datafusion_datasource_parquet::file_format::ParquetFormatFactory {options: Some(options),@@ -711,7 +712,7 @@ mod parquet {return exec_err!("Unsupported FileFormatFactory type");};- let proto = TableParquetOptionsProto::from_factory(&ParquetFormatFactory {+ let proto = TableParquetOptionsProto::from_proto(&ParquetFormatFactory {options: Some(options),});diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rsindex 78ffd362c8..c68b83964f 100644--- a/datafusion/proto/src/logical_plan/from_proto.rs+++ b/datafusion/proto/src/logical_plan/from_proto.rs@@ -54,10 +54,12 @@ use crate::protobuf::{},};+use crate::convert::{FromProto, TryFromProto};+use super::{AsLogicalPlan, LogicalExtensionCodec};-impl From<&protobuf::UnnestOptions> for UnnestOptions {- fn from(opts: &protobuf::UnnestOptions) -> Self {+impl FromProto<&protobuf::UnnestOptions> for UnnestOptions {+ fn from_proto(opts: &protobuf::UnnestOptions) -> Self {Self {preserve_nulls: opts.preserve_nulls,recursions: opts@@ -73,8 +75,8 @@ impl From<&protobuf::UnnestOptions> for UnnestOptions {}}-impl From<protobuf::WindowFrameUnits> for WindowFrameUnits {- fn from(units: protobuf::WindowFrameUnits) -> Self {+impl FromProto<protobuf::WindowFrameUnits> for WindowFrameUnits {+ fn from_proto(units: protobuf::WindowFrameUnits) -> Self {match units {protobuf::WindowFrameUnits::Rows => Self::Rows,protobuf::WindowFrameUnits::Range => Self::Range,@@ -83,10 +85,10 @@ impl From<protobuf::WindowFrameUnits> for WindowFrameUnits {}}-impl TryFrom<protobuf::TableReference> for TableReference {+impl TryFromProto<protobuf::TableReference> for TableReference {type Error = Error;- fn try_from(value: protobuf::TableReference) -> Result<Self, Self::Error> {+ fn try_from_proto(value: protobuf::TableReference) -> Result<Self, Self::Error> {use protobuf::table_reference::TableReferenceEnum;let table_reference_enum = value.table_reference_enum@@ -109,8 +111,8 @@ impl TryFrom<protobuf::TableReference> for TableReference {}}-impl From<&protobuf::StringifiedPlan> for StringifiedPlan {- fn from(stringified_plan: &protobuf::StringifiedPlan) -> Self {+impl FromProto<&protobuf::StringifiedPlan> for StringifiedPlan {+ fn from_proto(stringified_plan: &protobuf::StringifiedPlan) -> Self {Self {plan_type: match stringified_plan.plan_type@@ -152,19 +154,25 @@ impl From<&protobuf::StringifiedPlan> for StringifiedPlan {}}-impl TryFrom<protobuf::WindowFrame> for WindowFrame {+impl TryFromProto<protobuf::WindowFrame> for WindowFrame {type Error = Error;- fn try_from(window: protobuf::WindowFrame) -> Result<Self, Self::Error> {- let units = protobuf::WindowFrameUnits::try_from(window.window_frame_units)- .map_err(|_| Error::unknown("WindowFrameUnits", window.window_frame_units))?- .into();- let start_bound = window.start_bound.required("start_bound")?;+ fn try_from_proto(window: protobuf::WindowFrame) -> Result<Self, Self::Error> {+ let units = WindowFrameUnits::from_proto(+ protobuf::WindowFrameUnits::try_from(window.window_frame_units).map_err(+ |_| Error::unknown("WindowFrameUnits", window.window_frame_units),+ )?,+ );+ let start_bound = WindowFrameBound::try_from_proto(+ window+ .start_bound+ .ok_or_else(|| Error::required("start_bound"))?,+ )?;let end_bound = window.end_bound.map(|end_bound| match end_bound {protobuf::window_frame::EndBound::Bound(end_bound) => {- end_bound.try_into()+ WindowFrameBound::try_from_proto(end_bound)}}).transpose()?@@ -173,10 +181,10 @@ impl TryFrom<protobuf::WindowFrame> for WindowFrame {}}-impl TryFrom<protobuf::WindowFrameBound> for WindowFrameBound {+impl TryFromProto<protobuf::WindowFrameBound> for WindowFrameBound {type Error = Error;- fn try_from(bound: protobuf::WindowFrameBound) -> Result<Self, Self::Error> {+ fn try_from_proto(bound: protobuf::WindowFrameBound) -> Result<Self, Self::Error> {let bound_type =protobuf::WindowFrameBoundType::try_from(bound.window_frame_bound_type).map_err(|_| {@@ -196,8 +204,8 @@ impl TryFrom<protobuf::WindowFrameBound> for WindowFrameBound {}}-impl From<protobuf::JoinType> for JoinType {- fn from(t: protobuf::JoinType) -> Self {+impl FromProto<protobuf::JoinType> for JoinType {+ fn from_proto(t: protobuf::JoinType) -> Self {match t {protobuf::JoinType::Inner => JoinType::Inner,protobuf::JoinType::Left => JoinType::Left,@@ -213,8 +221,8 @@ impl From<protobuf::JoinType> for JoinType {}}-impl From<protobuf::JoinConstraint> for JoinConstraint {- fn from(t: protobuf::JoinConstraint) -> Self {+impl FromProto<protobuf::JoinConstraint> for JoinConstraint {+ fn from_proto(t: protobuf::JoinConstraint) -> Self {match t {protobuf::JoinConstraint::On => JoinConstraint::On,protobuf::JoinConstraint::Using => JoinConstraint::Using,@@ -222,8 +230,8 @@ impl From<protobuf::JoinConstraint> for JoinConstraint {}}-impl From<protobuf::NullEquality> for NullEquality {- fn from(t: protobuf::NullEquality) -> Self {+impl FromProto<protobuf::NullEquality> for NullEquality {+ fn from_proto(t: protobuf::NullEquality) -> Self {match t {protobuf::NullEquality::NullEqualsNothing => NullEquality::NullEqualsNothing,protobuf::NullEquality::NullEqualsNull => NullEquality::NullEqualsNull,@@ -231,8 +239,8 @@ impl From<protobuf::NullEquality> for NullEquality {}}-impl From<protobuf::dml_node::Type> for WriteOp {- fn from(t: protobuf::dml_node::Type) -> Self {+impl FromProto<protobuf::dml_node::Type> for WriteOp {+ fn from_proto(t: protobuf::dml_node::Type) -> Self {match t {protobuf::dml_node::Type::Update => WriteOp::Update,protobuf::dml_node::Type::Delete => WriteOp::Delete,@@ -247,8 +255,8 @@ impl From<protobuf::dml_node::Type> for WriteOp {}}-impl From<protobuf::NullTreatment> for NullTreatment {- fn from(t: protobuf::NullTreatment) -> Self {+impl FromProto<protobuf::NullTreatment> for NullTreatment {+ fn from_proto(t: protobuf::NullTreatment) -> Self {match t {protobuf::NullTreatment::RespectNulls => NullTreatment::RespectNulls,protobuf::NullTreatment::IgnoreNulls => NullTreatment::IgnoreNulls,@@ -304,7 +312,7 @@ pub fn parse_expr(.window_frame.as_ref().map::<Result<WindowFrame, _>, _>(|window_frame| {- let window_frame: WindowFrame = window_frame.clone().try_into()?;+ let window_frame = WindowFrame::try_from_proto(window_frame.clone())?;window_frame.regularize_order_bys(&mut order_by).map(|_| window_frame)@@ -322,7 +330,7 @@ pub fn parse_expr("Received a WindowExprNode message with unknown NullTreatment {null_treatment}",))})?;- Some(NullTreatment::from(null_treatment))+ Some(NullTreatment::from_proto(null_treatment))}None => None,};@@ -371,7 +379,7 @@ pub fn parse_expr(alias.relation.first()- .map(|r| TableReference::try_from(r.clone()))+ .map(|r| TableReference::try_from_proto(r.clone())).transpose()?,alias.alias.clone(),))),@@ -571,7 +579,10 @@ pub fn parse_expr(in_list.negated,))),ExprType::Wildcard(protobuf::Wildcard { qualifier }) => {- let qualifier = qualifier.to_owned().map(|x| x.try_into()).transpose()?;+ let qualifier = qualifier+ .to_owned()+ .map(TableReference::try_from_proto)+ .transpose()?;#[expect(deprecated)]Ok(Expr::Wildcard {qualifier,@@ -609,7 +620,7 @@ pub fn parse_expr("Received an AggregateUdfExprNode message with unknown NullTreatment {null_treatment}",))})?;- Some(NullTreatment::from(null_treatment))+ Some(NullTreatment::from_proto(null_treatment))}None => None,};@@ -742,42 +753,11 @@ fn parse_escape_char(s: &str) -> Result<Option<char>> {}pub fn from_proto_binary_op(op: &str) -> Result<Operator, Error> {- match op {- "And" => Ok(Operator::And),- "Or" => Ok(Operator::Or),- "Eq" => Ok(Operator::Eq),- "NotEq" => Ok(Operator::NotEq),- "LtEq" => Ok(Operator::LtEq),- "Lt" => Ok(Operator::Lt),- "Gt" => Ok(Operator::Gt),- "GtEq" => Ok(Operator::GtEq),- "Plus" => Ok(Operator::Plus),- "Minus" => Ok(Operator::Minus),- "Multiply" => Ok(Operator::Multiply),- "Divide" => Ok(Operator::Divide),- "Modulo" => Ok(Operator::Modulo),- "IsDistinctFrom" => Ok(Operator::IsDistinctFrom),- "IsNotDistinctFrom" => Ok(Operator::IsNotDistinctFrom),- "BitwiseAnd" => Ok(Operator::BitwiseAnd),- "BitwiseOr" => Ok(Operator::BitwiseOr),- "BitwiseXor" => Ok(Operator::BitwiseXor),- "BitwiseShiftLeft" => Ok(Operator::BitwiseShiftLeft),- "BitwiseShiftRight" => Ok(Operator::BitwiseShiftRight),- "RegexIMatch" => Ok(Operator::RegexIMatch),- "RegexMatch" => Ok(Operator::RegexMatch),- "RegexNotIMatch" => Ok(Operator::RegexNotIMatch),- "RegexNotMatch" => Ok(Operator::RegexNotMatch),- "LikeMatch" => Ok(Operator::LikeMatch),- "ILikeMatch" => Ok(Operator::ILikeMatch),- "NotLikeMatch" => Ok(Operator::NotLikeMatch),- "NotILikeMatch" => Ok(Operator::NotILikeMatch),- "StringConcat" => Ok(Operator::StringConcat),- "AtArrow" => Ok(Operator::AtArrow),- "ArrowAt" => Ok(Operator::ArrowAt),- other => Err(proto_error(format!(- "Unsupported binary operator '{other:?}'"- ))),- }+ // The proto-string <-> `Operator` mapping is canonically owned by+ // `datafusion-expr-common` so `datafusion-proto` (logical plans) and+ // `PhysicalExpr` decoders (e.g. `BinaryExpr`) share one source of truth.+ Operator::from_proto_name(op)+ .ok_or_else(|| proto_error(format!("Unsupported binary operator '{op:?}'")))}fn parse_optional_expr(diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex 8cdd5c5dea..542cae890d 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -19,13 +19,14 @@ use std::collections::HashMap;use std::fmt::Debug;use std::sync::Arc;+use crate::convert::{FromProto, TryFromProto};use crate::protobuf::logical_plan_node::LogicalPlanType::CustomScan;use crate::protobuf::{ColumnUnnestListItem, ColumnUnnestListRecursion, CteWorkTableScanNode,CustomTableScanNode, DmlNode, SortExprNodeCollection, dml_node,};use crate::{- convert_required, into_required,+ convert_required,protobuf::{self, LogicalExtensionNode, LogicalPlanNode,listing_table_scan_node::FileFormatType, logical_plan_node::LogicalPlanType,@@ -39,7 +40,7 @@ use datafusion_catalog::empty::EmptyTable;use datafusion_common::file_options::file_type::FileType;use datafusion_common::format::ExplainFormat;use datafusion_common::{- Result, TableReference, ToDFSchema, assert_or_internal_err, context,+ NullEquality, Result, TableReference, ToDFSchema, assert_or_internal_err, context,internal_datafusion_err, internal_err, not_impl_err, plan_err,};use datafusion_datasource::file_format::FileFormat;@@ -57,11 +58,11 @@ use datafusion_datasource_json::file_format::{use datafusion_datasource_parquet::file_format::{ParquetFormat, ParquetFormatFactory};use datafusion_expr::{AggregateUDF, DmlStatement, FetchType, HigherOrderUDF, RecursiveQuery, SkipType,- TableSource, Unnest,+ TableSource, Unnest, WriteOp,};use datafusion_expr::{- DistinctOn, DropView, Expr, LogicalPlan, LogicalPlanBuilder, ScalarUDF, SortExpr,- Statement, WindowUDF, dml,+ DistinctOn, DropView, Expr, JoinConstraint, LogicalPlan, LogicalPlanBuilder,+ ScalarUDF, SortExpr, Statement, WindowUDF, dml,logical_plan::{Aggregate, CreateCatalog, CreateCatalogSchema, CreateExternalTable, CreateView,DdlStatement, Distinct, EmptyRelation, Extension, Join, Prepare, Projection,@@ -341,7 +342,7 @@ fn from_table_reference()})?;- Ok(table_ref.clone().try_into()?)+ Ok(TableReference::try_from_proto(table_ref.clone())?)}/// Converts [LogicalPlan::TableScan] to [TableSource]@@ -900,9 +901,9 @@ impl AsLogicalPlan for LogicalPlanNode {Arc::new(right),on,filter,- join_type.into(),- join_constraint.into(),- null_equality.into(),+ datafusion_expr::JoinType::from_proto(join_type),+ JoinConstraint::from_proto(join_constraint),+ NullEquality::from_proto(null_equality),join.null_aware,)?))}@@ -1065,7 +1066,13 @@ impl AsLogicalPlan for LogicalPlanNode {LogicalPlanBuilder::from(input).unnest_columns_with_options(unnest.exec_columns.iter().map(|c| c.into()).collect(),- into_required!(unnest.options)?,+ unnest+ .options+ .as_ref()+ .map(datafusion_common::UnnestOptions::from_proto)+ .ok_or_else(|| {+ proto_error("Missing required field in protobuf")+ })?,)?.build()}@@ -1137,7 +1144,7 @@ impl AsLogicalPlan for LogicalPlanNode {Ok(LogicalPlan::Dml(datafusion_expr::DmlStatement::new(from_table_reference(dml_node.table_name.as_ref(), "DML ")?,to_table_source(&dml_node.target, ctx, extension_codec)?,- dml_node.dml_type().into(),+ WriteOp::from_proto(dml_node.dml_type()),Arc::new(into_logical_plan!(dml_node.input, ctx, extension_codec)?),)))}@@ -1292,7 +1299,9 @@ impl AsLogicalPlan for LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::ListingScan(protobuf::ListingTableScanNode {file_format_type: Some(file_format_type),- table_name: Some(table_name.clone().into()),+ table_name: Some(protobuf::TableReference::from_proto(+ table_name.clone(),+ )),collect_stat: options.collect_stat,file_extension: options.file_extension.clone(),table_partition_cols: partition_columns,@@ -1314,7 +1323,9 @@ impl AsLogicalPlan for LogicalPlanNode {Ok(LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::ViewScan(Box::new(protobuf::ViewTableScanNode {- table_name: Some(table_name.clone().into()),+ table_name: Some(protobuf::TableReference::from_proto(+ table_name.clone(),+ )),input: Some(Box::new(LogicalPlanNode::try_from_logical_plan(view_table.logical_plan(),@@ -1351,7 +1362,9 @@ impl AsLogicalPlan for LogicalPlanNode {Ok(LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::EmptyTableScan(protobuf::EmptyTableScanNode {- table_name: Some(table_name.clone().into()),+ table_name: Some(protobuf::TableReference::from_proto(+ table_name.clone(),+ )),schema: Some(schema),projection,filters,@@ -1365,7 +1378,9 @@ impl AsLogicalPlan for LogicalPlanNode {.try_encode_table_provider(table_name, provider, &mut bytes).map_err(|e| context!("Error serializing custom table", e))?;let scan = CustomScan(CustomTableScanNode {- table_name: Some(table_name.clone().into()),+ table_name: Some(protobuf::TableReference::from_proto(+ table_name.clone(),+ )),projection,schema: Some(schema),filters,@@ -1516,11 +1531,11 @@ impl AsLogicalPlan for LogicalPlanNode {.collect::<Result<Vec<_>, ToProtoError>>()?.into_iter().unzip();- let join_type: protobuf::JoinType = join_type.to_owned().into();- let join_constraint: protobuf::JoinConstraint =- join_constraint.to_owned().into();- let null_equality: protobuf::NullEquality =- null_equality.to_owned().into();+ let join_type = protobuf::JoinType::from_proto(join_type.to_owned());+ let join_constraint =+ protobuf::JoinConstraint::from_proto(join_constraint.to_owned());+ let null_equality =+ protobuf::NullEquality::from_proto(null_equality.to_owned());let filter = filter.as_ref().map(|e| serialize_expr(e, extension_codec).map(Box::new))@@ -1559,7 +1574,9 @@ impl AsLogicalPlan for LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::SubqueryAlias(Box::new(protobuf::SubqueryAliasNode {input: Some(Box::new(input)),- alias: Some((*alias).clone().into()),+ alias: Some(protobuf::TableReference::from_proto(+ (*alias).clone(),+ )),},))),})@@ -1690,7 +1707,9 @@ impl AsLogicalPlan for LogicalPlanNode {Ok(LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::CreateExternalTable(protobuf::CreateExternalTableNode {- name: Some(name.clone().into()),+ name: Some(protobuf::TableReference::from_proto(+ name.clone(),+ )),location: location.clone(),file_type: file_type.clone(),schema: Some(df_schema.try_into()?),@@ -1717,7 +1736,7 @@ impl AsLogicalPlan for LogicalPlanNode {})) => Ok(LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::CreateView(Box::new(protobuf::CreateViewNode {- name: Some(name.clone().into()),+ name: Some(protobuf::TableReference::from_proto(name.clone())),input: Some(Box::new(LogicalPlanNode::try_from_logical_plan(input,extension_codec,@@ -1889,7 +1908,7 @@ impl AsLogicalPlan for LogicalPlanNode {.map(|c| *c as u64).collect(),schema: Some(schema.try_into()?),- options: Some(options.into()),+ options: Some(protobuf::UnnestOptions::from_proto(options)),},))),})@@ -1910,7 +1929,7 @@ impl AsLogicalPlan for LogicalPlanNode {})) => Ok(LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::DropView(protobuf::DropViewNode {- name: Some(name.clone().into()),+ name: Some(protobuf::TableReference::from_proto(name.clone())),if_exists: *if_exists,schema: Some(schema.try_into()?),},@@ -1937,7 +1956,7 @@ impl AsLogicalPlan for LogicalPlanNode {}) => {let input =LogicalPlanNode::try_from_logical_plan(input, extension_codec)?;- let dml_type: dml_node::Type = op.into();+ let dml_type = dml_node::Type::from_proto(op);Ok(LogicalPlanNode {logical_plan_type: Some(LogicalPlanType::Dml(Box::new(DmlNode {input: Some(Box::new(input)),@@ -1946,7 +1965,9 @@ impl AsLogicalPlan for LogicalPlanNode {Arc::clone(target),extension_codec,)?)),- table_name: Some(table_name.clone().into()),+ table_name: Some(protobuf::TableReference::from_proto(+ table_name.clone(),+ )),dml_type: dml_type.into(),}))),})diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rsindex d79107d1d0..71a6bd824a 100644--- a/datafusion/proto/src/logical_plan/to_proto.rs+++ b/datafusion/proto/src/logical_plan/to_proto.rs@@ -50,10 +50,11 @@ use crate::protobuf::{};use super::{AsLogicalPlan, LogicalExtensionCodec};+use crate::convert::{FromProto, TryFromProto};use crate::protobuf::LogicalPlanNode;-impl From<&UnnestOptions> for protobuf::UnnestOptions {- fn from(opts: &UnnestOptions) -> Self {+impl FromProto<&UnnestOptions> for protobuf::UnnestOptions {+ fn from_proto(opts: &UnnestOptions) -> Self {Self {preserve_nulls: opts.preserve_nulls,recursions: opts@@ -69,8 +70,8 @@ impl From<&UnnestOptions> for protobuf::UnnestOptions {}}-impl From<&StringifiedPlan> for protobuf::StringifiedPlan {- fn from(stringified_plan: &StringifiedPlan) -> Self {+impl FromProto<&StringifiedPlan> for protobuf::StringifiedPlan {+ fn from_proto(stringified_plan: &StringifiedPlan) -> Self {Self {plan_type: match stringified_plan.clone().plan_type {PlanType::InitialLogicalPlan => Some(protobuf::PlanType {@@ -130,8 +131,8 @@ impl From<&StringifiedPlan> for protobuf::StringifiedPlan {}}-impl From<WindowFrameUnits> for protobuf::WindowFrameUnits {- fn from(units: WindowFrameUnits) -> Self {+impl FromProto<WindowFrameUnits> for protobuf::WindowFrameUnits {+ fn from_proto(units: WindowFrameUnits) -> Self {match units {WindowFrameUnits::Rows => Self::Rows,WindowFrameUnits::Range => Self::Range,@@ -140,10 +141,10 @@ impl From<WindowFrameUnits> for protobuf::WindowFrameUnits {}}-impl TryFrom<&WindowFrameBound> for protobuf::WindowFrameBound {+impl TryFromProto<&WindowFrameBound> for protobuf::WindowFrameBound {type Error = Error;- fn try_from(bound: &WindowFrameBound) -> Result<Self, Self::Error> {+ fn try_from_proto(bound: &WindowFrameBound) -> Result<Self, Self::Error> {Ok(match bound {WindowFrameBound::CurrentRow => Self {window_frame_bound_type: protobuf::WindowFrameBoundType::CurrentRow@@ -162,15 +163,18 @@ impl TryFrom<&WindowFrameBound> for protobuf::WindowFrameBound {}}-impl TryFrom<&WindowFrame> for protobuf::WindowFrame {+impl TryFromProto<&WindowFrame> for protobuf::WindowFrame {type Error = Error;- fn try_from(window: &WindowFrame) -> Result<Self, Self::Error> {+ fn try_from_proto(window: &WindowFrame) -> Result<Self, Self::Error> {Ok(Self {- window_frame_units: protobuf::WindowFrameUnits::from(window.units).into(),- start_bound: Some((&window.start_bound).try_into()?),+ window_frame_units: protobuf::WindowFrameUnits::from_proto(window.units)+ .into(),+ start_bound: Some(protobuf::WindowFrameBound::try_from_proto(+ &window.start_bound,+ )?),end_bound: Some(protobuf::window_frame::EndBound::Bound(- (&window.end_bound).try_into()?,+ protobuf::WindowFrameBound::try_from_proto(&window.end_bound)?,)),})}@@ -209,7 +213,7 @@ pub fn serialize_expr(expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)),relation: relation.to_owned()- .map(|r| vec![r.into()])+ .map(|r| vec![protobuf::TableReference::from_proto(r)]).unwrap_or(vec![]),alias: name.to_owned(),metadata: metadata@@ -339,8 +343,7 @@ pub fn serialize_expr(let partition_by = serialize_exprs(partition_by, codec)?;let order_by = serialize_sorts(order_by, codec)?;- let window_frame: Option<protobuf::WindowFrame> =- Some(window_frame.try_into()?);+ let window_frame = Some(protobuf::WindowFrame::try_from_proto(window_frame)?);let window_expr = protobuf::WindowExprNode {exprs: serialize_exprs(args, codec)?,@@ -354,7 +357,7 @@ pub fn serialize_expr(None => None,},null_treatment: null_treatment- .map(|nt| protobuf::NullTreatment::from(nt).into()),+ .map(|nt| protobuf::NullTreatment::from_proto(nt).into()),fun_definition,};protobuf::LogicalExprNode {@@ -387,7 +390,7 @@ pub fn serialize_expr(order_by: serialize_sorts(order_by, codec)?,fun_definition: (!buf.is_empty()).then_some(buf),null_treatment: null_treatment- .map(|nt| protobuf::NullTreatment::from(nt).into()),+ .map(|nt| protobuf::NullTreatment::from_proto(nt).into()),},))),}@@ -578,7 +581,9 @@ pub fn serialize_expr(#[expect(deprecated)]Expr::Wildcard { qualifier, .. } => protobuf::LogicalExprNode {expr_type: Some(ExprType::Wildcard(protobuf::Wildcard {- qualifier: qualifier.to_owned().map(|x| x.into()),+ qualifier: qualifier+ .to_owned()+ .map(protobuf::TableReference::from_proto),})),},Expr::ScalarSubquery(subquery) => protobuf::LogicalExprNode {@@ -681,8 +686,8 @@ where.collect::<Result<Vec<_>, Error>>()}-impl From<TableReference> for protobuf::TableReference {- fn from(t: TableReference) -> Self {+impl FromProto<TableReference> for protobuf::TableReference {+ fn from_proto(t: TableReference) -> Self {use protobuf::table_reference::TableReferenceEnum;let table_reference_enum = match t {TableReference::Bare { table } => {@@ -713,8 +718,8 @@ impl From<TableReference> for protobuf::TableReference {}}-impl From<JoinType> for protobuf::JoinType {- fn from(t: JoinType) -> Self {+impl FromProto<JoinType> for protobuf::JoinType {+ fn from_proto(t: JoinType) -> Self {match t {JoinType::Inner => protobuf::JoinType::Inner,JoinType::Left => protobuf::JoinType::Left,@@ -730,8 +735,8 @@ impl From<JoinType> for protobuf::JoinType {}}-impl From<JoinConstraint> for protobuf::JoinConstraint {- fn from(t: JoinConstraint) -> Self {+impl FromProto<JoinConstraint> for protobuf::JoinConstraint {+ fn from_proto(t: JoinConstraint) -> Self {match t {JoinConstraint::On => protobuf::JoinConstraint::On,JoinConstraint::Using => protobuf::JoinConstraint::Using,@@ -739,8 +744,8 @@ impl From<JoinConstraint> for protobuf::JoinConstraint {}}-impl From<NullEquality> for protobuf::NullEquality {- fn from(t: NullEquality) -> Self {+impl FromProto<NullEquality> for protobuf::NullEquality {+ fn from_proto(t: NullEquality) -> Self {match t {NullEquality::NullEqualsNothing => protobuf::NullEquality::NullEqualsNothing,NullEquality::NullEqualsNull => protobuf::NullEquality::NullEqualsNull,@@ -748,8 +753,8 @@ impl From<NullEquality> for protobuf::NullEquality {}}-impl From<&WriteOp> for protobuf::dml_node::Type {- fn from(t: &WriteOp) -> Self {+impl FromProto<&WriteOp> for protobuf::dml_node::Type {+ fn from_proto(t: &WriteOp) -> Self {match t {WriteOp::Insert(InsertOp::Append) => protobuf::dml_node::Type::InsertAppend,WriteOp::Insert(InsertOp::Overwrite) => {@@ -764,8 +769,8 @@ impl From<&WriteOp> for protobuf::dml_node::Type {}}-impl From<NullTreatment> for protobuf::NullTreatment {- fn from(t: NullTreatment) -> Self {+impl FromProto<NullTreatment> for protobuf::NullTreatment {+ fn from_proto(t: NullTreatment) -> Self {match t {NullTreatment::RespectNulls => protobuf::NullTreatment::RespectNulls,NullTreatment::IgnoreNulls => protobuf::NullTreatment::IgnoreNulls,diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 43ebf04743..f5fd214ef6 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -57,19 +57,13 @@ use super::{DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,PhysicalProtoConverterExtension,};-use crate::logical_plan::{self};+use crate::convert::TryFromProto;use crate::protobuf::physical_expr_node::ExprType;-use crate::{convert_required, protobuf};+use crate::{convert_required, convert_required_proto, protobuf};use datafusion_physical_expr::expressions::{DynamicFilterInner, DynamicFilterPhysicalExpr,};-impl From<&protobuf::PhysicalColumn> for Column {- fn from(c: &protobuf::PhysicalColumn) -> Column {- Column::new(&c.name, c.index as usize)- }-}-/// Parses a physical sort expression from a protobuf.////// # Arguments@@ -154,7 +148,7 @@ pub fn parse_physical_window_expr(let window_frame = proto.window_frame.as_ref()- .map(|wf| wf.clone().try_into())+ .map(|wf| datafusion_expr::WindowFrame::try_from_proto(wf.clone())).transpose().map_err(|e| internal_datafusion_err!("{e}"))?.ok_or_else(|| {@@ -266,57 +260,27 @@ pub fn parse_physical_expr_with_converter(.as_ref().ok_or_else(|| proto_error("Unexpected empty physical expression"))?;+ // Decoder context handed to per-expression `try_from_proto` constructors.+ // This is the new shape the codebase is migrating toward (see #21835);+ // the remaining `ExprType` variants stay matched inline until they migrate.+ let decoder = ConverterDecoder {+ ctx,+ proto_converter,+ };+ let decode_ctx =+ datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx::new(+ input_schema,+ &decoder,+ );+let pexpr: Arc<dyn PhysicalExpr> = match expr_type {- ExprType::Column(c) => {- let pcol: Column = c.into();- Arc::new(pcol)- }+ // Migrated expressions take the whole `PhysicalExprNode` and unwrap+ // their own `ExprType` variant — see #21835. This match only routes+ // to the right constructor.+ ExprType::Column(_) => Column::try_from_proto(proto, &decode_ctx)?,ExprType::UnknownColumn(c) => Arc::new(UnKnownColumn::new(&c.name)),ExprType::Literal(scalar) => Arc::new(Literal::new(scalar.try_into()?)),- ExprType::BinaryExpr(binary_expr) => {- let op = logical_plan::from_proto::from_proto_binary_op(&binary_expr.op)?;- if !binary_expr.operands.is_empty() {- // New linearized format: reduce the flat operands list back into- // a nested binary expression tree.- let operands: Vec<Arc<dyn PhysicalExpr>> = binary_expr- .operands- .iter()- .map(|e| proto_converter.proto_to_physical_expr(e, input_schema, ctx))- .collect::<Result<Vec<_>>>()?;-- if operands.len() < 2 {- return Err(proto_error(- "A binary expression must always have at least 2 operands",- ));- }-- operands- .into_iter()- .reduce(|left, right| Arc::new(BinaryExpr::new(left, op, right)))- .expect(- "Binary expression could not be reduced to a single expression.",- )- } else {- // Legacy format with l/r fields- Arc::new(BinaryExpr::new(- parse_required_physical_expr(- binary_expr.l.as_deref(),- ctx,- "left",- input_schema,- proto_converter,- )?,- op,- parse_required_physical_expr(- binary_expr.r.as_deref(),- ctx,- "right",- input_schema,- proto_converter,- )?,- ))- }- }+ ExprType::BinaryExpr(_) => BinaryExpr::try_from_proto(proto, &decode_ctx)?,ExprType::AggregateExpr(_) => {return not_impl_err!("Cannot convert aggregate expr node to physical expression"@@ -694,7 +658,7 @@ pub fn parse_protobuf_file_scan_config(let file_groups = proto.file_groups.iter()- .map(|f| f.try_into())+ .map(FileGroup::try_from_proto).collect::<Result<Vec<_>, _>>()?;let object_store_url = match proto.object_store_url.is_empty() {@@ -763,10 +727,10 @@ pub fn parse_record_batches(buf: &[u8]) -> Result<Vec<RecordBatch>> {Ok(batches)}-impl TryFrom<&protobuf::PartitionedFile> for PartitionedFile {+impl TryFromProto<&protobuf::PartitionedFile> for PartitionedFile {type Error = DataFusionError;- fn try_from(val: &protobuf::PartitionedFile) -> Result<Self, Self::Error> {+ fn try_from_proto(val: &protobuf::PartitionedFile) -> Result<Self, Self::Error> {let mut pf = PartitionedFile::new_from_meta(ObjectMeta {location: Path::parse(val.path.as_str()).map_err(|e| proto_error(format!("Invalid object_store path: {e}")))?,@@ -782,7 +746,7 @@ impl TryFrom<&protobuf::PartitionedFile> for PartitionedFile {.collect::<Result<Vec<_>, _>>()?,);if let Some(range) = val.range.as_ref() {- let file_range: FileRange = range.try_into()?;+ let file_range = FileRange::try_from_proto(range)?;pf = pf.with_range(file_range.start, file_range.end);}if let Some(proto_stats) = val.statistics.as_ref() {@@ -792,10 +756,10 @@ impl TryFrom<&protobuf::PartitionedFile> for PartitionedFile {}}-impl TryFrom<&protobuf::FileRange> for FileRange {+impl TryFromProto<&protobuf::FileRange> for FileRange {type Error = DataFusionError;- fn try_from(value: &protobuf::FileRange) -> Result<Self, Self::Error> {+ fn try_from_proto(value: &protobuf::FileRange) -> Result<Self, Self::Error> {Ok(FileRange {start: value.start,end: value.end,@@ -803,61 +767,61 @@ impl TryFrom<&protobuf::FileRange> for FileRange {}}-impl TryFrom<&protobuf::FileGroup> for FileGroup {+impl TryFromProto<&protobuf::FileGroup> for FileGroup {type Error = DataFusionError;- fn try_from(val: &protobuf::FileGroup) -> Result<Self, Self::Error> {+ fn try_from_proto(val: &protobuf::FileGroup) -> Result<Self, Self::Error> {let files = val.files.iter()- .map(|f| f.try_into())+ .map(PartitionedFile::try_from_proto).collect::<Result<Vec<_>, _>>()?;Ok(FileGroup::new(files))}}-impl TryFrom<&protobuf::JsonSink> for JsonSink {+impl TryFromProto<&protobuf::JsonSink> for JsonSink {type Error = DataFusionError;- fn try_from(value: &protobuf::JsonSink) -> Result<Self, Self::Error> {+ fn try_from_proto(value: &protobuf::JsonSink) -> Result<Self, Self::Error> {Ok(Self::new(- convert_required!(value.config)?,+ convert_required_proto!(FileSinkConfig, value.config)?,convert_required!(value.writer_options)?,))}}#[cfg(feature = "parquet")]-impl TryFrom<&protobuf::ParquetSink> for ParquetSink {+impl TryFromProto<&protobuf::ParquetSink> for ParquetSink {type Error = DataFusionError;- fn try_from(value: &protobuf::ParquetSink) -> Result<Self, Self::Error> {+ fn try_from_proto(value: &protobuf::ParquetSink) -> Result<Self, Self::Error> {Ok(Self::new(- convert_required!(value.config)?,+ convert_required_proto!(FileSinkConfig, value.config)?,convert_required!(value.parquet_options)?,))}}-impl TryFrom<&protobuf::CsvSink> for CsvSink {+impl TryFromProto<&protobuf::CsvSink> for CsvSink {type Error = DataFusionError;- fn try_from(value: &protobuf::CsvSink) -> Result<Self, Self::Error> {+ fn try_from_proto(value: &protobuf::CsvSink) -> Result<Self, Self::Error> {Ok(Self::new(- convert_required!(value.config)?,+ convert_required_proto!(FileSinkConfig, value.config)?,convert_required!(value.writer_options)?,))}}-impl TryFrom<&protobuf::FileSinkConfig> for FileSinkConfig {+impl TryFromProto<&protobuf::FileSinkConfig> for FileSinkConfig {type Error = DataFusionError;- fn try_from(conf: &protobuf::FileSinkConfig) -> Result<Self, Self::Error> {+ fn try_from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self, Self::Error> {let file_group = FileGroup::new(conf.file_groups.iter()- .map(|f| f.try_into())+ .map(PartitionedFile::try_from_proto).collect::<Result<Vec<_>>>()?,);let table_paths = conf@@ -904,6 +868,33 @@ impl TryFrom<&protobuf::FileSinkConfig> for FileSinkConfig {}}+/// Concrete [`PhysicalExprDecode`] driver that backs+/// [`PhysicalExprDecodeCtx`] inside `parse_physical_expr_with_converter`.+///+/// Today this is a thin wrapper that re-enters the central match through+/// `proto_to_physical_expr`; once more expressions migrate, the central match+/// shrinks and a future builder-style decoder can take over.+///+/// [`PhysicalExprDecode`]: datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecode+/// [`PhysicalExprDecodeCtx`]: datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx+struct ConverterDecoder<'a, 'b> {+ ctx: &'a PhysicalPlanDecodeContext<'b>,+ proto_converter: &'a dyn PhysicalProtoConverterExtension,+}++impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecode+ for ConverterDecoder<'_, '_>+{+ fn decode(+ &self,+ node: &protobuf::PhysicalExprNode,+ schema: &Schema,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ self.proto_converter+ .proto_to_physical_expr(node, schema, self.ctx)+ }+}+#[cfg(test)]mod tests {@@ -920,10 +911,10 @@ mod tests {version: None,});- let proto = protobuf::PartitionedFile::try_from(&pf).unwrap();+ let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap();assert_eq!(proto.path, path_str);- let pf2 = PartitionedFile::try_from(&proto).unwrap();+ let pf2 = PartitionedFile::try_from_proto(&proto).unwrap();assert_eq!(pf2.object_meta.location.as_ref(), path_str);assert_eq!(pf2.object_meta.location, pf.object_meta.location);assert_eq!(pf2.object_meta.size, pf.object_meta.size);@@ -941,7 +932,7 @@ mod tests {statistics: None,};- let err = PartitionedFile::try_from(&proto).unwrap_err();+ let err = PartitionedFile::try_from_proto(&proto).unwrap_err();assert!(err.to_string().contains("Invalid object_store path"));}}diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 27284664b0..89871318ee 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -25,8 +25,10 @@ use arrow::compute::SortOptions;use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};use datafusion_catalog::memory::MemorySourceConfig;use datafusion_common::config::CsvOptions;+use datafusion_common::display::StringifiedPlan;use datafusion_common::{- DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err,+ DataFusionError, JoinType, NullEquality, Result, internal_datafusion_err,+ internal_err, not_impl_err,};#[cfg(feature = "parquet")]use datafusion_datasource::file::FileSource;@@ -97,6 +99,8 @@ use prost::bytes::BufMut;use self::from_proto::parse_protobuf_partitioning;use self::to_proto::serialize_partitioning;use crate::common::{byte_to_string, str_to_byte};+use crate::convert::{FromProto, TryFromProto};+use crate::convert_required;use crate::physical_plan::from_proto::{parse_physical_expr_with_converter, parse_physical_sort_expr,parse_physical_sort_exprs, parse_physical_window_expr,@@ -114,7 +118,6 @@ use crate::protobuf::{self, ListUnnest as ProtoListUnnest, SortExprNode, SortMergeJoinExecNode,proto_error, window_agg_exec_node,};-use crate::{convert_required, into_required};pub mod from_proto;pub mod to_proto;@@ -270,8 +273,24 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {}}-impl protobuf::PhysicalPlanNode {- pub fn try_into_physical_plan_with_converter(+/// Extension methods on [`protobuf::PhysicalPlanNode`].+///+/// The prost-generated `PhysicalPlanNode` struct lives in+/// `datafusion-proto-models`, which is foreign to this crate, so the orphan+/// rule forbids inherent `impl` blocks here. Instead, all (de)serialization+/// helpers are exposed through this trait. Callers can bring it in scope with+/// `use datafusion_proto::physical_plan::PhysicalPlanNodeExt;`.+///+/// Method bodies live in the default trait implementation. To make the trait+/// usable as if it were inherent (i.e. let bodies access fields on `self`),+/// implementors provide [`PhysicalPlanNodeExt::node`] returning a reference+/// back to the concrete `protobuf::PhysicalPlanNode`. Default method bodies+/// then go through `self.node()` to read fields.+pub trait PhysicalPlanNodeExt: Sized {+ /// Returns a reference to the underlying [`protobuf::PhysicalPlanNode`].+ fn node(&self) -> &protobuf::PhysicalPlanNode;++ fn try_into_physical_plan_with_converter(&self,ctx: &TaskContext,codec: &dyn PhysicalExtensionCodec,@@ -281,14 +300,15 @@ impl protobuf::PhysicalPlanNode {self.try_into_physical_plan_with_context(&decode_ctx, proto_converter)}- pub(crate) fn try_into_physical_plan_with_context(+ fn try_into_physical_plan_with_context(&self,ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {- let plan = self.physical_plan_type.as_ref().ok_or_else(|| {+ let plan = self.node().physical_plan_type.as_ref().ok_or_else(|| {proto_error(format!(- "physical_plan::from_proto() Unsupported physical plan '{self:?}'"+ "physical_plan::from_proto() Unsupported physical plan '{:?}'",+ self.node(),))})?;match plan {@@ -415,14 +435,11 @@ impl protobuf::PhysicalPlanNode {}}- pub fn try_from_physical_plan_with_converter(+ fn try_from_physical_plan_with_converter(plan: Arc<dyn ExecutionPlan>,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self>- where- Self: Sized,- {+ ) -> Result<protobuf::PhysicalPlanNode> {let plan_clone = Arc::clone(&plan);let plan = plan.as_ref() as &dyn Any;@@ -695,9 +712,7 @@ impl protobuf::PhysicalPlanNode {),}}-}-impl protobuf::PhysicalPlanNode {fn try_into_explain_physical_plan(&self,explain: &protobuf::ExplainExecNode,@@ -709,7 +724,7 @@ impl protobuf::PhysicalPlanNode {explain.stringified_plans.iter()- .map(|plan| plan.into())+ .map(StringifiedPlan::from_proto).collect(),explain.verbose,)))@@ -1498,10 +1513,10 @@ impl protobuf::PhysicalPlanNode {right,on,filter,- &join_type.into(),+ &JoinType::from_proto(join_type),projection,partition_mode,- null_equality.into(),+ NullEquality::from_proto(null_equality),hashjoin.null_aware,)?;@@ -1640,8 +1655,8 @@ impl protobuf::PhysicalPlanNode {right,on,filter,- &join_type.into(),- null_equality.into(),+ &JoinType::from_proto(join_type),+ NullEquality::from_proto(null_equality),left_sort_exprs,right_sort_exprs,partition_mode,@@ -1713,6 +1728,7 @@ impl protobuf::PhysicalPlanNode {ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {+ let node = self.node();let input = into_physical_plan(&sort.input, ctx, proto_converter)?;let exprs = sort.expr@@ -1720,7 +1736,7 @@ impl protobuf::PhysicalPlanNode {.map(|expr| {let expr = expr.expr_type.as_ref().ok_or_else(|| {proto_error(format!(- "physical_plan::from_proto() Unexpected expr {self:?}"+ "physical_plan::from_proto() Unexpected expr {node:?}"))})?;if let ExprType::Sort(sort_expr) = expr {@@ -1729,7 +1745,7 @@ impl protobuf::PhysicalPlanNode {.as_ref().ok_or_else(|| {proto_error(format!(- "physical_plan::from_proto() Unexpected sort expr {self:?}"+ "physical_plan::from_proto() Unexpected sort expr {node:?}"))})?.as_ref();@@ -1746,7 +1762,7 @@ impl protobuf::PhysicalPlanNode {})} else {internal_err!(- "physical_plan::from_proto() {self:?}"+ "physical_plan::from_proto() {node:?}")}})@@ -1786,6 +1802,7 @@ impl protobuf::PhysicalPlanNode {ctx: &PhysicalPlanDecodeContext<'_>,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Arc<dyn ExecutionPlan>> {+ let node = self.node();let input = into_physical_plan(&sort.input, ctx, proto_converter)?;let exprs = sort.expr@@ -1793,7 +1810,7 @@ impl protobuf::PhysicalPlanNode {.map(|expr| {let expr = expr.expr_type.as_ref().ok_or_else(|| {proto_error(format!(- "physical_plan::from_proto() Unexpected expr {self:?}"+ "physical_plan::from_proto() Unexpected expr {node:?}"))})?;if let ExprType::Sort(sort_expr) = expr {@@ -1802,7 +1819,7 @@ impl protobuf::PhysicalPlanNode {.as_ref().ok_or_else(|| {proto_error(format!(- "physical_plan::from_proto() Unexpected sort expr {self:?}"+ "physical_plan::from_proto() Unexpected sort expr {node:?}"))})?.as_ref();@@ -1818,7 +1835,7 @@ impl protobuf::PhysicalPlanNode {},})} else {- internal_err!("physical_plan::from_proto() {self:?}")+ internal_err!("physical_plan::from_proto() {node:?}")}}).collect::<Result<Vec<_>>>()?;@@ -1922,7 +1939,7 @@ impl protobuf::PhysicalPlanNode {left,right,filter,- &join_type.into(),+ &JoinType::from_proto(join_type),projection,)?))}@@ -1963,11 +1980,11 @@ impl protobuf::PhysicalPlanNode {) -> Result<Arc<dyn ExecutionPlan>> {let input = into_physical_plan(&sink.input, ctx, proto_converter)?;- let data_sink: JsonSink = sink- .sink- .as_ref()- .ok_or_else(|| proto_error("Missing required field in protobuf"))?- .try_into()?;+ let data_sink = JsonSink::try_from_proto(+ sink.sink+ .as_ref()+ .ok_or_else(|| proto_error("Missing required field in protobuf"))?,+ )?;let sink_schema = input.schema();let sort_order = sink.sort_order@@ -2000,11 +2017,11 @@ impl protobuf::PhysicalPlanNode {) -> Result<Arc<dyn ExecutionPlan>> {let input = into_physical_plan(&sink.input, ctx, proto_converter)?;- let data_sink: CsvSink = sink- .sink- .as_ref()- .ok_or_else(|| proto_error("Missing required field in protobuf"))?- .try_into()?;+ let data_sink = CsvSink::try_from_proto(+ sink.sink+ .as_ref()+ .ok_or_else(|| proto_error("Missing required field in protobuf"))?,+ )?;let sink_schema = input.schema();let sort_order = sink.sort_order@@ -2040,11 +2057,11 @@ impl protobuf::PhysicalPlanNode {{let input = into_physical_plan(&sink.input, ctx, proto_converter)?;- let data_sink: ParquetSink = sink- .sink- .as_ref()- .ok_or_else(|| proto_error("Missing required field in protobuf"))?- .try_into()?;+ let data_sink = ParquetSink::try_from_proto(+ sink.sink+ .as_ref()+ .ok_or_else(|| proto_error("Missing required field in protobuf"))?,+ )?;let sink_schema = input.schema();let sort_order = sink.sort_order@@ -2092,7 +2109,11 @@ impl protobuf::PhysicalPlanNode {.collect(),unnest.struct_type_columns.iter().map(|c| *c as _).collect(),Arc::new(convert_required!(unnest.schema)?),- into_required!(unnest.options)?,+ unnest+ .options+ .as_ref()+ .map(datafusion_common::UnnestOptions::from_proto)+ .ok_or_else(|| proto_error("Missing required field in protobuf"))?,)?))}@@ -2204,9 +2225,9 @@ impl protobuf::PhysicalPlanNode {right,on,filter,- join_type.into(),+ JoinType::from_proto(join_type),sort_options,- null_equality.into(),+ NullEquality::from_proto(null_equality),)?))}@@ -2219,7 +2240,9 @@ impl protobuf::PhysicalPlanNode {let args = match &generate_series.args {Some(protobuf::generate_series_node::Args::ContainsNull(args)) => {GenSeriesArgs::ContainsNull {- name: Self::generate_series_name_to_str(args.name()),+ name: protobuf::PhysicalPlanNode::generate_series_name_to_str(+ args.name(),+ ),}}Some(protobuf::generate_series_node::Args::Int64Args(args)) => {@@ -2228,7 +2251,9 @@ impl protobuf::PhysicalPlanNode {end: args.end,step: args.step,include_end: args.include_end,- name: Self::generate_series_name_to_str(args.name()),+ name: protobuf::PhysicalPlanNode::generate_series_name_to_str(+ args.name(),+ ),}}Some(protobuf::generate_series_node::Args::TimestampArgs(args)) => {@@ -2246,7 +2271,9 @@ impl protobuf::PhysicalPlanNode {step,tz: args.tz.as_ref().map(|s| Arc::from(s.as_str())),include_end: args.include_end,- name: Self::generate_series_name_to_str(args.name()),+ name: protobuf::PhysicalPlanNode::generate_series_name_to_str(+ args.name(),+ ),}}Some(protobuf::generate_series_node::Args::DateArgs(args)) => {@@ -2263,7 +2290,9 @@ impl protobuf::PhysicalPlanNode {end: args.end,step,include_end: args.include_end,- name: Self::generate_series_name_to_str(args.name()),+ name: protobuf::PhysicalPlanNode::generate_series_name_to_str(+ args.name(),+ ),}}None => return internal_err!("Missing args in GenerateSeriesNode"),@@ -2371,7 +2400,7 @@ impl protobuf::PhysicalPlanNode {fn try_from_explain_exec(exec: &ExplainExec,_codec: &dyn PhysicalExtensionCodec,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Explain(protobuf::ExplainExecNode {@@ -2379,7 +2408,7 @@ impl protobuf::PhysicalPlanNode {stringified_plans: exec.stringified_plans().iter()- .map(|plan| plan.into())+ .map(protobuf::StringifiedPlan::from_proto).collect(),verbose: exec.verbose(),},@@ -2391,7 +2420,7 @@ impl protobuf::PhysicalPlanNode {exec: &ProjectionExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -2424,7 +2453,7 @@ impl protobuf::PhysicalPlanNode {exec: &AnalyzeExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -2452,7 +2481,7 @@ impl protobuf::PhysicalPlanNode {exec: &FilterExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -2484,7 +2513,7 @@ impl protobuf::PhysicalPlanNode {limit: &GlobalLimitExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(limit.input().to_owned(),codec,@@ -2509,7 +2538,7 @@ impl protobuf::PhysicalPlanNode {limit: &LocalLimitExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(limit.input().to_owned(),codec,@@ -2529,7 +2558,7 @@ impl protobuf::PhysicalPlanNode {exec: &HashJoinExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),codec,@@ -2552,8 +2581,8 @@ impl protobuf::PhysicalPlanNode {})}).collect::<Result<_>>()?;- let join_type: protobuf::JoinType = exec.join_type().to_owned().into();- let null_equality: protobuf::NullEquality = exec.null_equality().into();+ let join_type = protobuf::JoinType::from_proto(exec.join_type().to_owned());+ let null_equality = protobuf::NullEquality::from_proto(exec.null_equality());let filter = exec.filter().as_ref()@@ -2619,7 +2648,7 @@ impl protobuf::PhysicalPlanNode {exec: &SymmetricHashJoinExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),codec,@@ -2642,8 +2671,8 @@ impl protobuf::PhysicalPlanNode {})}).collect::<Result<_>>()?;- let join_type: protobuf::JoinType = exec.join_type().to_owned().into();- let null_equality: protobuf::NullEquality = exec.null_equality().into();+ let join_type = protobuf::JoinType::from_proto(exec.join_type().to_owned());+ let null_equality = protobuf::NullEquality::from_proto(exec.null_equality());let filter = exec.filter().as_ref()@@ -2740,7 +2769,7 @@ impl protobuf::PhysicalPlanNode {exec: &SortMergeJoinExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),codec,@@ -2763,8 +2792,8 @@ impl protobuf::PhysicalPlanNode {})}).collect::<Result<_>>()?;- let join_type: protobuf::JoinType = exec.join_type().to_owned().into();- let null_equality: protobuf::NullEquality = exec.null_equality().into();+ let join_type = protobuf::JoinType::from_proto(exec.join_type().to_owned());+ let null_equality = protobuf::NullEquality::from_proto(exec.null_equality());let filter = exec.filter().as_ref()@@ -2827,7 +2856,7 @@ impl protobuf::PhysicalPlanNode {exec: &CrossJoinExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),codec,@@ -2852,7 +2881,7 @@ impl protobuf::PhysicalPlanNode {exec: &AggregateExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let groups: Vec<bool> = exec.group_expr().groups()@@ -2955,7 +2984,7 @@ impl protobuf::PhysicalPlanNode {fn try_from_empty_exec(empty: &EmptyExec,_codec: &dyn PhysicalExtensionCodec,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let schema = empty.schema().as_ref().try_into()?;Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Empty(protobuf::EmptyExecNode {@@ -2967,7 +2996,7 @@ impl protobuf::PhysicalPlanNode {fn try_from_placeholder_row_exec(empty: &PlaceholderRowExec,_codec: &dyn PhysicalExtensionCodec,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let schema = empty.schema().as_ref().try_into()?;Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::PlaceholderRow(@@ -2983,7 +3012,7 @@ impl protobuf::PhysicalPlanNode {coalesce_batches: &CoalesceBatchesExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(coalesce_batches.input().to_owned(),codec,@@ -3004,7 +3033,7 @@ impl protobuf::PhysicalPlanNode {data_source_exec: &DataSourceExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Option<Self>> {+ ) -> Result<Option<protobuf::PhysicalPlanNode>> {let data_source = data_source_exec.data_source();if let Some(maybe_csv) = data_source.downcast_ref::<FileScanConfig>() {let source = maybe_csv.file_source();@@ -3176,7 +3205,7 @@ impl protobuf::PhysicalPlanNode {exec: &CoalescePartitionsExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -3196,7 +3225,7 @@ impl protobuf::PhysicalPlanNode {exec: &RepartitionExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -3221,7 +3250,7 @@ impl protobuf::PhysicalPlanNode {exec: &SortExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = proto_converter.execution_plan_to_proto(exec.input(), codec)?;let expr = exec.expr()@@ -3268,7 +3297,7 @@ impl protobuf::PhysicalPlanNode {union: &UnionExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let mut inputs: Vec<protobuf::PhysicalPlanNode> = vec![];for input in union.inputs() {inputs.push(@@ -3290,7 +3319,7 @@ impl protobuf::PhysicalPlanNode {interleave: &InterleaveExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let mut inputs: Vec<protobuf::PhysicalPlanNode> = vec![];for input in interleave.inputs() {inputs.push(@@ -3312,7 +3341,7 @@ impl protobuf::PhysicalPlanNode {exec: &SortPreservingMergeExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -3350,7 +3379,7 @@ impl protobuf::PhysicalPlanNode {exec: &NestedLoopJoinExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.left().to_owned(),codec,@@ -3362,7 +3391,7 @@ impl protobuf::PhysicalPlanNode {proto_converter,)?;- let join_type: protobuf::JoinType = exec.join_type().to_owned().into();+ let join_type = protobuf::JoinType::from_proto(exec.join_type().to_owned());let filter = exec.filter().as_ref()@@ -3408,7 +3437,7 @@ impl protobuf::PhysicalPlanNode {exec: &WindowAggExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -3443,7 +3472,7 @@ impl protobuf::PhysicalPlanNode {exec: &BoundedWindowAggExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -3494,7 +3523,7 @@ impl protobuf::PhysicalPlanNode {exec: &DataSinkExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Option<Self>> {+ ) -> Result<Option<protobuf::PhysicalPlanNode>> {let input: protobuf::PhysicalPlanNode =protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),@@ -3530,7 +3559,7 @@ impl protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::JsonSink(Box::new(protobuf::JsonSinkExecNode {input: Some(Box::new(input)),- sink: Some(sink.try_into()?),+ sink: Some(protobuf::JsonSink::try_from_proto(sink)?),sink_schema: Some(exec.schema().as_ref().try_into()?),sort_order,},@@ -3543,7 +3572,7 @@ impl protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::CsvSink(Box::new(protobuf::CsvSinkExecNode {input: Some(Box::new(input)),- sink: Some(sink.try_into()?),+ sink: Some(protobuf::CsvSink::try_from_proto(sink)?),sink_schema: Some(exec.schema().as_ref().try_into()?),sort_order,},@@ -3557,7 +3586,7 @@ impl protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::ParquetSink(Box::new(protobuf::ParquetSinkExecNode {input: Some(Box::new(input)),- sink: Some(sink.try_into()?),+ sink: Some(protobuf::ParquetSink::try_from_proto(sink)?),sink_schema: Some(exec.schema().as_ref().try_into()?),sort_order,},@@ -3573,7 +3602,7 @@ impl protobuf::PhysicalPlanNode {exec: &UnnestExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -3598,7 +3627,7 @@ impl protobuf::PhysicalPlanNode {.iter().map(|c| *c as _).collect(),- options: Some(exec.options().into()),+ options: Some(protobuf::UnnestOptions::from_proto(exec.options())),},))),})@@ -3608,7 +3637,7 @@ impl protobuf::PhysicalPlanNode {exec: &CooperativeExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(exec.input().to_owned(),codec,@@ -3632,7 +3661,9 @@ impl protobuf::PhysicalPlanNode {}}- fn try_from_lazy_memory_exec(exec: &LazyMemoryExec) -> Result<Option<Self>> {+ fn try_from_lazy_memory_exec(+ exec: &LazyMemoryExec,+ ) -> Result<Option<protobuf::PhysicalPlanNode>> {let generators = exec.generators();// ensure we only have one generator@@ -3650,7 +3681,9 @@ impl protobuf::PhysicalPlanNode {target_batch_size: 8192, // Default batch sizeargs: Some(protobuf::generate_series_node::Args::ContainsNull(protobuf::GenerateSeriesArgsContainsNull {- name: Self::str_to_generate_series_name(empty_gen.name())? as i32,+ name: protobuf::PhysicalPlanNode::str_to_generate_series_name(+ empty_gen.name(),+ )? as i32,},)),};@@ -3674,7 +3707,9 @@ impl protobuf::PhysicalPlanNode {end: *int_64.end(),step: *int_64.step(),include_end: int_64.include_end(),- name: Self::str_to_generate_series_name(int_64.name())? as i32,+ name: protobuf::PhysicalPlanNode::str_to_generate_series_name(+ int_64.name(),+ )? as i32,},)),};@@ -3701,7 +3736,9 @@ impl protobuf::PhysicalPlanNode {nanos: step_value.nanoseconds,});let include_end = timestamp_args.include_end();- let name = Self::str_to_generate_series_name(timestamp_args.name())? as i32;+ let name = protobuf::PhysicalPlanNode::str_to_generate_series_name(+ timestamp_args.name(),+ )? as i32;let args = match timestamp_args.current().tz_str() {Some(tz) => protobuf::generate_series_node::Args::TimestampArgs(@@ -3743,7 +3780,7 @@ impl protobuf::PhysicalPlanNode {exec: &AsyncFuncExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(Arc::clone(exec.input()),codec,@@ -3774,7 +3811,7 @@ impl protobuf::PhysicalPlanNode {exec: &BufferExec,extension_codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(Arc::clone(exec.input()),extension_codec,@@ -3795,7 +3832,7 @@ impl protobuf::PhysicalPlanNode {exec: &ScalarSubqueryExec,codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,- ) -> Result<Self> {+ ) -> Result<protobuf::PhysicalPlanNode> {let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(Arc::clone(exec.input()),codec,@@ -3824,6 +3861,12 @@ impl protobuf::PhysicalPlanNode {}}+impl PhysicalPlanNodeExt for protobuf::PhysicalPlanNode {+ fn node(&self) -> &protobuf::PhysicalPlanNode {+ self+ }+}+pub trait AsExecutionPlan: Debug + Send + Sync + Clone {fn try_decode(buf: &[u8]) -> Result<Self>wherediff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex ec8e168178..5181c97401 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -36,9 +36,8 @@ use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_plan::expressions::{- BinaryExpr, CaseExpr, CastExpr, Column, DynamicFilterPhysicalExpr, InListExpr,- IsNotNullExpr, IsNullExpr, LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr,- UnKnownColumn,+ CaseExpr, CastExpr, DynamicFilterPhysicalExpr, InListExpr, IsNotNullExpr, IsNullExpr,+ LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,};use datafusion_physical_plan::joins::{HashExpr, HashTableLookupExpr};use datafusion_physical_plan::udaf::AggregateFunctionExpr;@@ -49,6 +48,7 @@ use super::{DefaultPhysicalProtoConverter, PhysicalExtensionCodec,PhysicalProtoConverterExtension, encode_human_display_alias,};+use crate::convert::TryFromProto;use crate::protobuf::{self, PhysicalSortExprNode, PhysicalSortExprNodeCollection,physical_aggregate_expr_node, physical_window_expr_node,@@ -178,9 +178,7 @@ pub fn serialize_physical_window_expr(codec,proto_converter,)?;- let window_frame: protobuf::WindowFrame = window_frame- .as_ref()- .try_into()+ let window_frame = protobuf::WindowFrame::try_from_proto(window_frame.as_ref()).map_err(|e| internal_datafusion_err!("{e}"))?;Ok(protobuf::PhysicalWindowExprNode {@@ -253,6 +251,29 @@ pub fn serialize_physical_expr()}+/// Concrete [`PhysicalExprEncode`] driver used to back+/// [`PhysicalExprEncodeCtx`] when expressions invoke `PhysicalExpr::to_proto`.+///+/// Wraps the existing extension codec + converter pair so individual+/// expressions can recurse into children without depending on+/// `datafusion-proto` directly.+///+/// [`PhysicalExprEncode`]: datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncode+/// [`PhysicalExprEncodeCtx`]: datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx+struct ConverterEncoder<'a> {+ codec: &'a dyn PhysicalExtensionCodec,+ proto_converter: &'a dyn PhysicalProtoConverterExtension,+}++impl datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncode+ for ConverterEncoder<'_>+{+ fn encode(&self, expr: &Arc<dyn PhysicalExpr>) -> Result<protobuf::PhysicalExprNode> {+ self.proto_converter+ .physical_expr_to_proto(expr, self.codec)+ }+}+/// Serialize a `PhysicalExpr` to default protobuf representation.////// If required, a [`PhysicalExtensionCodec`] can be provided which can handle@@ -266,6 +287,21 @@ pub fn serialize_physical_expr_with_converter() -> Result<protobuf::PhysicalExprNode> {let expr = value.as_ref();let expr_id = value.expression_id();++ // Give the expression a chance to serialize itself first. Returning+ // `Ok(Some(node))` lets expressions with private state (e.g.+ // `DynamicFilterPhysicalExpr`) avoid exposing pub-for-proto accessors.+ // `Ok(None)` falls through to the downcast chain below — that's the+ // default for built-in expressions which haven't been migrated yet.+ let encoder = ConverterEncoder {+ codec,+ proto_converter,+ };+ let ctx = datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx::new(&encoder);+ if let Some(node) = expr.try_to_proto(&ctx)? {+ return Ok(node);+ }+// HashTableLookupExpr is used for dynamic filter pushdown in hash joins.// It contains an Arc<dyn JoinHashMapType> (the build-side hash table) which// cannot be serialized - the hash table is a runtime structure built during@@ -291,17 +327,7 @@ pub fn serialize_physical_expr_with_converter(});}- if let Some(expr) = expr.downcast_ref::<Column>() {- Ok(protobuf::PhysicalExprNode {- expr_id,- expr_type: Some(protobuf::physical_expr_node::ExprType::Column(- protobuf::PhysicalColumn {- name: expr.name().to_string(),- index: expr.index() as u32,- },- )),- })- } else if let Some(expr) = expr.downcast_ref::<UnKnownColumn>() {+ if let Some(expr) = expr.downcast_ref::<UnKnownColumn>() {Ok(protobuf::PhysicalExprNode {expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::UnknownColumn(@@ -310,46 +336,6 @@ pub fn serialize_physical_expr_with_converter(},)),})- } else if let Some(expr) = expr.downcast_ref::<BinaryExpr>() {- // Linearize a nested binary expression tree of the same operator- // into a flat vector of operands to avoid deep recursion in proto.- let op = expr.op();- let mut operand_refs: Vec<&Arc<dyn PhysicalExpr>> = vec![expr.right()];- let mut current_expr: &BinaryExpr = expr;- loop {- match current_expr.left().downcast_ref::<BinaryExpr>() {- Some(bin) if bin.op() == op => {- operand_refs.push(bin.right());- current_expr = bin;- }- _ => {- operand_refs.push(current_expr.left());- break;- }- }- }-- // Reverse so operands are ordered from left innermost to right outermost- operand_refs.reverse();-- let operands = operand_refs- .iter()- .map(|e| proto_converter.physical_expr_to_proto(e, codec))- .collect::<Result<Vec<_>>>()?;-- let binary_expr = Box::new(protobuf::PhysicalBinaryExprNode {- l: None,- r: None,- op: format!("{:?}", op),- operands,- });-- Ok(protobuf::PhysicalExprNode {- expr_id,- expr_type: Some(protobuf::physical_expr_node::ExprType::BinaryExpr(- binary_expr,- )),- })} else if let Some(expr) = expr.downcast_ref::<CaseExpr>() {Ok(protobuf::PhysicalExprNode {expr_id,@@ -642,10 +628,10 @@ fn serialize_when_then_expr(})}-impl TryFrom<&PartitionedFile> for protobuf::PartitionedFile {+impl TryFromProto<&PartitionedFile> for protobuf::PartitionedFile {type Error = DataFusionError;- fn try_from(pf: &PartitionedFile) -> Result<Self> {+ fn try_from_proto(pf: &PartitionedFile) -> Result<Self> {let last_modified = pf.object_meta.last_modified;let last_modified_ns = last_modified.timestamp_nanos_opt().ok_or_else(|| {DataFusionError::Plan(format!(@@ -661,16 +647,20 @@ impl TryFrom<&PartitionedFile> for protobuf::PartitionedFile {.iter().map(|v| v.try_into()).collect::<Result<Vec<_>, _>>()?,- range: pf.range.as_ref().map(|r| r.try_into()).transpose()?,+ range: pf+ .range+ .as_ref()+ .map(protobuf::FileRange::try_from_proto)+ .transpose()?,statistics: pf.statistics.as_ref().map(|s| s.as_ref().into()),})}}-impl TryFrom<&FileRange> for protobuf::FileRange {+impl TryFromProto<&FileRange> for protobuf::FileRange {type Error = DataFusionError;- fn try_from(value: &FileRange) -> Result<Self> {+ fn try_from_proto(value: &FileRange) -> Result<Self> {Ok(protobuf::FileRange {start: value.start,end: value.end,@@ -678,14 +668,14 @@ impl TryFrom<&FileRange> for protobuf::FileRange {}}-impl TryFrom<&[PartitionedFile]> for protobuf::FileGroup {+impl TryFromProto<&[PartitionedFile]> for protobuf::FileGroup {type Error = DataFusionError;- fn try_from(gr: &[PartitionedFile]) -> Result<Self, Self::Error> {+ fn try_from_proto(gr: &[PartitionedFile]) -> Result<Self, Self::Error> {Ok(protobuf::FileGroup {files: gr.iter()- .map(|f| f.try_into())+ .map(protobuf::PartitionedFile::try_from_proto).collect::<Result<Vec<_>, _>>()?,})}@@ -699,7 +689,7 @@ pub fn serialize_file_scan_config(let file_groups = conf.file_groups.iter()- .map(|p| p.files().try_into())+ .map(|p| protobuf::FileGroup::try_from_proto(p.files())).collect::<Result<Vec<_>, _>>()?;let mut output_orderings = vec![];@@ -797,48 +787,48 @@ pub fn serialize_record_batches(batches: &[RecordBatch]) -> Result<Vec<u8>> {Ok(buf)}-impl TryFrom<&JsonSink> for protobuf::JsonSink {+impl TryFromProto<&JsonSink> for protobuf::JsonSink {type Error = DataFusionError;- fn try_from(value: &JsonSink) -> Result<Self, Self::Error> {+ fn try_from_proto(value: &JsonSink) -> Result<Self, Self::Error> {Ok(Self {- config: Some(value.config().try_into()?),+ config: Some(protobuf::FileSinkConfig::try_from_proto(value.config())?),writer_options: Some(value.writer_options().try_into()?),})}}-impl TryFrom<&CsvSink> for protobuf::CsvSink {+impl TryFromProto<&CsvSink> for protobuf::CsvSink {type Error = DataFusionError;- fn try_from(value: &CsvSink) -> Result<Self, Self::Error> {+ fn try_from_proto(value: &CsvSink) -> Result<Self, Self::Error> {Ok(Self {- config: Some(value.config().try_into()?),+ config: Some(protobuf::FileSinkConfig::try_from_proto(value.config())?),writer_options: Some(value.writer_options().try_into()?),})}}#[cfg(feature = "parquet")]-impl TryFrom<&ParquetSink> for protobuf::ParquetSink {+impl TryFromProto<&ParquetSink> for protobuf::ParquetSink {type Error = DataFusionError;- fn try_from(value: &ParquetSink) -> Result<Self, Self::Error> {+ fn try_from_proto(value: &ParquetSink) -> Result<Self, Self::Error> {Ok(Self {- config: Some(value.config().try_into()?),+ config: Some(protobuf::FileSinkConfig::try_from_proto(value.config())?),parquet_options: Some(value.parquet_options().try_into()?),})}}-impl TryFrom<&FileSinkConfig> for protobuf::FileSinkConfig {+impl TryFromProto<&FileSinkConfig> for protobuf::FileSinkConfig {type Error = DataFusionError;- fn try_from(conf: &FileSinkConfig) -> Result<Self, Self::Error> {+ fn try_from_proto(conf: &FileSinkConfig) -> Result<Self, Self::Error> {let file_groups = conf.file_group.iter()- .map(TryInto::try_into)+ .map(protobuf::PartitionedFile::try_from_proto).collect::<Result<Vec<_>>>()?;let table_paths = conf.table_pathsdiff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex c2b62c1745..58b79d641b 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -128,7 +128,7 @@ use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_convuse datafusion_proto::physical_plan::{AsExecutionPlan, DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext,- PhysicalProtoConverterExtension,+ PhysicalPlanNodeExt, PhysicalProtoConverterExtension,};use datafusion_proto::protobuf;use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txtindex f5ce368df7..77da7db87e 100644--- a/dev/release/rat_exclude_files.txt+++ b/dev/release/rat_exclude_files.txt@@ -52,9 +52,9 @@ Cargo.lock.historyparquet-testing/**rat.txt-datafusion/proto/src/generated/datafusion_proto_common.rs-datafusion/proto/src/generated/pbjson.rs-datafusion/proto/src/generated/prost.rs+datafusion/proto-models/src/generated/datafusion_proto_common.rs+datafusion/proto-models/src/generated/pbjson.rs+datafusion/proto-models/src/generated/prost.rsdatafusion/proto-common/src/generated/pbjson.rsdatafusion/proto-common/src/generated/prost.rs.github/ISSUE_TEMPLATE/bug_report.ymldiff --git a/licenserc.toml b/licenserc.tomlindex 105d969ea5..a1e01a5fd0 100644--- a/licenserc.toml+++ b/licenserc.toml@@ -27,4 +27,5 @@ excludes = [# generated code"datafusion/proto/src/generated/","datafusion/proto-common/src/generated/",+ "datafusion/proto-models/src/generated/",]