Apache DataFusion SQL Query Engine
refactor(proto): put Partitioning / sort-expression serde on the types (#24003)
Adrian Garcia Badaraccoa589b4bbae72
diff --git a/datafusion/physical-expr-common/src/sort_expr.rs b/datafusion/physical-expr-common/src/sort_expr.rsindex 84ffb92eaa..72e8772347 100644--- a/datafusion/physical-expr-common/src/sort_expr.rs+++ b/datafusion/physical-expr-common/src/sort_expr.rs@@ -183,6 +183,102 @@ impl PhysicalSortExpr {}}+/// Protobuf conversions for [`PhysicalSortExpr`].+///+/// This is the flat [`PhysicalSortExprNode`] representation used wherever the+/// wire format stores an ordering (scan output orderings, range partitioning,+/// window frames, …). It is *not* the `PhysicalExprNode::Sort` wrapping that+/// `SortExec` uses for its own `expr` field.+///+/// [`PhysicalSortExprNode`]: datafusion_proto_models::protobuf::PhysicalSortExprNode+#[cfg(feature = "proto")]+impl PhysicalSortExpr {+ /// Serialize this sort expression, encoding its child expression through+ /// `ctx`.+ pub fn try_to_proto(+ &self,+ ctx: &crate::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<datafusion_proto_models::protobuf::PhysicalSortExprNode> {+ Ok(datafusion_proto_models::protobuf::PhysicalSortExprNode {+ expr: Some(Box::new(ctx.encode_child(&self.expr)?)),+ asc: !self.options.descending,+ nulls_first: self.options.nulls_first,+ })+ }++ /// Reconstruct a [`PhysicalSortExpr`] from its protobuf representation.+ pub fn try_from_proto(+ node: &datafusion_proto_models::protobuf::PhysicalSortExprNode,+ ctx: &crate::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+ ) -> Result<Self> {+ let expr = ctx.decode_required_expression(+ node.expr.as_deref(),+ "PhysicalSortExpr",+ "expr",+ )?;+ Ok(PhysicalSortExpr {+ expr,+ options: SortOptions {+ descending: !node.asc,+ nulls_first: node.nulls_first,+ },+ })+ }+}++/// Serialize a sequence of sort expressions into the flat+/// [`PhysicalSortExprNode`] list the wire format uses for an ordering.+///+/// Accepts anything that yields [`PhysicalSortExpr`]s by value or by reference,+/// so a [`LexOrdering`], a `&[PhysicalSortExpr]`, or a [`LexRequirement`]+/// mapped through [`PhysicalSortExpr::from`] all work:+///+/// ```ignore+/// let nodes = sort_exprs_try_to_proto(ordering.iter(), ctx)?;+/// let nodes = sort_exprs_try_to_proto(+/// requirement.iter().map(|req| PhysicalSortExpr::from(req.clone())),+/// ctx,+/// )?;+/// ```+///+/// The `PhysicalSortExprNodeCollection` message some plans use is just this+/// list in a wrapper, so those callers wrap the result themselves rather than+/// this function guessing which shape they mean.+///+/// [`PhysicalSortExprNode`]: datafusion_proto_models::protobuf::PhysicalSortExprNode+#[cfg(feature = "proto")]+pub fn sort_exprs_try_to_proto<E: std::borrow::Borrow<PhysicalSortExpr>>(+ exprs: impl IntoIterator<Item = E>,+ ctx: &crate::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+) -> Result<Vec<datafusion_proto_models::protobuf::PhysicalSortExprNode>> {+ exprs+ .into_iter()+ .map(|expr| expr.borrow().try_to_proto(ctx))+ .collect()+}++/// Reconstruct a sequence of sort expressions from the flat+/// [`PhysicalSortExprNode`] list, the counterpart of+/// [`sort_exprs_try_to_proto`].+///+/// Returns the expressions rather than a [`LexOrdering`] or a+/// [`LexRequirement`], because callers differ in what an empty list means:+/// `LexOrdering::new` / `LexRequirement::new` return `None` for it, which is+/// "no ordering declared" for a scan and an error for an operator that requires+/// one.+///+/// [`PhysicalSortExprNode`]: datafusion_proto_models::protobuf::PhysicalSortExprNode+#[cfg(feature = "proto")]+pub fn sort_exprs_try_from_proto(+ nodes: &[datafusion_proto_models::protobuf::PhysicalSortExprNode],+ ctx: &crate::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+) -> Result<Vec<PhysicalSortExpr>> {+ nodes+ .iter()+ .map(|node| PhysicalSortExpr::try_from_proto(node, ctx))+ .collect()+}+impl PartialEq for PhysicalSortExpr {fn eq(&self, other: &Self) -> bool {self.options == other.options && self.expr.eq(&other.expr)diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rsindex 59d36c4efc..98f082f725 100644--- a/datafusion/physical-expr/src/partitioning.rs+++ b/datafusion/physical-expr/src/partitioning.rs@@ -25,6 +25,10 @@ pub use datafusion_common::SplitPoint;use datafusion_common::{Result, validate_range_split_points};use datafusion_physical_expr_common::physical_expr::format_physical_expr_list;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};+#[cfg(feature = "proto")]+use datafusion_physical_expr_common::sort_expr::{+ sort_exprs_try_from_proto, sort_exprs_try_to_proto,+};use std::fmt;use std::fmt::Display;use std::sync::Arc;@@ -515,6 +519,156 @@ impl Partitioning {}}+/// Protobuf conversions for [`Partitioning`].+///+/// Child expressions (hash keys, range orderings) and `ScalarValue` split+/// points are (de)serialized through the expression-level context, so this is+/// the single copy of the partitioning wire format: `RepartitionExec` and+/// `datafusion-proto`'s central serializer route through it, and the remaining+/// per-plan migrations (`FileScanConfig` and friends) are meant to do the same+/// rather than grow another copy.+///+/// [`protobuf::Partitioning`]: datafusion_proto_models::protobuf::Partitioning+#[cfg(feature = "proto")]+impl Partitioning {+ /// Serialize this partitioning into its protobuf representation.+ pub fn try_to_proto(+ &self,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<datafusion_proto_models::protobuf::Partitioning> {+ use datafusion_proto_models::protobuf;++ let partition_method = match self {+ Partitioning::RoundRobinBatch(n) => {+ protobuf::partitioning::PartitionMethod::RoundRobin(wire_partition_count(+ *n,+ )?)+ }+ Partitioning::Hash(exprs, n) => {+ protobuf::partitioning::PartitionMethod::Hash(+ protobuf::PhysicalHashRepartition {+ hash_expr: ctx.encode_children_expressions(exprs)?,+ partition_count: wire_partition_count(*n)?,+ },+ )+ }+ Partitioning::Range(range) => {+ let sort_expr = sort_exprs_try_to_proto(range.ordering().iter(), ctx)?;+ let split_point = range+ .split_points()+ .iter()+ .map(|split_point| {+ let value = split_point+ .values()+ .iter()+ .map(|value| value.try_into().map_err(Into::into))+ .collect::<Result<Vec<_>>>()?;+ Ok(protobuf::PhysicalRangeSplitPoint { value })+ })+ .collect::<Result<Vec<_>>>()?;+ protobuf::partitioning::PartitionMethod::Range(+ protobuf::PhysicalRangePartitioning {+ sort_expr,+ split_point,+ },+ )+ }+ Partitioning::UnknownPartitioning(n) => {+ protobuf::partitioning::PartitionMethod::Unknown(wire_partition_count(+ *n,+ )?)+ }+ };+ Ok(protobuf::Partitioning {+ partition_method: Some(partition_method),+ })+ }++ /// Reconstruct a [`Partitioning`] from its protobuf representation.+ ///+ /// Returns `Ok(None)` when the message carries no `partition_method`, which+ /// the wire format uses to mean "no output partitioning declared"; callers+ /// for which it is required should turn that into their own error.+ pub fn try_from_proto(+ node: &datafusion_proto_models::protobuf::Partitioning,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+ ) -> Result<Option<Self>> {+ use datafusion_common::{ScalarValue, internal_datafusion_err, internal_err};+ use datafusion_proto_models::protobuf;++ let Some(partition_method) = node.partition_method.as_ref() else {+ return Ok(None);+ };+ let partitioning = match partition_method {+ protobuf::partitioning::PartitionMethod::RoundRobin(n) => {+ Partitioning::RoundRobinBatch(partition_count(*n)?)+ }+ protobuf::partitioning::PartitionMethod::Hash(hash) => {+ let exprs = hash+ .hash_expr+ .iter()+ .map(|expr| ctx.decode(expr))+ .collect::<Result<Vec<_>>>()?;+ Partitioning::Hash(exprs, partition_count(hash.partition_count)?)+ }+ protobuf::partitioning::PartitionMethod::Unknown(n) => {+ Partitioning::UnknownPartitioning(partition_count(*n)?)+ }+ protobuf::partitioning::PartitionMethod::Range(range) => {+ let sort_exprs = sort_exprs_try_from_proto(&range.sort_expr, ctx)?;+ let sort_expr_count = sort_exprs.len();+ let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {+ internal_datafusion_err!(+ "Range partitioning requires non-empty ordering"+ )+ })?;+ if ordering.len() != sort_expr_count {+ return internal_err!(+ "Range partitioning ordering must not contain duplicate expressions"+ );+ }+ let split_points = range+ .split_point+ .iter()+ .map(|split_point| {+ let values = split_point+ .value+ .iter()+ .map(|value| ScalarValue::try_from(value).map_err(Into::into))+ .collect::<Result<Vec<_>>>()?;+ Ok(SplitPoint::new(values))+ })+ .collect::<Result<Vec<_>>>()?;+ Partitioning::Range(RangePartitioning::try_new(ordering, split_points)?)+ }+ };+ Ok(Some(partitioning))+ }+}++/// Narrow a wire partition count to `usize`.+#[cfg(feature = "proto")]+fn partition_count(count: u64) -> Result<usize> {+ usize::try_from(count).map_err(|_| {+ datafusion_common::internal_datafusion_err!(+ "Partition count {count} exceeds usize::MAX"+ )+ })+}++/// Widen a partition count to its `u64` wire representation.+///+/// The mirror of [`partition_count`]: an out-of-range count is an error on both+/// sides rather than a silent truncation on the way out.+#[cfg(feature = "proto")]+fn wire_partition_count(count: usize) -> Result<u64> {+ u64::try_from(count).map_err(|_| {+ datafusion_common::internal_datafusion_err!(+ "Partition count {count} exceeds u64::MAX"+ )+ })+}+impl PartialEq for Partitioning {fn eq(&self, other: &Partitioning) -> bool {match (self, other) {@@ -1138,3 +1292,239 @@ mod tests {Ok(())}}++#[cfg(all(test, feature = "proto"))]+mod ordering_proto_tests {+ use std::sync::Arc;++ use arrow::compute::SortOptions;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;+ use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+ use datafusion_physical_expr_common::sort_expr::{+ LexRequirement, PhysicalSortExpr, PhysicalSortRequirement,+ sort_exprs_try_from_proto, sort_exprs_try_to_proto,+ };++ use crate::expressions::Column;+ use crate::proto_test_util::{StubDecoder, StubEncoder};++ fn schema() -> Schema {+ Schema::new(vec![Field::new("a", DataType::Int32, false)])+ }++ fn sort_expr(descending: bool, nulls_first: bool) -> PhysicalSortExpr {+ PhysicalSortExpr::new(+ Arc::new(Column::new("a", 0)),+ SortOptions {+ descending,+ nulls_first,+ },+ )+ }++ #[test]+ fn sort_exprs_round_trip_preserves_options_and_order() {+ let encoder = StubEncoder::ok();+ let encode_ctx = PhysicalExprEncodeCtx::new(&encoder);+ let exprs = vec![sort_expr(true, false), sort_expr(false, true)];++ let nodes = sort_exprs_try_to_proto(&exprs, &encode_ctx).unwrap();+ // `asc` is the inverse of `descending` on the wire.+ assert_eq!(+ nodes+ .iter()+ .map(|node| (node.asc, node.nulls_first))+ .collect::<Vec<_>>(),+ vec![(false, false), (true, true)]+ );++ let schema = schema();+ let decoder = StubDecoder::ok();+ let decode_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);+ let decoded = sort_exprs_try_from_proto(&nodes, &decode_ctx).unwrap();+ assert_eq!(+ decoded.iter().map(|expr| expr.options).collect::<Vec<_>>(),+ exprs.iter().map(|expr| expr.options).collect::<Vec<_>>()+ );+ }++ #[test]+ fn sort_exprs_accepts_owned_requirements() {+ let encoder = StubEncoder::ok();+ let encode_ctx = PhysicalExprEncodeCtx::new(&encoder);+ let requirement = LexRequirement::from([PhysicalSortRequirement::new(+ Arc::new(Column::new("a", 0)),+ Some(SortOptions {+ descending: true,+ nulls_first: true,+ }),+ )]);++ let nodes = sort_exprs_try_to_proto(+ requirement+ .iter()+ .map(|req| PhysicalSortExpr::from(req.clone())),+ &encode_ctx,+ )+ .unwrap();++ assert_eq!(nodes.len(), 1);+ assert!(!nodes[0].asc);+ assert!(nodes[0].nulls_first);+ }++ #[test]+ fn sort_exprs_propagate_encode_errors() {+ let encoder = StubEncoder::failing_on(2);+ let encode_ctx = PhysicalExprEncodeCtx::new(&encoder);+ let exprs = vec![sort_expr(false, false), sort_expr(true, true)];++ let err = sort_exprs_try_to_proto(&exprs, &encode_ctx).unwrap_err();+ assert!(err.to_string().contains("stub encode failure on call 2"));+ }++ #[test]+ fn sort_exprs_reject_missing_inner_expr() {+ let encoder = StubEncoder::ok();+ let encode_ctx = PhysicalExprEncodeCtx::new(&encoder);+ let mut nodes =+ sort_exprs_try_to_proto(&[sort_expr(false, false)], &encode_ctx).unwrap();+ nodes[0].expr = None;++ let schema = schema();+ let decoder = StubDecoder::ok();+ let decode_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);+ let err = sort_exprs_try_from_proto(&nodes, &decode_ctx).unwrap_err();+ assert!(+ err.to_string()+ .contains("PhysicalSortExpr is missing required field 'expr'")+ );+ }+}++/// Partition counts are `usize` in memory and `u64` on the wire, so every+/// counted [`Partitioning`] variant crosses a width boundary in both+/// directions. These pin that neither crossing wraps or panics.+#[cfg(all(test, feature = "proto"))]+mod partition_count_proto_tests {+ use std::sync::Arc;++ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;+ use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;+ use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+ use datafusion_proto_models::protobuf;++ use super::{Partitioning, partition_count, wire_partition_count};+ use crate::expressions::Column;+ use crate::proto_test_util::{StubDecoder, StubEncoder, column_node};++ fn partitioning_node(+ method: protobuf::partitioning::PartitionMethod,+ ) -> protobuf::Partitioning {+ protobuf::Partitioning {+ partition_method: Some(method),+ }+ }++ /// The counted variants, each carrying `count`. `Range` is excluded: it+ /// derives its partition count from its split points rather than reading+ /// one off the wire.+ fn counted_methods(count: u64) -> Vec<protobuf::partitioning::PartitionMethod> {+ use protobuf::partitioning::PartitionMethod;++ vec![+ PartitionMethod::RoundRobin(count),+ PartitionMethod::Unknown(count),+ PartitionMethod::Hash(protobuf::PhysicalHashRepartition {+ hash_expr: vec![column_node("a")],+ partition_count: count,+ }),+ ]+ }++ #[test]+ fn partition_count_round_trips_at_the_usize_ceiling() {+ // `usize::MAX` is the largest count that can exist in memory, so it has+ // to widen onto the wire and narrow back unchanged.+ let wire = wire_partition_count(usize::MAX).unwrap();+ assert_eq!(wire, u64::try_from(usize::MAX).unwrap());+ assert_eq!(partition_count(wire).unwrap(), usize::MAX);+ }++ #[test]+ fn out_of_range_partition_count_is_reported_not_wrapped() {+ // A count wider than the target's `usize` can only be reached by+ // decoding on a narrower host than the one that encoded. That used to+ // wrap (`as usize`) or panic (`unwrap`); it is an error now. On a+ // 64-bit target every `u64` fits, so the same input has to decode+ // losslessly instead of being rejected.+ let narrowed = partition_count(u64::MAX);++ #[cfg(target_pointer_width = "64")]+ assert_eq!(narrowed.unwrap(), usize::MAX);++ #[cfg(not(target_pointer_width = "64"))]+ assert!(+ narrowed+ .unwrap_err()+ .to_string()+ .contains("Partition count 18446744073709551615 exceeds usize::MAX")+ );+ }++ #[test]+ fn try_from_proto_narrows_every_counted_variant() {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let decoder = StubDecoder::ok();+ let decode_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);++ for method in counted_methods(u64::MAX) {+ let decoded =+ Partitioning::try_from_proto(&partitioning_node(method), &decode_ctx);++ #[cfg(target_pointer_width = "64")]+ assert_eq!(decoded.unwrap().unwrap().partition_count(), usize::MAX);++ #[cfg(not(target_pointer_width = "64"))]+ assert!(+ decoded+ .unwrap_err()+ .to_string()+ .contains("exceeds usize::MAX")+ );+ }+ }++ #[test]+ fn try_to_proto_widens_every_counted_variant() {+ use protobuf::partitioning::PartitionMethod;++ let encoder = StubEncoder::ok();+ let encode_ctx = PhysicalExprEncodeCtx::new(&encoder);+ let hash_key: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));++ let encoded = [+ Partitioning::RoundRobinBatch(usize::MAX),+ Partitioning::UnknownPartitioning(usize::MAX),+ Partitioning::Hash(vec![hash_key], usize::MAX),+ ]+ .iter()+ .map(|partitioning| {+ match partitioning+ .try_to_proto(&encode_ctx)+ .unwrap()+ .partition_method+ {+ Some(PartitionMethod::RoundRobin(n) | PartitionMethod::Unknown(n)) => n,+ Some(PartitionMethod::Hash(hash)) => hash.partition_count,+ other => panic!("expected a counted partition method, got {other:?}"),+ }+ })+ .collect::<Vec<_>>();++ // Every variant widens to the same wire value, with no truncation.+ assert_eq!(encoded, vec![u64::try_from(usize::MAX).unwrap(); 3]);+ }+}diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex db1eb951d6..0523628ad7 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -2264,17 +2264,11 @@ fn encode_aggregate_expr(let expressions = aggr_expr.expressions();let expr = ctx.encode_expressions(expressions.iter())?;- let ordering_req = aggr_expr- .order_bys()- .iter()- .map(|sort_expr| {- Ok(protobuf::PhysicalSortExprNode {- expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)),- asc: !sort_expr.options.descending,- nulls_first: sort_expr.options.nulls_first,- })- })- .collect::<Result<Vec<_>>>()?;+ let ordering_req =+ datafusion_physical_expr_common::sort_expr::sort_exprs_try_to_proto(+ aggr_expr.order_bys(),+ &ctx.expr_ctx(),+ )?;let name = aggr_expr.fun().name().to_string();// The context already applies `(!buf.is_empty()).then_some(buf)`.let fun_definition = ctx.encode_udaf(aggr_expr.fun())?;@@ -2314,7 +2308,6 @@ impl AggregateExec {node: &datafusion_proto_models::protobuf::PhysicalPlanNode,ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,) -> Result<Arc<dyn ExecutionPlan>> {- use datafusion_physical_expr::PhysicalSortExpr;use datafusion_physical_expr::aggregate::AggregateExprBuilder;use datafusion_proto_models::protobuf;use protobuf::physical_aggregate_expr_node::AggregateFunction;@@ -2421,24 +2414,11 @@ impl AggregateExec {.iter().map(|expr| ctx.decode_expr(expr, input_schema.as_ref())).collect::<Result<Vec<_>>>()?;- let order_by = aggregate- .ordering_req- .iter()- .map(|sort_expr| {- let expr = sort_expr.expr.as_deref().ok_or_else(|| {- datafusion_common::internal_datafusion_err!(- "AggregateExec ordering expression is missing its inner expr"- )- })?;- Ok(PhysicalSortExpr {- expr: ctx.decode_expr(expr, input_schema.as_ref())?,- options: arrow::compute::SortOptions {- descending: !sort_expr.asc,- nulls_first: sort_expr.nulls_first,- },- })- })- .collect::<Result<Vec<_>>>()?;+ let order_by =+ datafusion_physical_expr_common::sort_expr::sort_exprs_try_from_proto(+ &aggregate.ordering_req,+ &ctx.expr_ctx(input_schema.as_ref()),+ )?;let Some(AggregateFunction::UserDefinedAggrFunction(udaf_name)) =aggregate.aggregate_function.as_ref()else {@@ -2448,10 +2428,8 @@ impl AggregateExec {};// The context owns the payload-to-codec and// registry-to-codec fallback order.- let udaf = ctx.decode_udaf(- udaf_name,- aggregate.fun_definition.as_deref(),- )?;+ let udaf =+ ctx.decode_udaf(udaf_name, aggregate.fun_definition.as_deref())?;let (human_display, human_display_alias) =split_human_display_alias(&aggregate.human_display, name);let builder = AggregateExprBuilder::new(udaf, args)diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex 95f4c35871..eb358b10b4 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -698,23 +698,18 @@ impl ExecutionPlan for SymmetricHashJoinExec {})}).transpose()?;+ let expr_ctx = ctx.expr_ctx();let encode_sort_exprs =|exprs: Option<&LexOrdering>| -> Result<Vec<protobuf::PhysicalSortExprNode>> {- exprs- .map(|exprs| {- exprs- .iter()- .map(|expr| {- Ok(protobuf::PhysicalSortExprNode {- expr: Some(Box::new(ctx.encode_expr(&expr.expr)?)),- asc: !expr.options.descending,- nulls_first: expr.options.nulls_first,- })- })- .collect::<Result<Vec<_>>>()- })- .transpose()- .map(Option::unwrap_or_default)+ exprs.map_or_else(+ || Ok(vec![]),+ |exprs| {+ datafusion_physical_expr_common::sort_expr::sort_exprs_try_to_proto(+ exprs.iter(),+ &expr_ctx,+ )+ },+ )};let left_sort_exprs = encode_sort_exprs(self.left_sort_exprs())?;let right_sort_exprs = encode_sort_exprs(self.right_sort_exprs())?;@@ -751,7 +746,6 @@ impl SymmetricHashJoinExec {ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,) -> Result<Arc<dyn ExecutionPlan>> {use datafusion_common::internal_datafusion_err;- use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_proto_models::protobuf;let sym_join = crate::expect_plan_variant!(@@ -883,39 +877,19 @@ impl SymmetricHashJoinExec {}).transpose()?;let decode_sort_exprs = |sort_exprs: &[protobuf::PhysicalSortExprNode],- schema: &Schema,- field: &str|+ schema: &Schema|-> Result<Option<LexOrdering>> {- let sort_exprs = sort_exprs- .iter()- .map(|sort_expr| {- let expr = ctx.decode_required_expr(- sort_expr.expr.as_deref(),- schema,- "SymmetricHashJoinExec",- field,- )?;- Ok(PhysicalSortExpr {- expr,- options: arrow::compute::SortOptions {- descending: !sort_expr.asc,- nulls_first: sort_expr.nulls_first,- },- })- })- .collect::<Result<Vec<_>>>()?;+ let sort_exprs =+ datafusion_physical_expr_common::sort_expr::sort_exprs_try_from_proto(+ sort_exprs,+ &ctx.expr_ctx(schema),+ )?;Ok(LexOrdering::new(sort_exprs))};- let left_sort_exprs = decode_sort_exprs(- &sym_join.left_sort_exprs,- left_schema.as_ref(),- "left_sort_exprs",- )?;- let right_sort_exprs = decode_sort_exprs(- &sym_join.right_sort_exprs,- right_schema.as_ref(),- "right_sort_exprs",- )?;+ let left_sort_exprs =+ decode_sort_exprs(&sym_join.left_sort_exprs, left_schema.as_ref())?;+ let right_sort_exprs =+ decode_sort_exprs(&sym_join.right_sort_exprs, right_schema.as_ref())?;Self::try_new(left,diff --git a/datafusion/physical-plan/src/proto.rs b/datafusion/physical-plan/src/proto.rsindex f84cd67d46..7640d76c3e 100644--- a/datafusion/physical-plan/src/proto.rs+++ b/datafusion/physical-plan/src/proto.rs@@ -66,6 +66,12 @@ use datafusion_execution::TaskContext;use datafusion_expr::physical_planning_context::ScalarSubqueryResults;use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};use datafusion_physical_expr::PhysicalExpr;+use datafusion_physical_expr_common::physical_expr::proto_decode::{+ PhysicalExprDecode, PhysicalExprDecodeCtx,+};+use datafusion_physical_expr_common::physical_expr::proto_encode::{+ PhysicalExprEncode, PhysicalExprEncodeCtx,+};use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalPlanNode};use crate::ExecutionPlan;@@ -210,6 +216,25 @@ impl<'a> ExecutionPlanEncodeCtx<'a> {pub fn encode_udwf(&self, udwf: &WindowUDF) -> Result<Option<Vec<u8>>> {self.encoder.encode_udwf(udwf)}++ /// An expression-level encode context backed by this plan context.+ ///+ /// Lets a plan hand `ctx` to expression-level conversions that own their own+ /// wire logic — e.g.+ /// [`Partitioning::try_to_proto`](datafusion_physical_expr::Partitioning::try_to_proto)+ /// and+ /// [`PhysicalSortExpr::try_to_proto`](datafusion_physical_expr::PhysicalSortExpr::try_to_proto).+ pub fn expr_ctx(&self) -> PhysicalExprEncodeCtx<'_> {+ PhysicalExprEncodeCtx::new(self)+ }+}++/// Lets [`ExecutionPlanEncodeCtx`] back a [`PhysicalExprEncodeCtx`], so+/// expression-level conversions can be reused from plan hooks.+impl PhysicalExprEncode for ExecutionPlanEncodeCtx<'_> {+ fn encode(&self, expr: &Arc<dyn PhysicalExpr>) -> Result<PhysicalExprNode> {+ self.encode_expr(expr)+ }}/// Context handed to a plan's `try_from_proto` associated function.@@ -317,6 +342,28 @@ impl<'a> ExecutionPlanDecodeCtx<'a> {) -> Result<Arc<WindowUDF>> {self.decoder.decode_udwf(name, payload)}++ /// An expression-level decode context backed by this plan context, bound to+ /// `input_schema`.+ ///+ /// The decode counterpart of+ /// [`ExecutionPlanEncodeCtx::expr_ctx`], for calling conversions such as+ /// [`Partitioning::try_from_proto`](datafusion_physical_expr::Partitioning::try_from_proto).+ pub fn expr_ctx<'s>(&'s self, input_schema: &'s Schema) -> PhysicalExprDecodeCtx<'s> {+ PhysicalExprDecodeCtx::new(input_schema, self)+ }+}++/// Lets [`ExecutionPlanDecodeCtx`] back a [`PhysicalExprDecodeCtx`], so+/// expression-level conversions can be reused from plan hooks.+impl PhysicalExprDecode for ExecutionPlanDecodeCtx<'_> {+ fn decode(+ &self,+ node: &PhysicalExprNode,+ schema: &Schema,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ self.decode_expr(node, schema)+ }}/// Assert that a [`PhysicalPlanNode`] carries the expected `PhysicalPlanType`diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 3473aad9b3..873f35fd6a 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -1713,64 +1713,14 @@ impl ExecutionPlan for RepartitionExec {let input = ctx.encode_child(self.input())?;- // Keep the existing protobuf wire representation unchanged.- let partition_method = match self.partitioning() {- Partitioning::RoundRobinBatch(n) => {- protobuf::partitioning::PartitionMethod::RoundRobin(*n as u64)- }- Partitioning::Hash(exprs, n) => {- let hash_expr = ctx.encode_expressions(exprs)?;- protobuf::partitioning::PartitionMethod::Hash(- protobuf::PhysicalHashRepartition {- hash_expr,- partition_count: *n as u64,- },- )- }- Partitioning::Range(range) => {- let sort_expr = range- .ordering()- .iter()- .map(|sort_expr| {- Ok(protobuf::PhysicalSortExprNode {- expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)),- asc: !sort_expr.options.descending,- nulls_first: sort_expr.options.nulls_first,- })- })- .collect::<Result<Vec<_>>>()?;- let split_point = range- .split_points()- .iter()- .map(|split_point| {- let value = split_point- .values()- .iter()- .map(|value| value.try_into().map_err(Into::into))- .collect::<Result<Vec<_>>>()?;- Ok(protobuf::PhysicalRangeSplitPoint { value })- })- .collect::<Result<Vec<_>>>()?;- protobuf::partitioning::PartitionMethod::Range(- protobuf::PhysicalRangePartitioning {- sort_expr,- split_point,- },- )- }- Partitioning::UnknownPartitioning(n) => {- protobuf::partitioning::PartitionMethod::Unknown(*n as u64)- }- };+ let partitioning = self.partitioning().try_to_proto(&ctx.expr_ctx())?;Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(protobuf::physical_plan_node::PhysicalPlanType::Repartition(Box::new(protobuf::RepartitionExecNode {input: Some(Box::new(input)),- partitioning: Some(protobuf::Partitioning {- partition_method: Some(partition_method),- }),+ partitioning: Some(partitioning),preserve_order: self.preserve_order(),},)),@@ -1800,84 +1750,23 @@ impl RepartitionExec {)?;let input_schema = input.schema();- let partition_method = repart+ let partitioning = repart.partitioning.as_ref()- .and_then(|p| p.partition_method.as_ref())+ .map(|partitioning| {+ Partitioning::try_from_proto(+ partitioning,+ &ctx.expr_ctx(input_schema.as_ref()),+ )+ })+ .transpose()?+ .flatten().ok_or_else(|| {datafusion_common::internal_datafusion_err!("RepartitionExec is missing required field 'partitioning'")})?;- let partitioning = match partition_method {- protobuf::partitioning::PartitionMethod::RoundRobin(n) => {- Partitioning::RoundRobinBatch(*n as usize)- }- protobuf::partitioning::PartitionMethod::Hash(hash) => {- let exprs = hash- .hash_expr- .iter()- .map(|expr| ctx.decode_expr(expr, input_schema.as_ref()))- .collect::<Result<Vec<_>>>()?;- let partition_count =- usize::try_from(hash.partition_count).map_err(|_| {- datafusion_common::internal_datafusion_err!(- "Hash partition count {} exceeds usize::MAX",- hash.partition_count- )- })?;- Partitioning::Hash(exprs, partition_count)- }- protobuf::partitioning::PartitionMethod::Unknown(n) => {- Partitioning::UnknownPartitioning(*n as usize)- }- protobuf::partitioning::PartitionMethod::Range(range) => {- let sort_exprs = range- .sort_expr- .iter()- .map(|sort_expr| {- let expr = sort_expr.expr.as_ref().ok_or_else(|| {- datafusion_common::internal_datafusion_err!(- "Unexpected empty physical expression"- )- })?;- Ok(PhysicalSortExpr {- expr: ctx.decode_expr(expr, input_schema.as_ref())?,- options: SortOptions {- descending: !sort_expr.asc,- nulls_first: sort_expr.nulls_first,- },- })- })- .collect::<Result<Vec<_>>>()?;- let sort_expr_count = sort_exprs.len();- let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {- datafusion_common::internal_datafusion_err!(- "Range partitioning requires non-empty ordering"- )- })?;- if ordering.len() != sort_expr_count {- return datafusion_common::internal_err!(- "Range partitioning ordering must not contain duplicate expressions"- );- }- let split_points = range- .split_point- .iter()- .map(|split_point| {- let values = split_point- .value- .iter()- .map(|value| ScalarValue::try_from(value).map_err(Into::into))- .collect::<Result<Vec<_>>>()?;- Ok(SplitPoint::new(values))- })- .collect::<Result<Vec<_>>>()?;- Partitioning::Range(RangePartitioning::try_new(ordering, split_points)?)- }- };-let mut repart_exec = RepartitionExec::try_new(input, partitioning)?;if repart.preserve_order {repart_exec = repart_exec.with_preserve_order();diff --git a/datafusion/physical-plan/src/windows/proto.rs b/datafusion/physical-plan/src/windows/proto.rsindex aa62158d18..e96b0a9fb1 100644--- a/datafusion/physical-plan/src/windows/proto.rs+++ b/datafusion/physical-plan/src/windows/proto.rs@@ -19,7 +19,6 @@use std::sync::Arc;-use arrow::compute::SortOptions;use arrow::datatypes::Schema;use datafusion_common::{Result, ScalarValue, internal_datafusion_err, internal_err, not_impl_err,@@ -28,7 +27,9 @@ use datafusion_expr::{WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition,};use datafusion_physical_expr::window::SlidingAggregateWindowExpr;-use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+use datafusion_physical_expr_common::sort_expr::{+ sort_exprs_try_from_proto, sort_exprs_try_to_proto,+};use datafusion_proto_common::protobuf_common;use datafusion_proto_models::protobuf::{self, physical_window_expr_node};@@ -93,17 +94,7 @@ pub(super) fn encode_physical_window_expr(let args = ctx.encode_expressions(&args)?;let partition_by = ctx.encode_expressions(window_expr.partition_by())?;- let order_by = window_expr- .order_by()- .iter()- .map(|sort_expr| {- Ok(protobuf::PhysicalSortExprNode {- expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)),- asc: !sort_expr.options.descending,- nulls_first: sort_expr.options.nulls_first,- })- })- .collect::<Result<Vec<_>>>()?;+ let order_by = sort_exprs_try_to_proto(window_expr.order_by(), &ctx.expr_ctx())?;Ok(protobuf::PhysicalWindowExprNode {args,@@ -133,24 +124,8 @@ pub(super) fn decode_physical_window_expr(.iter().map(|expr| ctx.decode_expr(expr, input_schema)).collect::<Result<Vec<_>>>()?;- let order_by = proto- .order_by- .iter()- .map(|sort_expr| {- let expr = sort_expr.expr.as_ref().ok_or_else(|| {- internal_datafusion_err!(- "Missing expr in window order_by sort expression"- )- })?;- Ok(PhysicalSortExpr {- expr: ctx.decode_expr(expr, input_schema)?,- options: SortOptions {- descending: !sort_expr.asc,- nulls_first: sort_expr.nulls_first,- },- })- })- .collect::<Result<Vec<_>>>()?;+ let order_by =+ sort_exprs_try_from_proto(&proto.order_by, &ctx.expr_ctx(input_schema))?;let window_frame = proto.window_frame.as_ref()diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 34ad8c7a62..60647bd7aa 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -23,9 +23,7 @@ use arrow::array::RecordBatch;use arrow::compute::SortOptions;use arrow::datatypes::{Field, Schema};use arrow::ipc::reader::StreamReader;-use datafusion_common::{- DataFusionError, Result, ScalarValue, internal_datafusion_err, not_impl_err,-};+use datafusion_common::{DataFusionError, Result, internal_datafusion_err, not_impl_err};use datafusion_datasource::file::FileSource;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};@@ -51,9 +49,7 @@ use datafusion_physical_plan::expressions::{};use datafusion_physical_plan::joins::HashExpr;use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};-use datafusion_physical_plan::{- Partitioning, PhysicalExpr, RangePartitioning, SplitPoint, WindowExpr,-};+use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};use datafusion_proto_common::common::proto_error;use super::{@@ -399,22 +395,16 @@ pub fn parse_protobuf_hash_partitioning(input_schema: &Schema,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Option<Partitioning>> {- match partitioning {- Some(hash_part) => {- let expr = parse_physical_exprs(- &hash_part.hash_expr,- ctx,- input_schema,- proto_converter,- )?;-- Ok(Some(Partitioning::Hash(- expr,- hash_part.partition_count.try_into().unwrap(),- )))- }- None => Ok(None),- }+ // Delegate to the shared decoder rather than keep a second copy of the hash+ // wire format: a partition count that does not fit in `usize` (a 32-bit+ // target reading a plan written on a 64-bit one) is then an error here too+ // instead of a panic.+ let hash = partitioning.map(|hash_part| protobuf::Partitioning {+ partition_method: Some(protobuf::partitioning::PartitionMethod::Hash(+ hash_part.clone(),+ )),+ });+ parse_protobuf_partitioning(hash.as_ref(), ctx, input_schema, proto_converter)}pub fn parse_protobuf_partitioning(@@ -423,83 +413,20 @@ pub fn parse_protobuf_partitioning(input_schema: &Schema,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<Option<Partitioning>> {- match partitioning {- Some(protobuf::Partitioning { partition_method }) => match partition_method {- Some(protobuf::partitioning::PartitionMethod::RoundRobin(- partition_count,- )) => Ok(Some(Partitioning::RoundRobinBatch(- *partition_count as usize,- ))),- Some(protobuf::partitioning::PartitionMethod::Hash(hash_repartition)) => {- parse_protobuf_hash_partitioning(- Some(hash_repartition),- ctx,- input_schema,- proto_converter,- )- }- Some(protobuf::partitioning::PartitionMethod::Range(range_partitioning)) => {- Ok(Some(parse_protobuf_range_partitioning(- range_partitioning,- ctx,- input_schema,- proto_converter,- )?))- }- Some(protobuf::partitioning::PartitionMethod::Unknown(partition_count)) => {- Ok(Some(Partitioning::UnknownPartitioning(- *partition_count as usize,- )))- }- None => Ok(None),- },- None => Ok(None),- }-}--fn parse_protobuf_range_partitioning(- range_partitioning: &protobuf::PhysicalRangePartitioning,- ctx: &PhysicalPlanDecodeContext<'_>,- input_schema: &Schema,- proto_converter: &dyn PhysicalProtoConverterExtension,-) -> Result<Partitioning> {- let sort_exprs = parse_physical_sort_exprs(- &range_partitioning.sort_expr,+ let decoder = ConverterDecoder {ctx,- input_schema,proto_converter,- )?;- let sort_expr_count = sort_exprs.len();- let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {- internal_datafusion_err!("Range partitioning requires non-empty ordering")- })?;- if ordering.len() != sort_expr_count {- return Err(internal_datafusion_err!(- "Range partitioning ordering must not contain duplicate expressions"- ));- }- let split_points = range_partitioning- .split_point- .iter()- .map(parse_protobuf_range_split_point)- .collect::<Result<_>>()?;- Ok(Partitioning::Range(RangePartitioning::try_new(- ordering,- split_points,- )?))-}--fn parse_protobuf_range_split_point(- split_point: &protobuf::PhysicalRangeSplitPoint,-) -> Result<SplitPoint> {- let values = split_point- .value- .iter()- .map(|value| ScalarValue::try_from(value).map_err(Into::into))- .collect::<Result<_>>()?;- Ok(SplitPoint::new(values))+ };+ let decode_ctx =+ datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx::new(+ input_schema,+ &decoder,+ );+ partitioning+ .map(|partitioning| Partitioning::try_from_proto(partitioning, &decode_ctx))+ .transpose()+ .map(Option::flatten)}-pub fn parse_protobuf_file_scan_schema(proto: &protobuf::FileScanExecConf,) -> Result<Arc<Schema>> {@@ -776,6 +703,7 @@ mod tests {use super::*;use arrow::datatypes::{DataType, Field, Schema};use chrono::{TimeZone, Utc};+ use datafusion_common::ScalarValue;use object_store::ObjectMeta;use object_store::path::Path;diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex 5189972f0e..bab7af2ab4 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -36,9 +36,7 @@ use datafusion_physical_expr::{HigherOrderFunctionExpr, ScalarFunctionExpr};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_plan::udaf::AggregateFunctionExpr;use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};-use datafusion_physical_plan::{- Partitioning, PhysicalExpr, RangePartitioning, SplitPoint, WindowExpr,-};+use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};use super::{DefaultPhysicalProtoConverter, PhysicalExtensionCodec,@@ -358,70 +356,13 @@ pub fn serialize_partitioning(codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<protobuf::Partitioning> {- let serialized_partitioning = match partitioning {- Partitioning::RoundRobinBatch(partition_count) => protobuf::Partitioning {- partition_method: Some(protobuf::partitioning::PartitionMethod::RoundRobin(- *partition_count as u64,- )),- },- Partitioning::Hash(exprs, partition_count) => {- let serialized_exprs =- serialize_physical_exprs(exprs, codec, proto_converter)?;- protobuf::Partitioning {- partition_method: Some(protobuf::partitioning::PartitionMethod::Hash(- protobuf::PhysicalHashRepartition {- hash_expr: serialized_exprs,- partition_count: *partition_count as u64,- },- )),- }- }- Partitioning::Range(range) => protobuf::Partitioning {- partition_method: Some(protobuf::partitioning::PartitionMethod::Range(- serialize_range_partitioning(range, codec, proto_converter)?,- )),- },- Partitioning::UnknownPartitioning(partition_count) => protobuf::Partitioning {- partition_method: Some(protobuf::partitioning::PartitionMethod::Unknown(- *partition_count as u64,- )),- },+ let encoder = ConverterEncoder {+ codec,+ proto_converter,};- Ok(serialized_partitioning)-}--fn serialize_range_partitioning(- range: &RangePartitioning,- codec: &dyn PhysicalExtensionCodec,- proto_converter: &dyn PhysicalProtoConverterExtension,-) -> Result<protobuf::PhysicalRangePartitioning> {- Ok(protobuf::PhysicalRangePartitioning {- sort_expr: serialize_physical_sort_exprs(- range.ordering().iter().cloned(),- codec,- proto_converter,- )?,- split_point: range- .split_points()- .iter()- .map(serialize_range_split_point)- .collect::<Result<_>>()?,- })-}--fn serialize_range_split_point(- split_point: &SplitPoint,-) -> Result<protobuf::PhysicalRangeSplitPoint> {- Ok(protobuf::PhysicalRangeSplitPoint {- value: split_point- .values()- .iter()- .map(|value| {- TryInto::<datafusion_proto_common::ScalarValue>::try_into(value)- .map_err(Into::into)- })- .collect::<Result<_>>()?,- })+ partitioning.try_to_proto(+ &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx::new(&encoder),+ )}/// Thin shim over `TryFrom<&PartitionedFile>`, which owns the wire logic.diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 864e6d6867..508ba5c020 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -2374,6 +2374,89 @@ fn roundtrip_range_partitioning() -> Result<()> {roundtrip_test(Arc::new(repartition))}+/// `parse_protobuf_hash_partitioning` has no in-tree callers left; it delegates+/// to the shared `Partitioning::try_from_proto`, so pin that it still decodes+/// the hash message it is handed.+#[test]+fn parse_hash_partitioning_delegates_to_shared_decoder() -> Result<()> {+ use datafusion_proto::physical_plan::from_proto::parse_protobuf_hash_partitioning;++ let schema = Schema::new(vec![Field::new("a", DataType::Int64, false)]);+ let ctx = SessionContext::new();+ let task_ctx = ctx.task_ctx();+ let codec = DefaultPhysicalExtensionCodec {};+ let decode_ctx = PhysicalPlanDecodeContext::new(&task_ctx, &codec);+ let proto_converter = DefaultPhysicalProtoConverter {};++ let hash_expr = serialize_physical_expr_with_converter(+ &col("a", &schema)?,+ &codec,+ &proto_converter,+ )?;+ let hash = protobuf::PhysicalHashRepartition {+ hash_expr: vec![hash_expr],+ partition_count: 4,+ };++ let partitioning = parse_protobuf_hash_partitioning(+ Some(&hash),+ &decode_ctx,+ &schema,+ &proto_converter,+ )?;+ let Some(Partitioning::Hash(exprs, count)) = partitioning else {+ panic!("expected hash partitioning, got {partitioning:?}");+ };+ assert_eq!(count, 4);+ assert_eq!(exprs.len(), 1);+ assert_eq!(exprs[0].to_string(), col("a", &schema)?.to_string());++ // No message means no partitioning, as before.+ assert!(+ parse_protobuf_hash_partitioning(None, &decode_ctx, &schema, &proto_converter)?+ .is_none()+ );++ // The count is a `u64` on the wire and a `usize` in memory, so decoding+ // narrows it. A count that does not fit is the case that motivated routing+ // this through the shared decoder: it used to `unwrap()` and panic, and now+ // reports an error. Only a target narrower than 64 bits can reach that arm+ // -- on a 64-bit target every `u64` fits, and the assertion there is that+ // the largest possible count survives whole rather than being truncated.+ let oversized = protobuf::PhysicalHashRepartition {+ hash_expr: vec![serialize_physical_expr_with_converter(+ &col("a", &schema)?,+ &codec,+ &proto_converter,+ )?],+ partition_count: u64::MAX,+ };+ let decoded = parse_protobuf_hash_partitioning(+ Some(&oversized),+ &decode_ctx,+ &schema,+ &proto_converter,+ );++ #[cfg(target_pointer_width = "64")]+ {+ let Some(Partitioning::Hash(_, count)) = decoded? else {+ panic!("expected hash partitioning");+ };+ assert_eq!(count, usize::MAX);+ }++ #[cfg(not(target_pointer_width = "64"))]+ assert!(+ decoded+ .unwrap_err()+ .to_string()+ .contains("Partition count 18446744073709551615 exceeds usize::MAX")+ );++ Ok(())+}+#[test]fn roundtrip_interleave() -> Result<()> {let field_a = Field::new("col", DataType::Int64, false);