Apache DataFusion SQL Query Engine
feat: destructure proto hooks (#25204)
RIchard Baahecb0fea9e372
diff --git a/datafusion/physical-expr-common/src/sort_expr.rs b/datafusion/physical-expr-common/src/sort_expr.rsindex 072fb641e3..43073ec36e 100644--- a/datafusion/physical-expr-common/src/sort_expr.rs+++ b/datafusion/physical-expr-common/src/sort_expr.rs@@ -199,10 +199,11 @@ impl PhysicalSortExpr {&self,ctx: &crate::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,) -> Result<datafusion_proto_models::protobuf::PhysicalSortExprNode> {+ let Self { expr, options } = self;Ok(datafusion_proto_models::protobuf::PhysicalSortExprNode {- expr: Some(Box::new(ctx.encode_child(&self.expr)?)),- asc: !self.options.descending,- nulls_first: self.options.nulls_first,+ expr: Some(Box::new(ctx.encode_child(expr)?)),+ asc: !options.descending,+ nulls_first: options.nulls_first,})}@@ -211,16 +212,18 @@ impl PhysicalSortExpr {node: &datafusion_proto_models::protobuf::PhysicalSortExprNode,ctx: &crate::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,) -> Result<Self> {- let expr = ctx.decode_required_expression(- node.expr.as_deref(),- "PhysicalSortExpr",- "expr",- )?;+ let datafusion_proto_models::protobuf::PhysicalSortExprNode {+ expr,+ asc,+ nulls_first,+ } = node;+ let expr =+ ctx.decode_required_expression(expr.as_deref(), "PhysicalSortExpr", "expr")?;Ok(PhysicalSortExpr {expr,options: SortOptions {- descending: !node.asc,- nulls_first: node.nulls_first,+ descending: !asc,+ nulls_first: *nulls_first,},})}diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rsindex e1ca2df922..052fec6ebe 100644--- a/datafusion/physical-expr/src/partitioning.rs+++ b/datafusion/physical-expr/src/partitioning.rs@@ -629,9 +629,12 @@ impl Partitioning {)}Partitioning::Range(range) => {- let sort_expr = sort_exprs_try_to_proto(range.ordering().iter(), ctx)?;- let split_point = range- .split_points()+ let RangePartitioning {+ ordering,+ split_points,+ } = range;+ let sort_expr = sort_exprs_try_to_proto(ordering.iter(), ctx)?;+ let split_point = split_points.iter().map(|split_point| {let value = split_point@@ -673,7 +676,8 @@ impl Partitioning {let partition_count =|count: u64| usize_from_wire(count, "Partitioning", "partition_count");- let Some(partition_method) = node.partition_method.as_ref() else {+ let protobuf::Partitioning { partition_method } = node;+ let Some(partition_method) = partition_method.as_ref() else {return Ok(None);};let partitioning = match partition_method {@@ -681,18 +685,25 @@ impl Partitioning {Partitioning::RoundRobinBatch(partition_count(*n)?)}protobuf::partitioning::PartitionMethod::Hash(hash) => {- let exprs = hash- .hash_expr+ let protobuf::PhysicalHashRepartition {+ hash_expr,+ partition_count: hash_partition_count,+ } = hash;+ let exprs = hash_expr.iter().map(|expr| ctx.decode(expr)).collect::<Result<Vec<_>>>()?;- Partitioning::Hash(exprs, partition_count(hash.partition_count)?)+ Partitioning::Hash(exprs, partition_count(*hash_partition_count)?)}protobuf::partitioning::PartitionMethod::Unknown(n) => {Partitioning::UnknownPartitioning(partition_count(*n)?)}protobuf::partitioning::PartitionMethod::Range(range) => {- let sort_exprs = sort_exprs_try_from_proto(&range.sort_expr, ctx)?;+ let protobuf::PhysicalRangePartitioning {+ sort_expr,+ split_point,+ } = range;+ let sort_exprs = sort_exprs_try_from_proto(sort_expr, ctx)?;let sort_expr_count = sort_exprs.len();let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {internal_datafusion_err!(@@ -704,12 +715,11 @@ impl Partitioning {"Range partitioning ordering must not contain duplicate expressions");}- let split_points = range- .split_point+ let split_points = split_point.iter().map(|split_point| {- let values = split_point- .value+ let protobuf::PhysicalRangeSplitPoint { value } = split_point;+ let values = value.iter().map(|value| ScalarValue::try_from(value).map_err(Into::into)).collect::<Result<Vec<_>>>()?;@@ -1575,13 +1585,16 @@ mod ordering_proto_tests {mod partition_count_proto_tests {use std::sync::Arc;+ use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::{ScalarValue, SplitPoint};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;+ use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use datafusion_proto_models::protobuf;- use super::Partitioning;+ use super::{Partitioning, RangePartitioning};use crate::expressions::Column;use crate::proto_test_util::{StubDecoder, StubEncoder, column_node};@@ -1662,4 +1675,51 @@ mod partition_count_proto_tests {// Every variant widens to the same wire value, with no truncation.assert_eq!(encoded, vec![u64::try_from(usize::MAX).unwrap(); 3]);}++ #[test]+ fn range_partitioning_round_trip_preserves_split_points_and_options() {+ // Single-column ordering so the stub encoder (which emits an identical+ // placeholder for every expression) cannot produce duplicates that the+ // decoder's deduplication step would collapse.+ let sort_options = SortOptions {+ descending: false,+ nulls_first: true,+ };+ let ordering = LexOrdering::from([PhysicalSortExpr::new(+ Arc::new(Column::new("a", 0)),+ sort_options,+ )]);+ let split_points = vec![+ SplitPoint::new(vec![ScalarValue::Int32(Some(10))]),+ SplitPoint::new(vec![ScalarValue::Int32(Some(30))]),+ ];+ let range =+ RangePartitioning::try_new(ordering.clone(), split_points.clone()).unwrap();++ let encoder = StubEncoder::ok();+ let encode_ctx = PhysicalExprEncodeCtx::new(&encoder);+ let node = Partitioning::Range(range)+ .try_to_proto(&encode_ctx)+ .unwrap();++ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let decoder = StubDecoder::ok();+ let decode_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);+ let decoded = Partitioning::try_from_proto(&node, &decode_ctx)+ .unwrap()+ .unwrap();++ let Partitioning::Range(decoded_range) = decoded else {+ panic!("expected Range partitioning, got {decoded:?}");+ };+ assert_eq!(+ decoded_range+ .ordering()+ .iter()+ .map(|e| e.options)+ .collect::<Vec<_>>(),+ [sort_options],+ );+ assert_eq!(decoded_range.split_points(), split_points);+ }}diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex b653a74fa5..794888b4ea 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -849,17 +849,16 @@ impl PhysicalExpr for RangeExpr {&self,ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,) -> Result<Option<protobuf::PhysicalExprNode>> {+ let Self { on_columns, router } = self;// Encode the raw ordered children: rebuilding a `LexOrdering` would// deduplicate equivalent children after dynamic-filter remapping.- let sort_exprs = self- .on_columns+ let sort_exprs = on_columns.iter()- .zip(self.router.sort_options())+ .zip(router.sort_options()).map(|(expr, options)| PhysicalSortExpr::new(Arc::clone(expr), *options)).collect::<Vec<_>>();let sort_expr = sort_exprs_try_to_proto(&sort_exprs, ctx)?;- let split_point = self- .router+ let split_point = router.split_points().iter().map(|split_point| {@@ -890,24 +889,31 @@ impl RangeExpr {node: &protobuf::PhysicalExprNode,ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,) -> Result<PhysicalExprRef> {+ let protobuf::PhysicalExprNode {+ expr_id: _,+ expr_type,+ } = node;// Decode the raw ordered children for the same reason as `try_to_proto`.let Some(protobuf::physical_expr_node::ExprType::RangeExpr(range_expr)) =- &node.expr_type+ expr_typeelse {return internal_err!("PhysicalExprNode is not a RangeExpr");};- let sort_exprs = sort_exprs_try_from_proto(&range_expr.sort_expr, ctx)?;+ let protobuf::PhysicalRangeExprNode {+ sort_expr,+ split_point,+ } = range_expr;+ let sort_exprs = sort_exprs_try_from_proto(sort_expr, ctx)?;let (on_columns, sort_options): (Vec<PhysicalExprRef>, Vec<SortOptions>) =sort_exprs.into_iter().map(|sort_expr| (sort_expr.expr, sort_expr.options)).unzip();- let split_points = range_expr- .split_point+ let split_points = split_point.iter().map(|split_point| {- let values = split_point- .value+ let protobuf::PhysicalRangeSplitPoint { value } = split_point;+ let values = value.iter().map(|value| ScalarValue::try_from(value).map_err(Into::into)).collect::<Result<Vec<_>>>()?;@@ -2067,9 +2073,6 @@ impl ExecutionPlan for RepartitionExec {&self,ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,) -> Result<Option<protobuf::PhysicalPlanNode>> {- // Destructure exhaustively (no `..`) so that adding a field to- // `RepartitionExec` is a compile error here until it is either- // serialized or explicitly documented as not needing to be.let Self {input,// Execution-time channel state, created on `execute()`.@@ -2115,9 +2118,6 @@ impl RepartitionExec {protobuf::physical_plan_node::PhysicalPlanType::Repartition,"RepartitionExec",);- // Destructure exhaustively so that a new field on- // `RepartitionExecNode` is a compile error here rather than a silently- // dropped field.let protobuf::RepartitionExecNode {input,partitioning,@@ -5084,3 +5084,80 @@ mod test {Ok(())}}++#[cfg(all(test, feature = "proto"))]+mod range_expr_proto_tests {+ use std::sync::Arc;++ use arrow::compute::SortOptions;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::{Result, ScalarValue, SplitPoint};+ use datafusion_physical_expr::expressions::col;+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;++ use super::*;+ use crate::proto::{ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx};+ use crate::proto_test_util::{StubPlanDecoder, StubPlanEncoder};++ fn schema() -> Schema {+ Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ])+ }++ fn split_points() -> Vec<SplitPoint> {+ vec![SplitPoint::new(vec![+ ScalarValue::Int32(Some(10)),+ ScalarValue::Int32(Some(20)),+ ])]+ }++ fn sort_options() -> [SortOptions; 2] {+ [+ SortOptions {+ descending: true,+ nulls_first: false,+ },+ SortOptions {+ descending: false,+ nulls_first: true,+ },+ ]+ }++ #[test]+ fn range_expr_round_trip_preserves_sort_options_and_split_points() -> Result<()> {+ let schema = schema();+ let sort_opts = sort_options();+ let split_pts = split_points();+ let on_columns = vec![col("a", &schema)?, col("b", &schema)?];+ let range_partitioning = RangePartitioning::try_new(+ [+ PhysicalSortExpr::new(Arc::clone(&on_columns[0]), sort_opts[0]),+ PhysicalSortExpr::new(Arc::clone(&on_columns[1]), sort_opts[1]),+ ]+ .into(),+ split_pts.clone(),+ )?;+ let expr =+ RangeExpr::try_new_with_schema(on_columns, &range_partitioning, &schema)?;++ let encoder = StubPlanEncoder::ok();+ let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);+ let node = PhysicalExpr::try_to_proto(&expr, &encode_ctx.expr_ctx())+ .unwrap()+ .expect("RangeExpr should encode to Some(node)");++ let decoder = StubPlanDecoder::ok();+ let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);+ let decoded = RangeExpr::try_from_proto(&node, &decode_ctx.expr_ctx(&schema))?;+ let decoded = decoded+ .downcast_ref::<RangeExpr>()+ .expect("decoded expression should be a RangeExpr");++ assert_eq!(decoded.sort_options(), sort_opts);+ assert_eq!(decoded.split_points(), split_pts);+ Ok(())+ }+}