Skip to content

Apache DataFusion SQL Query Engine

feat: destructure proto hooks (#25204)

RIchard Baahecb0fea9e372
diff --git a/datafusion/physical-expr-common/src/sort_expr.rs b/datafusion/physical-expr-common/src/sort_expr.rs
index 072fb641e3..43073ec36e 100644
--- a/datafusion/physical-expr-common/src/sort_expr.rs
+++ b/datafusion/physical-expr-common/src/sort_expr.rs
@@ -199,10 +199,11 @@ impl PhysicalSortExpr {
&self,
ctx: &crate::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
) -> Result<datafusion_proto_models::protobuf::PhysicalSortExprNode> {
+ let Self { expr, options } = self;
Ok(datafusion_proto_models::protobuf::PhysicalSortExprNode {
- expr: Some(Box::new(ctx.encode_child(&self.expr)?)),
- asc: !self.options.descending,
- nulls_first: self.options.nulls_first,
+ expr: Some(Box::new(ctx.encode_child(expr)?)),
+ asc: !options.descending,
+ nulls_first: options.nulls_first,
})
}
@@ -211,16 +212,18 @@ impl PhysicalSortExpr {
node: &datafusion_proto_models::protobuf::PhysicalSortExprNode,
ctx: &crate::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
) -> Result<Self> {
- let expr = ctx.decode_required_expression(
- node.expr.as_deref(),
- "PhysicalSortExpr",
- "expr",
- )?;
+ let datafusion_proto_models::protobuf::PhysicalSortExprNode {
+ expr,
+ asc,
+ nulls_first,
+ } = node;
+ let expr =
+ ctx.decode_required_expression(expr.as_deref(), "PhysicalSortExpr", "expr")?;
Ok(PhysicalSortExpr {
expr,
options: SortOptions {
- descending: !node.asc,
- nulls_first: node.nulls_first,
+ descending: !asc,
+ nulls_first: *nulls_first,
},
})
}
diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rs
index e1ca2df922..052fec6ebe 100644
--- a/datafusion/physical-expr/src/partitioning.rs
+++ b/datafusion/physical-expr/src/partitioning.rs
@@ -629,9 +629,12 @@ impl Partitioning {
)
}
Partitioning::Range(range) => {
- let sort_expr = sort_exprs_try_to_proto(range.ordering().iter(), ctx)?;
- let split_point = range
- .split_points()
+ let RangePartitioning {
+ ordering,
+ split_points,
+ } = range;
+ let sort_expr = sort_exprs_try_to_proto(ordering.iter(), ctx)?;
+ let split_point = split_points
.iter()
.map(|split_point| {
let value = split_point
@@ -673,7 +676,8 @@ impl Partitioning {
let partition_count =
|count: u64| usize_from_wire(count, "Partitioning", "partition_count");
- let Some(partition_method) = node.partition_method.as_ref() else {
+ let protobuf::Partitioning { partition_method } = node;
+ let Some(partition_method) = partition_method.as_ref() else {
return Ok(None);
};
let partitioning = match partition_method {
@@ -681,18 +685,25 @@ impl Partitioning {
Partitioning::RoundRobinBatch(partition_count(*n)?)
}
protobuf::partitioning::PartitionMethod::Hash(hash) => {
- let exprs = hash
- .hash_expr
+ let protobuf::PhysicalHashRepartition {
+ hash_expr,
+ partition_count: hash_partition_count,
+ } = hash;
+ let exprs = hash_expr
.iter()
.map(|expr| ctx.decode(expr))
.collect::<Result<Vec<_>>>()?;
- Partitioning::Hash(exprs, partition_count(hash.partition_count)?)
+ Partitioning::Hash(exprs, partition_count(*hash_partition_count)?)
}
protobuf::partitioning::PartitionMethod::Unknown(n) => {
Partitioning::UnknownPartitioning(partition_count(*n)?)
}
protobuf::partitioning::PartitionMethod::Range(range) => {
- let sort_exprs = sort_exprs_try_from_proto(&range.sort_expr, ctx)?;
+ let protobuf::PhysicalRangePartitioning {
+ sort_expr,
+ split_point,
+ } = range;
+ let sort_exprs = sort_exprs_try_from_proto(sort_expr, ctx)?;
let sort_expr_count = sort_exprs.len();
let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {
internal_datafusion_err!(
@@ -704,12 +715,11 @@ impl Partitioning {
"Range partitioning ordering must not contain duplicate expressions"
);
}
- let split_points = range
- .split_point
+ let split_points = split_point
.iter()
.map(|split_point| {
- let values = split_point
- .value
+ let protobuf::PhysicalRangeSplitPoint { value } = split_point;
+ let values = value
.iter()
.map(|value| ScalarValue::try_from(value).map_err(Into::into))
.collect::<Result<Vec<_>>>()?;
@@ -1575,13 +1585,16 @@ mod ordering_proto_tests {
mod partition_count_proto_tests {
use std::sync::Arc;
+ use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::{ScalarValue, SplitPoint};
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx;
+ use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use datafusion_proto_models::protobuf;
- use super::Partitioning;
+ use super::{Partitioning, RangePartitioning};
use crate::expressions::Column;
use crate::proto_test_util::{StubDecoder, StubEncoder, column_node};
@@ -1662,4 +1675,51 @@ mod partition_count_proto_tests {
// Every variant widens to the same wire value, with no truncation.
assert_eq!(encoded, vec![u64::try_from(usize::MAX).unwrap(); 3]);
}
+
+ #[test]
+ fn range_partitioning_round_trip_preserves_split_points_and_options() {
+ // Single-column ordering so the stub encoder (which emits an identical
+ // placeholder for every expression) cannot produce duplicates that the
+ // decoder's deduplication step would collapse.
+ let sort_options = SortOptions {
+ descending: false,
+ nulls_first: true,
+ };
+ let ordering = LexOrdering::from([PhysicalSortExpr::new(
+ Arc::new(Column::new("a", 0)),
+ sort_options,
+ )]);
+ let split_points = vec![
+ SplitPoint::new(vec![ScalarValue::Int32(Some(10))]),
+ SplitPoint::new(vec![ScalarValue::Int32(Some(30))]),
+ ];
+ let range =
+ RangePartitioning::try_new(ordering.clone(), split_points.clone()).unwrap();
+
+ let encoder = StubEncoder::ok();
+ let encode_ctx = PhysicalExprEncodeCtx::new(&encoder);
+ let node = Partitioning::Range(range)
+ .try_to_proto(&encode_ctx)
+ .unwrap();
+
+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let decoder = StubDecoder::ok();
+ let decode_ctx = PhysicalExprDecodeCtx::new(&schema, &decoder);
+ let decoded = Partitioning::try_from_proto(&node, &decode_ctx)
+ .unwrap()
+ .unwrap();
+
+ let Partitioning::Range(decoded_range) = decoded else {
+ panic!("expected Range partitioning, got {decoded:?}");
+ };
+ assert_eq!(
+ decoded_range
+ .ordering()
+ .iter()
+ .map(|e| e.options)
+ .collect::<Vec<_>>(),
+ [sort_options],
+ );
+ assert_eq!(decoded_range.split_points(), split_points);
+ }
}
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index b653a74fa5..794888b4ea 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -849,17 +849,16 @@ impl PhysicalExpr for RangeExpr {
&self,
ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
) -> Result<Option<protobuf::PhysicalExprNode>> {
+ let Self { on_columns, router } = self;
// Encode the raw ordered children: rebuilding a `LexOrdering` would
// deduplicate equivalent children after dynamic-filter remapping.
- let sort_exprs = self
- .on_columns
+ let sort_exprs = on_columns
.iter()
- .zip(self.router.sort_options())
+ .zip(router.sort_options())
.map(|(expr, options)| PhysicalSortExpr::new(Arc::clone(expr), *options))
.collect::<Vec<_>>();
let sort_expr = sort_exprs_try_to_proto(&sort_exprs, ctx)?;
- let split_point = self
- .router
+ let split_point = router
.split_points()
.iter()
.map(|split_point| {
@@ -890,24 +889,31 @@ impl RangeExpr {
node: &protobuf::PhysicalExprNode,
ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
) -> Result<PhysicalExprRef> {
+ let protobuf::PhysicalExprNode {
+ expr_id: _,
+ expr_type,
+ } = node;
// Decode the raw ordered children for the same reason as `try_to_proto`.
let Some(protobuf::physical_expr_node::ExprType::RangeExpr(range_expr)) =
- &node.expr_type
+ expr_type
else {
return internal_err!("PhysicalExprNode is not a RangeExpr");
};
- let sort_exprs = sort_exprs_try_from_proto(&range_expr.sort_expr, ctx)?;
+ let protobuf::PhysicalRangeExprNode {
+ sort_expr,
+ split_point,
+ } = range_expr;
+ let sort_exprs = sort_exprs_try_from_proto(sort_expr, ctx)?;
let (on_columns, sort_options): (Vec<PhysicalExprRef>, Vec<SortOptions>) =
sort_exprs
.into_iter()
.map(|sort_expr| (sort_expr.expr, sort_expr.options))
.unzip();
- let split_points = range_expr
- .split_point
+ let split_points = split_point
.iter()
.map(|split_point| {
- let values = split_point
- .value
+ let protobuf::PhysicalRangeSplitPoint { value } = split_point;
+ let values = value
.iter()
.map(|value| ScalarValue::try_from(value).map_err(Into::into))
.collect::<Result<Vec<_>>>()?;
@@ -2067,9 +2073,6 @@ impl ExecutionPlan for RepartitionExec {
&self,
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
) -> Result<Option<protobuf::PhysicalPlanNode>> {
- // Destructure exhaustively (no `..`) so that adding a field to
- // `RepartitionExec` is a compile error here until it is either
- // serialized or explicitly documented as not needing to be.
let Self {
input,
// Execution-time channel state, created on `execute()`.
@@ -2115,9 +2118,6 @@ impl RepartitionExec {
protobuf::physical_plan_node::PhysicalPlanType::Repartition,
"RepartitionExec",
);
- // Destructure exhaustively so that a new field on
- // `RepartitionExecNode` is a compile error here rather than a silently
- // dropped field.
let protobuf::RepartitionExecNode {
input,
partitioning,
@@ -5084,3 +5084,80 @@ mod test {
Ok(())
}
}
+
+#[cfg(all(test, feature = "proto"))]
+mod range_expr_proto_tests {
+ use std::sync::Arc;
+
+ use arrow::compute::SortOptions;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::{Result, ScalarValue, SplitPoint};
+ use datafusion_physical_expr::expressions::col;
+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
+
+ use super::*;
+ use crate::proto::{ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx};
+ use crate::proto_test_util::{StubPlanDecoder, StubPlanEncoder};
+
+ fn schema() -> Schema {
+ Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ ])
+ }
+
+ fn split_points() -> Vec<SplitPoint> {
+ vec![SplitPoint::new(vec![
+ ScalarValue::Int32(Some(10)),
+ ScalarValue::Int32(Some(20)),
+ ])]
+ }
+
+ fn sort_options() -> [SortOptions; 2] {
+ [
+ SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ ]
+ }
+
+ #[test]
+ fn range_expr_round_trip_preserves_sort_options_and_split_points() -> Result<()> {
+ let schema = schema();
+ let sort_opts = sort_options();
+ let split_pts = split_points();
+ let on_columns = vec![col("a", &schema)?, col("b", &schema)?];
+ let range_partitioning = RangePartitioning::try_new(
+ [
+ PhysicalSortExpr::new(Arc::clone(&on_columns[0]), sort_opts[0]),
+ PhysicalSortExpr::new(Arc::clone(&on_columns[1]), sort_opts[1]),
+ ]
+ .into(),
+ split_pts.clone(),
+ )?;
+ let expr =
+ RangeExpr::try_new_with_schema(on_columns, &range_partitioning, &schema)?;
+
+ let encoder = StubPlanEncoder::ok();
+ let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
+ let node = PhysicalExpr::try_to_proto(&expr, &encode_ctx.expr_ctx())
+ .unwrap()
+ .expect("RangeExpr should encode to Some(node)");
+
+ let decoder = StubPlanDecoder::ok();
+ let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
+ let decoded = RangeExpr::try_from_proto(&node, &decode_ctx.expr_ctx(&schema))?;
+ let decoded = decoded
+ .downcast_ref::<RangeExpr>()
+ .expect("decoded expression should be a RangeExpr");
+
+ assert_eq!(decoded.sort_options(), sort_opts);
+ assert_eq!(decoded.split_points(), split_pts);
+ Ok(())
+ }
+}