Skip to content

Apache DataFusion SQL Query Engine

feat: logical plan protobuf representation for range repartitioning (#23030)

Saad Tajwara1f56b7ec549
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index 2f5b75e409..8745100e55 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -148,9 +148,19 @@ message RepartitionNode {
oneof partition_method {
uint64 round_robin = 2;
HashRepartition hash = 3;
+ RangeRepartition range = 4;
}
}
+message RangeSplitPoint {
+ repeated datafusion_common.ScalarValue value = 1;
+}
+
+message RangeRepartition {
+ repeated SortExprNode sort_expr = 1;
+ repeated RangeSplitPoint split_point = 2;
+}
+
message HashRepartition {
repeated LogicalExprNode hash_expr = 1;
uint64 partition_count = 2;
diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs
index 733da68fe8..a85f807bc8 100644
--- a/datafusion/proto-models/src/generated/pbjson.rs
+++ b/datafusion/proto-models/src/generated/pbjson.rs
@@ -22350,6 +22350,207 @@ impl<'de> serde::Deserialize<'de> for ProjectionNode {
deserializer.deserialize_struct("datafusion.ProjectionNode", FIELDS, GeneratedVisitor)
}
}
+impl serde::Serialize for RangeRepartition {
+ #[allow(deprecated)]
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ use serde::ser::SerializeStruct;
+ let mut len = 0;
+ if !self.sort_expr.is_empty() {
+ len += 1;
+ }
+ if !self.split_point.is_empty() {
+ len += 1;
+ }
+ let mut struct_ser = serializer.serialize_struct("datafusion.RangeRepartition", len)?;
+ if !self.sort_expr.is_empty() {
+ struct_ser.serialize_field("sortExpr", &self.sort_expr)?;
+ }
+ if !self.split_point.is_empty() {
+ struct_ser.serialize_field("splitPoint", &self.split_point)?;
+ }
+ struct_ser.end()
+ }
+}
+impl<'de> serde::Deserialize<'de> for RangeRepartition {
+ #[allow(deprecated)]
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ const FIELDS: &[&str] = &[
+ "sort_expr",
+ "sortExpr",
+ "split_point",
+ "splitPoint",
+ ];
+
+ #[allow(clippy::enum_variant_names)]
+ enum GeneratedField {
+ SortExpr,
+ SplitPoint,
+ }
+ impl<'de> serde::Deserialize<'de> for GeneratedField {
+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ struct GeneratedVisitor;
+
+ impl serde::de::Visitor<'_> for GeneratedVisitor {
+ type Value = GeneratedField;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(formatter, "expected one of: {:?}", &FIELDS)
+ }
+
+ #[allow(unused_variables)]
+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
+ where
+ E: serde::de::Error,
+ {
+ match value {
+ "sortExpr" | "sort_expr" => Ok(GeneratedField::SortExpr),
+ "splitPoint" | "split_point" => Ok(GeneratedField::SplitPoint),
+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),
+ }
+ }
+ }
+ deserializer.deserialize_identifier(GeneratedVisitor)
+ }
+ }
+ struct GeneratedVisitor;
+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
+ type Value = RangeRepartition;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str("struct datafusion.RangeRepartition")
+ }
+
+ fn visit_map<V>(self, mut map_: V) -> std::result::Result<RangeRepartition, V::Error>
+ where
+ V: serde::de::MapAccess<'de>,
+ {
+ let mut sort_expr__ = None;
+ let mut split_point__ = None;
+ while let Some(k) = map_.next_key()? {
+ match k {
+ GeneratedField::SortExpr => {
+ if sort_expr__.is_some() {
+ return Err(serde::de::Error::duplicate_field("sortExpr"));
+ }
+ sort_expr__ = Some(map_.next_value()?);
+ }
+ GeneratedField::SplitPoint => {
+ if split_point__.is_some() {
+ return Err(serde::de::Error::duplicate_field("splitPoint"));
+ }
+ split_point__ = Some(map_.next_value()?);
+ }
+ }
+ }
+ Ok(RangeRepartition {
+ sort_expr: sort_expr__.unwrap_or_default(),
+ split_point: split_point__.unwrap_or_default(),
+ })
+ }
+ }
+ deserializer.deserialize_struct("datafusion.RangeRepartition", FIELDS, GeneratedVisitor)
+ }
+}
+impl serde::Serialize for RangeSplitPoint {
+ #[allow(deprecated)]
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ use serde::ser::SerializeStruct;
+ let mut len = 0;
+ if !self.value.is_empty() {
+ len += 1;
+ }
+ let mut struct_ser = serializer.serialize_struct("datafusion.RangeSplitPoint", len)?;
+ if !self.value.is_empty() {
+ struct_ser.serialize_field("value", &self.value)?;
+ }
+ struct_ser.end()
+ }
+}
+impl<'de> serde::Deserialize<'de> for RangeSplitPoint {
+ #[allow(deprecated)]
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ const FIELDS: &[&str] = &[
+ "value",
+ ];
+
+ #[allow(clippy::enum_variant_names)]
+ enum GeneratedField {
+ Value,
+ }
+ impl<'de> serde::Deserialize<'de> for GeneratedField {
+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ struct GeneratedVisitor;
+
+ impl serde::de::Visitor<'_> for GeneratedVisitor {
+ type Value = GeneratedField;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(formatter, "expected one of: {:?}", &FIELDS)
+ }
+
+ #[allow(unused_variables)]
+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
+ where
+ E: serde::de::Error,
+ {
+ match value {
+ "value" => Ok(GeneratedField::Value),
+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),
+ }
+ }
+ }
+ deserializer.deserialize_identifier(GeneratedVisitor)
+ }
+ }
+ struct GeneratedVisitor;
+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
+ type Value = RangeSplitPoint;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str("struct datafusion.RangeSplitPoint")
+ }
+
+ fn visit_map<V>(self, mut map_: V) -> std::result::Result<RangeSplitPoint, V::Error>
+ where
+ V: serde::de::MapAccess<'de>,
+ {
+ let mut value__ = None;
+ while let Some(k) = map_.next_key()? {
+ match k {
+ GeneratedField::Value => {
+ if value__.is_some() {
+ return Err(serde::de::Error::duplicate_field("value"));
+ }
+ value__ = Some(map_.next_value()?);
+ }
+ }
+ }
+ Ok(RangeSplitPoint {
+ value: value__.unwrap_or_default(),
+ })
+ }
+ }
+ deserializer.deserialize_struct("datafusion.RangeSplitPoint", FIELDS, GeneratedVisitor)
+ }
+}
impl serde::Serialize for RecursionUnnestOption {
#[allow(deprecated)]
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
@@ -22778,6 +22979,9 @@ impl serde::Serialize for RepartitionNode {
repartition_node::PartitionMethod::Hash(v) => {
struct_ser.serialize_field("hash", v)?;
}
+ repartition_node::PartitionMethod::Range(v) => {
+ struct_ser.serialize_field("range", v)?;
+ }
}
}
struct_ser.end()
@@ -22794,6 +22998,7 @@ impl<'de> serde::Deserialize<'de> for RepartitionNode {
"round_robin",
"roundRobin",
"hash",
+ "range",
];
#[allow(clippy::enum_variant_names)]
@@ -22801,6 +23006,7 @@ impl<'de> serde::Deserialize<'de> for RepartitionNode {
Input,
RoundRobin,
Hash,
+ Range,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -22825,6 +23031,7 @@ impl<'de> serde::Deserialize<'de> for RepartitionNode {
"input" => Ok(GeneratedField::Input),
"roundRobin" | "round_robin" => Ok(GeneratedField::RoundRobin),
"hash" => Ok(GeneratedField::Hash),
+ "range" => Ok(GeneratedField::Range),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -22865,6 +23072,13 @@ impl<'de> serde::Deserialize<'de> for RepartitionNode {
return Err(serde::de::Error::duplicate_field("hash"));
}
partition_method__ = map_.next_value::<::std::option::Option<_>>()?.map(repartition_node::PartitionMethod::Hash)
+;
+ }
+ GeneratedField::Range => {
+ if partition_method__.is_some() {
+ return Err(serde::de::Error::duplicate_field("range"));
+ }
+ partition_method__ = map_.next_value::<::std::option::Option<_>>()?.map(repartition_node::PartitionMethod::Range)
;
}
}
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index 4a2edeeb11..4da38881a8 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -210,7 +210,7 @@ pub struct SortNode {
pub struct RepartitionNode {
#[prost(message, optional, boxed, tag = "1")]
pub input: ::core::option::Option<::prost::alloc::boxed::Box<LogicalPlanNode>>,
- #[prost(oneof = "repartition_node::PartitionMethod", tags = "2, 3")]
+ #[prost(oneof = "repartition_node::PartitionMethod", tags = "2, 3, 4")]
pub partition_method: ::core::option::Option<repartition_node::PartitionMethod>,
}
/// Nested message and enum types in `RepartitionNode`.
@@ -221,9 +221,23 @@ pub mod repartition_node {
RoundRobin(u64),
#[prost(message, tag = "3")]
Hash(super::HashRepartition),
+ #[prost(message, tag = "4")]
+ Range(super::RangeRepartition),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct RangeSplitPoint {
+ #[prost(message, repeated, tag = "1")]
+ pub value: ::prost::alloc::vec::Vec<super::datafusion_common::ScalarValue>,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct RangeRepartition {
+ #[prost(message, repeated, tag = "1")]
+ pub sort_expr: ::prost::alloc::vec::Vec<SortExprNode>,
+ #[prost(message, repeated, tag = "2")]
+ pub split_point: ::prost::alloc::vec::Vec<RangeSplitPoint>,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
pub struct HashRepartition {
#[prost(message, repeated, tag = "1")]
pub hash_expr: ::prost::alloc::vec::Vec<LogicalExprNode>,
diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs
index b79b21b359..372213e38f 100644
--- a/datafusion/proto/src/logical_plan/from_proto.rs
+++ b/datafusion/proto/src/logical_plan/from_proto.rs
@@ -20,7 +20,7 @@ use std::sync::Arc;
use arrow::datatypes::{DataType, Field};
use datafusion_common::datatype::DataTypeExt;
use datafusion_common::{
- NullEquality, RecursionUnnestOption, Result, ScalarValue, TableReference,
+ NullEquality, RecursionUnnestOption, Result, ScalarValue, SplitPoint, TableReference,
UnnestOptions, exec_datafusion_err, internal_err, plan_datafusion_err,
};
use datafusion_execution::TaskContext;
@@ -899,3 +899,14 @@ fn parse_required_expr(
fn proto_error<S: Into<String>>(message: S) -> Error {
Error::General(message.into())
}
+
+pub(super) fn parse_protobuf_range_split_point(
+ split_point: &protobuf::RangeSplitPoint,
+) -> Result<SplitPoint, Error> {
+ let values = split_point
+ .value
+ .iter()
+ .map(ScalarValue::try_from)
+ .collect::<Result<_, Error>>()?;
+ Ok(SplitPoint::new(values))
+}
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index a0604cb6b0..3195b050b3 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -60,8 +60,8 @@ use datafusion_datasource_json::file_format::{
use datafusion_datasource_parquet::file_format::{ParquetFormat, ParquetFormatFactory};
use datafusion_expr::dml::InsertOp;
use datafusion_expr::{
- AggregateUDF, DmlStatement, FetchType, HigherOrderUDF, RecursiveQuery, SkipType,
- TableSource, Unnest, WriteOp,
+ AggregateUDF, DmlStatement, FetchType, HigherOrderUDF, RangePartitioning,
+ RecursiveQuery, SkipType, TableSource, Unnest, WriteOp,
};
use datafusion_expr::{
DistinctOn, DropView, Expr, JoinConstraint, LogicalPlan, LogicalPlanBuilder,
@@ -76,6 +76,7 @@ use datafusion_expr::{
use datafusion_proto_common::protobuf_common;
use self::to_proto::{serialize_expr, serialize_exprs};
+use crate::logical_plan::to_proto::serialize_range_split_point;
use crate::logical_plan::to_proto::serialize_sorts;
use datafusion_catalog::TableProvider;
use datafusion_catalog::default_table_source::{provider_as_source, source_as_provider};
@@ -745,6 +746,16 @@ impl AsLogicalPlan for LogicalPlanNode {
PartitionMethod::RoundRobin(partition_count) => {
Partitioning::RoundRobinBatch(*partition_count as usize)
}
+ PartitionMethod::Range(protobuf::RangeRepartition {
+ sort_expr: pb_sort_expr,
+ split_point,
+ }) => Partitioning::Range(RangePartitioning::try_new(
+ from_proto::parse_sorts(pb_sort_expr, ctx, extension_codec)?,
+ split_point
+ .iter()
+ .map(from_proto::parse_protobuf_range_split_point)
+ .collect::<Result<Vec<_>, _>>()?,
+ )?),
};
LogicalPlanBuilder::from(input)
@@ -1754,10 +1765,18 @@ impl AsLogicalPlan for LogicalPlanNode {
Partitioning::RoundRobinBatch(partition_count) => {
PartitionMethod::RoundRobin(*partition_count as u64)
}
- Partitioning::Range(_) => {
- // TODO: Support range repartition protobuf serialization.
- // Tracked by https://github.com/apache/datafusion/issues/22787
- return not_impl_err!("Range repartition");
+ Partitioning::Range(range_partitioning) => {
+ let ordering = range_partitioning.ordering();
+ let split_point = range_partitioning
+ .split_points()
+ .iter()
+ .map(serialize_range_split_point)
+ .collect::<Result<Vec<_>, _>>()?;
+
+ PartitionMethod::Range(protobuf::RangeRepartition {
+ sort_expr: serialize_sorts(ordering, extension_codec)?,
+ split_point,
+ })
}
Partitioning::DistributeBy(_) => {
return not_impl_err!("DistributeBy");
diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs
index 516aca4094..b2b035af88 100644
--- a/datafusion/proto/src/logical_plan/to_proto.rs
+++ b/datafusion/proto/src/logical_plan/to_proto.rs
@@ -21,7 +21,7 @@
use std::collections::HashMap;
-use datafusion_common::{NullEquality, TableReference, UnnestOptions};
+use datafusion_common::{NullEquality, SplitPoint, TableReference, UnnestOptions};
use datafusion_expr::dml::{
MergeIntoAction, MergeIntoClause, MergeIntoClauseKind, MergeIntoOp,
};
@@ -687,6 +687,18 @@ where
.collect::<Result<Vec<_>, Error>>()
}
+pub(super) fn serialize_range_split_point(
+ split_point: &SplitPoint,
+) -> Result<protobuf::RangeSplitPoint, Error> {
+ Ok(protobuf::RangeSplitPoint {
+ value: split_point
+ .values()
+ .iter()
+ .map(TryInto::<datafusion_proto_common::ScalarValue>::try_into)
+ .collect::<Result<_, Error>>()?,
+ })
+}
+
impl FromProto<TableReference> for protobuf::TableReference {
fn from_proto(t: TableReference) -> Self {
use protobuf::table_reference::TableReferenceEnum;
diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
index 9d8e5c2b1e..82ad94d8f7 100644
--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
@@ -45,6 +45,7 @@ use std::vec;
use datafusion::catalog::{TableProvider, TableProviderFactory};
use datafusion::datasource::DefaultTableSource;
+use datafusion::datasource::empty::EmptyTable;
use datafusion::datasource::file_format::arrow::ArrowFormatFactory;
use datafusion::datasource::file_format::csv::CsvFormatFactory;
use datafusion::datasource::file_format::parquet::ParquetFormatFactory;
@@ -73,8 +74,8 @@ use datafusion_common::format::{
};
use datafusion_common::scalar::ScalarStructBuilder;
use datafusion_common::{
- DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, TableReference,
- internal_datafusion_err, internal_err, not_impl_err, plan_err,
+ DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, SplitPoint,
+ TableReference, internal_datafusion_err, internal_err, not_impl_err, plan_err,
};
use datafusion_execution::TaskContext;
use datafusion_expr::dml::CopyTo;
@@ -91,9 +92,9 @@ use datafusion_expr::logical_plan::{
use datafusion_expr::{
Accumulator, AggregateUDF, ColumnarValue, DmlStatement, ExprFunctionExt,
ExprSchemable, LimitEffect, Literal, LogicalPlan, LogicalPlanBuilder, Operator,
- PartitionEvaluator, ScalarUDF, Signature, TryCast, Volatility, WindowFrame,
- WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, WindowUDF,
- WindowUDFImpl, WriteOp,
+ PartitionEvaluator, RangePartitioning, Repartition, ScalarUDF, Signature, TryCast,
+ Volatility, WindowFrame, WindowFrameBound, WindowFrameUnits,
+ WindowFunctionDefinition, WindowUDF, WindowUDFImpl, WriteOp,
};
use datafusion_functions_aggregate::average::avg_udaf;
use datafusion_functions_aggregate::expr_fn::{
@@ -3379,9 +3380,7 @@ async fn roundtrip_empty_table_scan() -> Result<()> {
Field::new("id", DataType::Int32, false),
Field::new("name", DataType::Utf8, true),
]));
- let table = Arc::new(datafusion::datasource::empty::EmptyTable::new(Arc::clone(
- &schema,
- )));
+ let table = Arc::new(EmptyTable::new(Arc::clone(&schema)));
let ctx = SessionContext::new();
ctx.register_table("empty", table)?;
@@ -3403,9 +3402,7 @@ async fn roundtrip_empty_table_scan_with_projection() -> Result<()> {
Field::new("id", DataType::Int32, false),
Field::new("name", DataType::Utf8, true),
]));
- let table = Arc::new(datafusion::datasource::empty::EmptyTable::new(Arc::clone(
- &schema,
- )));
+ let table = Arc::new(EmptyTable::new(Arc::clone(&schema)));
let ctx = SessionContext::new();
ctx.register_table("empty", table)?;
@@ -3481,14 +3478,8 @@ async fn roundtrip_join_null_equality() -> Result<()> {
let left_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
let right_schema =
Arc::new(Schema::new(vec![Field::new("b", DataType::Int32, true)]));
- ctx.register_table(
- "t1",
- Arc::new(datafusion::datasource::empty::EmptyTable::new(left_schema)),
- )?;
- ctx.register_table(
- "t2",
- Arc::new(datafusion::datasource::empty::EmptyTable::new(right_schema)),
- )?;
+ ctx.register_table("t1", Arc::new(EmptyTable::new(left_schema)))?;
+ ctx.register_table("t2", Arc::new(EmptyTable::new(right_schema)))?;
let left = ctx.table("t1").await?.into_optimized_plan()?;
let right = ctx.table("t2").await?.into_optimized_plan()?;
@@ -3509,3 +3500,91 @@ async fn roundtrip_join_null_equality() -> Result<()> {
Ok(())
}
+
+// Single column, single split point range partitioning
+#[tokio::test]
+async fn roundtrip_range_partitioning_single_col() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int32, false),
+ Field::new("name", DataType::Utf8, true),
+ ]));
+ let table = Arc::new(EmptyTable::new(Arc::clone(&schema)));
+
+ let ctx = SessionContext::new();
+ ctx.register_table("empty", table)?;
+
+ let scan_plan = ctx.table("empty").await?.into_optimized_plan()?;
+ let plan = LogicalPlan::Repartition(Repartition {
+ input: Arc::new(scan_plan),
+ partitioning_scheme: Partitioning::Range(RangePartitioning::try_new(
+ vec![col("id").sort(true, true)],
+ vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],
+ )?),
+ });
+ let bytes = logical_plan_to_bytes(&plan)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
+ assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
+ Ok(())
+}
+
+// Multi-column compound key with multiple split points for range partitioning
+#[tokio::test]
+async fn roundtrip_range_partitioning_multi_col() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("ts", DataType::Int64, false),
+ Field::new("region", DataType::Utf8, false),
+ ]));
+ let table = Arc::new(EmptyTable::new(Arc::clone(&schema)));
+
+ let ctx = SessionContext::new();
+ ctx.register_table("empty", table)?;
+
+ let scan_plan = ctx.table("empty").await?.into_optimized_plan()?;
+ let plan = LogicalPlan::Repartition(Repartition {
+ input: Arc::new(scan_plan),
+ partitioning_scheme: Partitioning::Range(RangePartitioning::try_new(
+ vec![col("ts").sort(true, true), col("region").sort(true, true)],
+ vec![
+ SplitPoint::new(vec![
+ ScalarValue::Int64(Some(1000)),
+ ScalarValue::Utf8(Some("east".to_string())),
+ ]),
+ SplitPoint::new(vec![
+ ScalarValue::Int64(Some(2000)),
+ ScalarValue::Utf8(Some("west".to_string())),
+ ]),
+ ],
+ )?),
+ });
+ let bytes = logical_plan_to_bytes(&plan)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
+ assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
+ Ok(())
+}
+
+// Non-default sort options: descending with nulls last for range partitioning
+#[tokio::test]
+async fn roundtrip_range_partitioning_desc_nulls_last() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "score",
+ DataType::Float64,
+ true,
+ )]));
+ let table = Arc::new(EmptyTable::new(Arc::clone(&schema)));
+
+ let ctx = SessionContext::new();
+ ctx.register_table("empty", table)?;
+
+ let scan_plan = ctx.table("empty").await?.into_optimized_plan()?;
+ let plan = LogicalPlan::Repartition(Repartition {
+ input: Arc::new(scan_plan),
+ partitioning_scheme: Partitioning::Range(RangePartitioning::try_new(
+ vec![col("score").sort(false, false)],
+ vec![SplitPoint::new(vec![ScalarValue::Float64(Some(50.0))])],
+ )?),
+ });
+ let bytes = logical_plan_to_bytes(&plan)?;
+ let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
+ assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
+ Ok(())
+}