Apache DataFusion SQL Query Engine
impl ser/de for preserve_order in RepartitionExec (#20798)
Huaijin4bac1cf8786f
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex 8de3224c91..197be2631a 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -1353,6 +1353,7 @@ message RepartitionExecNode{// uint64 unknown = 4;// }Partitioning partitioning = 5;+ bool preserve_order = 6;}message Partitioning {diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex 2b8089f563..7a4964c4c4 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -20929,6 +20929,9 @@ impl serde::Serialize for RepartitionExecNode {if self.partitioning.is_some() {len += 1;}+ if self.preserve_order {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.RepartitionExecNode", len)?;if let Some(v) = self.input.as_ref() {struct_ser.serialize_field("input", v)?;@@ -20936,6 +20939,9 @@ impl serde::Serialize for RepartitionExecNode {if let Some(v) = self.partitioning.as_ref() {struct_ser.serialize_field("partitioning", v)?;}+ if self.preserve_order {+ struct_ser.serialize_field("preserveOrder", &self.preserve_order)?;+ }struct_ser.end()}}@@ -20948,12 +20954,15 @@ impl<'de> serde::Deserialize<'de> for RepartitionExecNode {const FIELDS: &[&str] = &["input","partitioning",+ "preserve_order",+ "preserveOrder",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Input,Partitioning,+ PreserveOrder,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -20977,6 +20986,7 @@ impl<'de> serde::Deserialize<'de> for RepartitionExecNode {match value {"input" => Ok(GeneratedField::Input),"partitioning" => Ok(GeneratedField::Partitioning),+ "preserveOrder" | "preserve_order" => Ok(GeneratedField::PreserveOrder),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -20998,6 +21008,7 @@ impl<'de> serde::Deserialize<'de> for RepartitionExecNode {{let mut input__ = None;let mut partitioning__ = None;+ let mut preserve_order__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Input => {@@ -21012,11 +21023,18 @@ impl<'de> serde::Deserialize<'de> for RepartitionExecNode {}partitioning__ = map_.next_value()?;}+ GeneratedField::PreserveOrder => {+ if preserve_order__.is_some() {+ return Err(serde::de::Error::duplicate_field("preserveOrder"));+ }+ preserve_order__ = Some(map_.next_value()?);+ }}}Ok(RepartitionExecNode {input: input__,partitioning: partitioning__,+ preserve_order: preserve_order__.unwrap_or_default(),})}}diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex 6daf81bf1a..c708c61328 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -2001,6 +2001,8 @@ pub struct RepartitionExecNode {/// }#[prost(message, optional, tag = "5")]pub partitioning: ::core::option::Option<Partitioning>,+ #[prost(bool, tag = "6")]+ pub preserve_order: bool,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct Partitioning {diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 85406e31da..99cd39d9f2 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -1011,10 +1011,11 @@ impl protobuf::PhysicalPlanNode {codec,proto_converter,)?;- Ok(Arc::new(RepartitionExec::try_new(- input,- partitioning.unwrap(),- )?))+ let mut repart_exec = RepartitionExec::try_new(input, partitioning.unwrap())?;+ if repart.preserve_order {+ repart_exec = repart_exec.with_preserve_order();+ }+ Ok(Arc::new(repart_exec))}fn try_into_global_limit_physical_plan(@@ -3056,6 +3057,7 @@ impl protobuf::PhysicalPlanNode {protobuf::RepartitionExecNode {input: Some(Box::new(input)),partitioning: Some(pb_partitioning),+ preserve_order: exec.preserve_order(),},))),})diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 66ca903e4e..31626ef9fd 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -1781,6 +1781,35 @@ fn roundtrip_union() -> Result<()> {roundtrip_test(union)}+#[test]+fn roundtrip_repartition_preserve_order() -> Result<()> {+ let field_a = Field::new("a", DataType::Int64, false);+ let schema = Arc::new(Schema::new(vec![field_a]));+ let sort_exprs: LexOrdering = [PhysicalSortExpr {+ expr: col("a", &schema)?,+ options: SortOptions::default(),+ }]+ .into();++ // Create two sorted single-partition inputs, then union them to get+ // a sorted input with 2 partitions.+ let source1 = SortExec::new(+ sort_exprs.clone(),+ Arc::new(EmptyExec::new(Arc::clone(&schema))),+ );+ let source2 = SortExec::new(sort_exprs, Arc::new(EmptyExec::new(schema)));+ let union = UnionExec::try_new(vec![+ Arc::new(source1) as Arc<dyn ExecutionPlan>,+ Arc::new(source2) as Arc<dyn ExecutionPlan>,+ ])?;++ let repartition = RepartitionExec::try_new(union, Partitioning::RoundRobinBatch(10))?+ .with_preserve_order();+ assert!(repartition.preserve_order());++ roundtrip_test(Arc::new(repartition))+}+#[test]fn roundtrip_interleave() -> Result<()> {let field_a = Field::new("col", DataType::Int64, false);