Apache DataFusion SQL Query Engine
Add Physical `Partitioning::Range` enum variant (#22207)
Gene Bordegaray7a6b0626da7a
diff --git a/datafusion/ffi/src/physical_expr/partitioning.rs b/datafusion/ffi/src/physical_expr/partitioning.rsindex 434b6a097e..eec437639e 100644--- a/datafusion/ffi/src/physical_expr/partitioning.rs+++ b/datafusion/ffi/src/physical_expr/partitioning.rs@@ -45,6 +45,11 @@ impl From<&Partitioning> for FFI_Partitioning {.collect();Self::Hash(exprs, *size)}+ // FFI does not yet expose range partition metadata.+ // See https://github.com/apache/datafusion/issues/22394+ Partitioning::Range(range) => {+ Self::UnknownPartitioning(range.partition_count())+ }Partitioning::UnknownPartitioning(size) => Self::UnknownPartitioning(*size),}}diff --git a/datafusion/physical-expr/src/lib.rs b/datafusion/physical-expr/src/lib.rsindex e67046987b..c82d1c64dd 100644--- a/datafusion/physical-expr/src/lib.rs+++ b/datafusion/physical-expr/src/lib.rs@@ -62,7 +62,7 @@ pub use equivalence::{AcrossPartitions, ConstExpr, EquivalenceProperties, calculate_union,};pub use expressions::{DynamicFilterTracker, DynamicFilterTracking};-pub use partitioning::{Distribution, Partitioning};+pub use partitioning::{Distribution, Partitioning, RangePartitioning, SplitPoint};pub use physical_expr::{add_offset_to_expr, add_offset_to_physical_sort_exprs, create_lex_ordering,create_ordering, create_physical_sort_expr, create_physical_sort_exprs,diff --git a/datafusion/physical-expr/src/partitioning.rs b/datafusion/physical-expr/src/partitioning.rsindex d24c60b63e..bb46b8a957 100644--- a/datafusion/physical-expr/src/partitioning.rs+++ b/datafusion/physical-expr/src/partitioning.rs@@ -21,7 +21,10 @@ use crate::{EquivalenceProperties, PhysicalExpr, equivalence::ProjectionMapping,expressions::UnKnownColumn, physical_exprs_equal,};+use datafusion_common::{Result, ScalarValue, plan_err};use datafusion_physical_expr_common::physical_expr::format_physical_expr_list;+use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};+use std::cmp::Ordering;use std::fmt;use std::fmt::Display;use std::sync::Arc;@@ -117,6 +120,8 @@ pub enum Partitioning {/// Allocate rows based on a hash of one of more expressions and the specified number of/// partitionsHash(Vec<Arc<dyn PhysicalExpr>>, usize),+ /// Partition rows by source-declared ranges+ Range(RangePartitioning),/// Unknown partitioning scheme with a known number of partitionsUnknownPartitioning(usize),}@@ -133,6 +138,7 @@ impl Display for Partitioning {.join(", ");write!(f, "Hash([{phy_exprs_str}], {size})")}+ Partitioning::Range(range) => write!(f, "{range}"),Partitioning::UnknownPartitioning(size) => {write!(f, "UnknownPartitioning({size})")}@@ -140,6 +146,271 @@ impl Display for Partitioning {}}+/// Physical range partitioning.+///+/// [`RangePartitioning`] describes an ordered key space with split points.+///+/// - `ordering` defines the partitioning key and ordering.+/// - `split_points` define the boundaries between adjacent partitions.+///+/// Comparisons use the lexicographic order defined by `ordering`, including+/// `ASC`/`DESC` and null ordering. Split points must be strictly ordered+/// according to that ordering, and each split point must have one value per+/// ordering expression.+///+/// `N` split points define `N + 1` partitions:+///+/// ```text+/// partition 0: key < split_points[0]+/// partition 1: split_points[0] <= key < split_points[1]+/// ...+/// partition N - 1: split_points[N - 2] <= key < split_points[N - 1]+/// partition N: split_points[N - 1] <= key+/// ```+///+/// Values equal to split point `i` belong to partition `i + 1`, so interior+/// partitions are lower-inclusive and upper-exclusive.+///+/// For a single range key:+///+/// ```text+/// ordering = [date ASC NULLS LAST]+/// split_points = [+/// (2022-01-01),+/// (2023-01-01),+/// ]+///+/// partition 0: date before 2022-01-01+/// partition 1: date between 2022-01-01 (inclusive) and 2023-01-01 (exclusive)+/// partition 2: date at/after 2023-01-01+/// ```+///+/// The same model extends to compound keys.+/// For `ordering = [time ASC, city ASC]`, split points are ordered+/// lexicographically by `(time, city)`:+///+/// ```text+/// ordering = [time ASC NULLS LAST, city ASC NULLS LAST]+/// split_points = [+/// (2022, Allston),+/// (2023, Allston),+/// ]+///+/// partition 0: keys before (2022, Allston)+/// partition 1: keys between (2022, Allston) and (2023, Allston)+/// partition 2: keys at/after (2023, Allston)+/// ```+///+/// NOTE: Optimizer and execution behavior for this partitioning is intentionally+/// not implemented and will be introduced incrementally. See+/// <https://github.com/apache/datafusion/issues/22395>.+#[derive(Debug, Clone, PartialEq)]+pub struct RangePartitioning {+ /// Ordered partitioning key.+ ordering: LexOrdering,+ /// Boundaries between adjacent partitions.+ split_points: Vec<SplitPoint>,+}++/// A boundary between adjacent range partitions.+///+/// A split point is a tuple with one [`ScalarValue`] per sort expression in the+/// parent [`RangePartitioning`] ordering.+#[derive(Debug, Clone, PartialEq)]+pub struct SplitPoint {+ values: Vec<ScalarValue>,+}++impl SplitPoint {+ /// Creates a new split point from its tuple values.+ pub fn new(values: Vec<ScalarValue>) -> Self {+ Self { values }+ }++ /// Returns the tuple values for this split point.+ pub fn values(&self) -> &[ScalarValue] {+ &self.values+ }+}++impl Display for SplitPoint {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ let values = self+ .values+ .iter()+ .map(ToString::to_string)+ .collect::<Vec<_>>()+ .join(", ");+ write!(f, "({values})")+ }+}++impl RangePartitioning {+ /// Creates range partitioning metadata without validating split points.+ ///+ /// Use [`Self::try_new`] to validate the contract documented on+ /// [`RangePartitioning`].+ pub fn new(ordering: LexOrdering, split_points: Vec<SplitPoint>) -> Self {+ Self {+ ordering,+ split_points,+ }+ }++ /// Creates range partitioning metadata and validates split point shape and+ /// ordering.+ pub fn try_new(ordering: LexOrdering, split_points: Vec<SplitPoint>) -> Result<Self> {+ validate_range_split_points(&ordering, &split_points)?;+ Ok(Self::new(ordering, split_points))+ }++ /// Returns the ordering that defines the range key.+ pub fn ordering(&self) -> &LexOrdering {+ &self.ordering+ }++ /// Returns the ordered split points between partitions.+ pub fn split_points(&self) -> &[SplitPoint] {+ &self.split_points+ }++ /// Returns the number of partitions.+ pub fn partition_count(&self) -> usize {+ self.split_points.len() + 1+ }++ fn project(+ &self,+ mapping: &ProjectionMapping,+ input_eq_properties: &EquivalenceProperties,+ ) -> Option<Self> {+ let exprs = self+ .ordering+ .iter()+ .map(|sort_expr| Arc::clone(&sort_expr.expr))+ .collect::<Vec<_>>();+ let projected_exprs = input_eq_properties+ .project_expressions(&exprs, mapping)+ .collect::<Option<Vec<_>>>()?;+ let sort_exprs = self+ .ordering+ .iter()+ .zip(projected_exprs)+ .map(|(sort_expr, expr)| PhysicalSortExpr::new(expr, sort_expr.options))+ .collect::<Vec<_>>();+ let ordering = LexOrdering::new(sort_exprs)?;+ if ordering.len() != self.ordering.len() {+ return None;+ }++ Some(Self {+ ordering,+ split_points: self.split_points.clone(),+ })+ }+}++impl Display for RangePartitioning {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ let split_points = format_range_split_points(&self.split_points);+ write!(+ f,+ "Range([{}], [{}], {})",+ self.ordering,+ split_points,+ self.partition_count()+ )+ }+}++fn format_range_split_points(split_points: &[SplitPoint]) -> String {+ split_points+ .iter()+ .map(ToString::to_string)+ .collect::<Vec<_>>()+ .join(", ")+}++fn validate_range_split_points(+ ordering: &LexOrdering,+ split_points: &[SplitPoint],+) -> Result<()> {+ let width = ordering.len();+ for (idx, split_point) in split_points.iter().enumerate() {+ let split_point_width = split_point.values.len();+ if split_point_width != width {+ return plan_err!(+ "Range partitioning split point {idx} has width {split_point_width}, but ordering has width {width}"+ );+ }+ }++ for (idx, split_points) in split_points.windows(2).enumerate() {+ if compare_split_points(ordering, &split_points[0], &split_points[1])?+ != Ordering::Less+ {+ return plan_err!(+ "Range partitioning split points must be strictly ordered: split point {idx} ({}) must be less than split point {} ({})",+ split_points[0],+ idx + 1,+ split_points[1]+ );+ }+ }++ Ok(())+}++fn compare_split_points(+ ordering: &LexOrdering,+ left: &SplitPoint,+ right: &SplitPoint,+) -> Result<Ordering> {+ for ((left_value, right_value), sort_expr) in+ left.values.iter().zip(&right.values).zip(ordering.iter())+ {+ let value_ordering =+ compare_scalar_values_for_sort(left_value, right_value, sort_expr)?;+ if value_ordering != Ordering::Equal {+ return Ok(value_ordering);+ }+ }++ Ok(Ordering::Equal)+}++fn compare_scalar_values_for_sort(+ left: &ScalarValue,+ right: &ScalarValue,+ sort_expr: &PhysicalSortExpr,+) -> Result<Ordering> {+ match (left.is_null(), right.is_null()) {+ (true, true) => Ok(Ordering::Equal),+ (true, false) => Ok(if sort_expr.options.nulls_first {+ Ordering::Less+ } else {+ Ordering::Greater+ }),+ (false, true) => Ok(if sort_expr.options.nulls_first {+ Ordering::Greater+ } else {+ Ordering::Less+ }),+ (false, false) => {+ let Some(ordering) = left.partial_cmp(right) else {+ return plan_err!(+ "Range partitioning split point values are not comparable: {left:?} and {right:?}"+ );+ };+ Ok(if sort_expr.options.descending {+ ordering.reverse()+ } else {+ ordering+ })+ }+ }+}+/// Represents how a [`Partitioning`] satisfies a [`Distribution`] requirement.#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum PartitioningSatisfaction {@@ -167,6 +438,7 @@ impl Partitioning {use Partitioning::*;match self {RoundRobinBatch(n) | Hash(_, n) | UnknownPartitioning(n) => *n,+ Range(range) => range.partition_count(),}}@@ -265,9 +537,13 @@ impl Partitioning {PartitioningSatisfaction::NotSatisfied}- _ => PartitioningSatisfaction::NotSatisfied,+ Partitioning::RoundRobinBatch(_)+ | Partitioning::Range(_)+ | Partitioning::UnknownPartitioning(_) => {+ PartitioningSatisfaction::NotSatisfied+ }},- _ => PartitioningSatisfaction::NotSatisfied,+ Distribution::SinglePartition => PartitioningSatisfaction::NotSatisfied,}}@@ -277,19 +553,29 @@ impl Partitioning {mapping: &ProjectionMapping,input_eq_properties: &EquivalenceProperties,) -> Self {- if let Partitioning::Hash(exprs, part) = self {- let normalized_exprs = input_eq_properties- .project_expressions(exprs, mapping)- .zip(exprs)- .map(|(proj_expr, expr)| {- proj_expr.unwrap_or_else(|| {- Arc::new(UnKnownColumn::new(&expr.to_string()))+ match self {+ Partitioning::Hash(exprs, part) => {+ let normalized_exprs = input_eq_properties+ .project_expressions(exprs, mapping)+ .zip(exprs)+ .map(|(proj_expr, expr)| {+ proj_expr.unwrap_or_else(|| {+ Arc::new(UnKnownColumn::new(&expr.to_string()))+ })})- })- .collect();- Partitioning::Hash(normalized_exprs, *part)- } else {- self.clone()+ .collect();+ Partitioning::Hash(normalized_exprs, *part)+ }+ Partitioning::Range(range) => {+ if let Some(projected) = range.project(mapping, input_eq_properties) {+ Partitioning::Range(projected)+ } else {+ Partitioning::UnknownPartitioning(range.partition_count())+ }+ }+ Partitioning::RoundRobinBatch(_) | Partitioning::UnknownPartitioning(_) => {+ self.clone()+ }}}}@@ -306,6 +592,7 @@ impl PartialEq for Partitioning {{true}+ (Partitioning::Range(left), Partitioning::Range(right)) => left == right,_ => false,}}@@ -356,56 +643,156 @@ mod tests {use super::*;use crate::expressions::Column;+ use crate::projection::ProjectionTargets;- use arrow::datatypes::{DataType, Field, Schema};+ use arrow::compute::SortOptions;+ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion_common::Result;+ struct PartitioningTestFixture {+ schema: SchemaRef,+ cols: Vec<Arc<dyn PhysicalExpr>>,+ eq_properties: EquivalenceProperties,+ }++ impl PartitioningTestFixture {+ fn new(fields: Vec<(&str, DataType)>) -> Result<Self> {+ let schema = Arc::new(Schema::new(+ fields+ .iter()+ .map(|(name, data_type)| Field::new(*name, data_type.clone(), false))+ .collect::<Vec<_>>(),+ ));+ let cols = fields+ .iter()+ .map(|(name, _)| {+ Ok(Arc::new(Column::new_with_schema(name, &schema)?)+ as Arc<dyn PhysicalExpr>)+ })+ .collect::<Result<_>>()?;+ let eq_properties = EquivalenceProperties::new(Arc::clone(&schema));++ Ok(Self {+ schema,+ cols,+ eq_properties,+ })+ }++ fn int64(names: &[&str]) -> Result<Self> {+ Self::new(names.iter().map(|name| (*name, DataType::Int64)).collect())+ }++ fn col(&self, index: usize) -> Arc<dyn PhysicalExpr> {+ Arc::clone(&self.cols[index])+ }++ fn cols(+ &self,+ indices: impl IntoIterator<Item = usize>,+ ) -> Vec<Arc<dyn PhysicalExpr>> {+ indices.into_iter().map(|index| self.col(index)).collect()+ }++ fn hash_partitioning(+ &self,+ indices: impl IntoIterator<Item = usize>,+ partition_count: usize,+ ) -> Partitioning {+ Partitioning::Hash(self.cols(indices), partition_count)+ }++ fn hash_distribution(+ &self,+ indices: impl IntoIterator<Item = usize>,+ ) -> Distribution {+ Distribution::HashPartitioned(self.cols(indices))+ }++ fn range_sort_expr(+ &self,+ index: usize,+ options: SortOptions,+ ) -> PhysicalSortExpr {+ PhysicalSortExpr::new(self.col(index), options)+ }++ fn range_ordering(+ &self,+ indices: impl IntoIterator<Item = usize>,+ ) -> LexOrdering {+ LexOrdering::new(+ indices+ .into_iter()+ .map(|index| PhysicalSortExpr::new_default(self.col(index))),+ )+ .expect("ordering must not be empty")+ }++ fn range(+ &self,+ indices: impl IntoIterator<Item = usize>,+ split_points: Vec<SplitPoint>,+ ) -> RangePartitioning {+ RangePartitioning::try_new(self.range_ordering(indices), split_points)+ .expect("test range partitioning should be valid")+ }++ fn range_partitioning(+ &self,+ indices: impl IntoIterator<Item = usize>,+ split_points: Vec<SplitPoint>,+ ) -> Partitioning {+ Partitioning::Range(self.range(indices, split_points))+ }++ fn range_partitioning_with_ordering(+ &self,+ ordering: LexOrdering,+ split_points: Vec<SplitPoint>,+ ) -> Partitioning {+ Partitioning::Range(+ RangePartitioning::try_new(ordering, split_points)+ .expect("test range partitioning should be valid"),+ )+ }+ }+#[test]fn partitioning_satisfy_distribution() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("column_1", DataType::Int64, false),- Field::new("column_2", DataType::Utf8, false),- ]));-- let partition_exprs1: Vec<Arc<dyn PhysicalExpr>> = vec![- Arc::new(Column::new_with_schema("column_1", &schema).unwrap()),- Arc::new(Column::new_with_schema("column_2", &schema).unwrap()),- ];-- let partition_exprs2: Vec<Arc<dyn PhysicalExpr>> = vec![- Arc::new(Column::new_with_schema("column_2", &schema).unwrap()),- Arc::new(Column::new_with_schema("column_1", &schema).unwrap()),- ];+ let fixture = PartitioningTestFixture::new(vec![+ ("column_1", DataType::Int64),+ ("column_2", DataType::Utf8),+ ])?;let distribution_types = vec![Distribution::UnspecifiedDistribution,Distribution::SinglePartition,- Distribution::HashPartitioned(partition_exprs1.clone()),+ fixture.hash_distribution([0, 1]),];let single_partition = Partitioning::UnknownPartitioning(1);let unspecified_partition = Partitioning::UnknownPartitioning(10);let round_robin_partition = Partitioning::RoundRobinBatch(10);- let hash_partition1 = Partitioning::Hash(partition_exprs1, 10);- let hash_partition2 = Partitioning::Hash(partition_exprs2, 10);- let eq_properties = EquivalenceProperties::new(schema);+ let hash_partition1 = fixture.hash_partitioning([0, 1], 10);+ let hash_partition2 = fixture.hash_partitioning([1, 0], 10);for distribution in distribution_types {let result = (single_partition- .satisfaction(&distribution, &eq_properties, true)+ .satisfaction(&distribution, &fixture.eq_properties, true).is_satisfied(),unspecified_partition- .satisfaction(&distribution, &eq_properties, true)+ .satisfaction(&distribution, &fixture.eq_properties, true).is_satisfied(),round_robin_partition- .satisfaction(&distribution, &eq_properties, true)+ .satisfaction(&distribution, &fixture.eq_properties, true).is_satisfied(),hash_partition1- .satisfaction(&distribution, &eq_properties, true)+ .satisfaction(&distribution, &fixture.eq_properties, true).is_satisfied(),hash_partition2- .satisfaction(&distribution, &eq_properties, true)+ .satisfaction(&distribution, &fixture.eq_properties, true).is_satisfied(),);@@ -427,72 +814,41 @@ mod tests {#[test]fn test_partitioning_satisfy_by_subset() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int64, false),- Field::new("b", DataType::Int64, false),- Field::new("c", DataType::Int64, false),- ]));-- let col_a: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("a", &schema)?);- let col_b: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("b", &schema)?);- let col_c: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("c", &schema)?);- let eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ let fixture = PartitioningTestFixture::int64(&["a", "b", "c"])?;let test_cases = vec![("Hash([a]) vs Hash([a, b])",- Partitioning::Hash(vec![Arc::clone(&col_a)], 4),- Distribution::HashPartitioned(vec![- Arc::clone(&col_a),- Arc::clone(&col_b),- ]),+ fixture.hash_partitioning([0], 4),+ fixture.hash_distribution([0, 1]),PartitioningSatisfaction::Subset,PartitioningSatisfaction::NotSatisfied,),("Hash([a]) vs Hash([a, b, c])",- Partitioning::Hash(vec![Arc::clone(&col_a)], 4),- Distribution::HashPartitioned(vec![- Arc::clone(&col_a),- Arc::clone(&col_b),- Arc::clone(&col_c),- ]),+ fixture.hash_partitioning([0], 4),+ fixture.hash_distribution([0, 1, 2]),PartitioningSatisfaction::Subset,PartitioningSatisfaction::NotSatisfied,),("Hash([a, b]) vs Hash([a, b, c])",- Partitioning::Hash(vec![Arc::clone(&col_a), Arc::clone(&col_b)], 4),- Distribution::HashPartitioned(vec![- Arc::clone(&col_a),- Arc::clone(&col_b),- Arc::clone(&col_c),- ]),+ fixture.hash_partitioning([0, 1], 4),+ fixture.hash_distribution([0, 1, 2]),PartitioningSatisfaction::Subset,PartitioningSatisfaction::NotSatisfied,),("Hash([b]) vs Hash([a, b, c])",- Partitioning::Hash(vec![Arc::clone(&col_b)], 4),- Distribution::HashPartitioned(vec![- Arc::clone(&col_a),- Arc::clone(&col_b),- Arc::clone(&col_c),- ]),+ fixture.hash_partitioning([1], 4),+ fixture.hash_distribution([0, 1, 2]),PartitioningSatisfaction::Subset,PartitioningSatisfaction::NotSatisfied,),("Hash([b, a]) vs Hash([a, b, c])",- Partitioning::Hash(vec![Arc::clone(&col_a)], 4),- Distribution::HashPartitioned(vec![- Arc::clone(&col_a),- Arc::clone(&col_b),- Arc::clone(&col_c),- ]),+ fixture.hash_partitioning([1, 0], 4),+ fixture.hash_distribution([0, 1, 2]),PartitioningSatisfaction::Subset,PartitioningSatisfaction::NotSatisfied,),@@ -501,13 +857,13 @@ mod tests {for (desc, partition, required, expected_with_subset, expected_without_subset) intest_cases{- let result = partition.satisfaction(&required, &eq_properties, true);+ let result = partition.satisfaction(&required, &fixture.eq_properties, true);assert_eq!(result, expected_with_subset,"Failed for {desc} with subset enabled");- let result = partition.satisfaction(&required, &eq_properties, false);+ let result = partition.satisfaction(&required, &fixture.eq_properties, false);assert_eq!(result, expected_without_subset,"Failed for {desc} with subset disabled"@@ -519,48 +875,27 @@ mod tests {#[test]fn test_partitioning_current_superset() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int64, false),- Field::new("b", DataType::Int64, false),- Field::new("c", DataType::Int64, false),- ]));-- let col_a: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("a", &schema)?);- let col_b: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("b", &schema)?);- let col_c: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("c", &schema)?);- let eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ let fixture = PartitioningTestFixture::int64(&["a", "b", "c"])?;let test_cases = vec![("Hash([a, b]) vs Hash([a])",- Partitioning::Hash(vec![Arc::clone(&col_a), Arc::clone(&col_b)], 4),- Distribution::HashPartitioned(vec![Arc::clone(&col_a)]),+ fixture.hash_partitioning([0, 1], 4),+ fixture.hash_distribution([0]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,),("Hash([a, b, c]) vs Hash([a])",- Partitioning::Hash(- vec![Arc::clone(&col_a), Arc::clone(&col_b), Arc::clone(&col_c)],- 4,- ),- Distribution::HashPartitioned(vec![Arc::clone(&col_a)]),+ fixture.hash_partitioning([0, 1, 2], 4),+ fixture.hash_distribution([0]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,),("Hash([a, b, c]) vs Hash([a, b])",- Partitioning::Hash(- vec![Arc::clone(&col_a), Arc::clone(&col_b), Arc::clone(&col_c)],- 4,- ),- Distribution::HashPartitioned(vec![- Arc::clone(&col_a),- Arc::clone(&col_b),- ]),+ fixture.hash_partitioning([0, 1, 2], 4),+ fixture.hash_distribution([0, 1]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,),@@ -569,13 +904,13 @@ mod tests {for (desc, partition, required, expected_with_subset, expected_without_subset) intest_cases{- let result = partition.satisfaction(&required, &eq_properties, true);+ let result = partition.satisfaction(&required, &fixture.eq_properties, true);assert_eq!(result, expected_with_subset,"Failed for {desc} with subset enabled");- let result = partition.satisfaction(&required, &eq_properties, false);+ let result = partition.satisfaction(&required, &fixture.eq_properties, false);assert_eq!(result, expected_without_subset,"Failed for {desc} with subset disabled"@@ -587,24 +922,12 @@ mod tests {#[test]fn test_partitioning_partial_overlap() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int64, false),- Field::new("b", DataType::Int64, false),- Field::new("c", DataType::Int64, false),- ]));-- let col_a: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("a", &schema)?);- let col_b: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("b", &schema)?);- let col_c: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("c", &schema)?);- let eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ let fixture = PartitioningTestFixture::int64(&["a", "b", "c"])?;let test_cases = vec![("Partial overlap: Hash([a, c]) vs Hash([a, b])",- Partitioning::Hash(vec![Arc::clone(&col_a), Arc::clone(&col_c)], 4),- Distribution::HashPartitioned(vec![Arc::clone(&col_a), Arc::clone(&col_b)]),+ fixture.hash_partitioning([0, 2], 4),+ fixture.hash_distribution([0, 1]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,)];@@ -612,13 +935,13 @@ mod tests {for (desc, partition, required, expected_with_subset, expected_without_subset) intest_cases{- let result = partition.satisfaction(&required, &eq_properties, true);+ let result = partition.satisfaction(&required, &fixture.eq_properties, true);assert_eq!(result, expected_with_subset,"Failed for {desc} with subset enabled");- let result = partition.satisfaction(&required, &eq_properties, false);+ let result = partition.satisfaction(&required, &fixture.eq_properties, false);assert_eq!(result, expected_without_subset,"Failed for {desc} with subset disabled"@@ -630,35 +953,20 @@ mod tests {#[test]fn test_partitioning_no_overlap() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int64, false),- Field::new("b", DataType::Int64, false),- Field::new("c", DataType::Int64, false),- ]));-- let col_a: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("a", &schema)?);- let col_b: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("b", &schema)?);- let col_c: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("c", &schema)?);- let eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ let fixture = PartitioningTestFixture::int64(&["a", "b", "c"])?;let test_cases = vec![("Hash([a]) vs Hash([b, c])",- Partitioning::Hash(vec![Arc::clone(&col_a)], 4),- Distribution::HashPartitioned(vec![- Arc::clone(&col_b),- Arc::clone(&col_c),- ]),+ fixture.hash_partitioning([0], 4),+ fixture.hash_distribution([1, 2]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,),("Hash([a, b]) vs Hash([c])",- Partitioning::Hash(vec![Arc::clone(&col_a), Arc::clone(&col_b)], 4),- Distribution::HashPartitioned(vec![Arc::clone(&col_c)]),+ fixture.hash_partitioning([0, 1], 4),+ fixture.hash_distribution([2]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,),@@ -667,13 +975,13 @@ mod tests {for (desc, partition, required, expected_with_subset, expected_without_subset) intest_cases{- let result = partition.satisfaction(&required, &eq_properties, true);+ let result = partition.satisfaction(&required, &fixture.eq_properties, true);assert_eq!(result, expected_with_subset,"Failed for {desc} with subset enabled");- let result = partition.satisfaction(&required, &eq_properties, false);+ let result = partition.satisfaction(&required, &fixture.eq_properties, false);assert_eq!(result, expected_without_subset,"Failed for {desc} with subset disabled"@@ -685,32 +993,20 @@ mod tests {#[test]fn test_partitioning_exact_match() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int64, false),- Field::new("b", DataType::Int64, false),- ]));-- let col_a: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("a", &schema)?);- let col_b: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("b", &schema)?);- let eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ let fixture = PartitioningTestFixture::int64(&["a", "b"])?;let test_cases = vec![("Hash([a, b]) vs Hash([a, b])",- Partitioning::Hash(vec![Arc::clone(&col_a), Arc::clone(&col_b)], 4),- Distribution::HashPartitioned(vec![- Arc::clone(&col_a),- Arc::clone(&col_b),- ]),+ fixture.hash_partitioning([0, 1], 4),+ fixture.hash_distribution([0, 1]),PartitioningSatisfaction::Exact,PartitioningSatisfaction::Exact,),("Hash([a]) vs Hash([a])",- Partitioning::Hash(vec![Arc::clone(&col_a)], 4),- Distribution::HashPartitioned(vec![Arc::clone(&col_a)]),+ fixture.hash_partitioning([0], 4),+ fixture.hash_distribution([0]),PartitioningSatisfaction::Exact,PartitioningSatisfaction::Exact,),@@ -719,13 +1015,13 @@ mod tests {for (desc, partition, required, expected_with_subset, expected_without_subset) intest_cases{- let result = partition.satisfaction(&required, &eq_properties, true);+ let result = partition.satisfaction(&required, &fixture.eq_properties, true);assert_eq!(result, expected_with_subset,"Failed for {desc} with subset enabled");- let result = partition.satisfaction(&required, &eq_properties, false);+ let result = partition.satisfaction(&required, &fixture.eq_properties, false);assert_eq!(result, expected_without_subset,"Failed for {desc} with subset disabled"@@ -737,32 +1033,20 @@ mod tests {#[test]fn test_partitioning_unknown() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int64, false),- Field::new("b", DataType::Int64, false),- ]));-- let col_a: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("a", &schema)?);- let col_b: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("b", &schema)?);+ let fixture = PartitioningTestFixture::int64(&["a", "b"])?;let unknown: Arc<dyn PhysicalExpr> = Arc::new(UnKnownColumn::new("dropped"));- let eq_properties = EquivalenceProperties::new(Arc::clone(&schema));let test_cases = vec![("Hash([unknown]) vs Hash([a, b])",Partitioning::Hash(vec![Arc::clone(&unknown)], 4),- Distribution::HashPartitioned(vec![- Arc::clone(&col_a),- Arc::clone(&col_b),- ]),+ fixture.hash_distribution([0, 1]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,),("Hash([a, b]) vs Hash([unknown])",- Partitioning::Hash(vec![Arc::clone(&col_a), Arc::clone(&col_b)], 4),+ fixture.hash_partitioning([0, 1], 4),Distribution::HashPartitioned(vec![Arc::clone(&unknown)]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,@@ -779,13 +1063,13 @@ mod tests {for (desc, partition, required, expected_with_subset, expected_without_subset) intest_cases{- let result = partition.satisfaction(&required, &eq_properties, true);+ let result = partition.satisfaction(&required, &fixture.eq_properties, true);assert_eq!(result, expected_with_subset,"Failed for {desc} with subset enabled");- let result = partition.satisfaction(&required, &eq_properties, false);+ let result = partition.satisfaction(&required, &fixture.eq_properties, false);assert_eq!(result, expected_without_subset,"Failed for {desc} with subset disabled"@@ -797,23 +1081,19 @@ mod tests {#[test]fn test_partitioning_empty_hash() -> Result<()> {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));-- let col_a: Arc<dyn PhysicalExpr> =- Arc::new(Column::new_with_schema("a", &schema)?);- let eq_properties = EquivalenceProperties::new(Arc::clone(&schema));+ let fixture = PartitioningTestFixture::int64(&["a"])?;let test_cases = vec![("Hash([]) vs Hash([a])",Partitioning::Hash(vec![], 4),- Distribution::HashPartitioned(vec![Arc::clone(&col_a)]),+ fixture.hash_distribution([0]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,),("Hash([a]) vs Hash([])",- Partitioning::Hash(vec![Arc::clone(&col_a)], 4),+ fixture.hash_partitioning([0], 4),Distribution::HashPartitioned(vec![]),PartitioningSatisfaction::NotSatisfied,PartitioningSatisfaction::NotSatisfied,@@ -830,13 +1110,13 @@ mod tests {for (desc, partition, required, expected_with_subset, expected_without_subset) intest_cases{- let result = partition.satisfaction(&required, &eq_properties, true);+ let result = partition.satisfaction(&required, &fixture.eq_properties, true);assert_eq!(result, expected_with_subset,"Failed for {desc} with subset enabled");- let result = partition.satisfaction(&required, &eq_properties, false);+ let result = partition.satisfaction(&required, &fixture.eq_properties, false);assert_eq!(result, expected_without_subset,"Failed for {desc} with subset disabled"@@ -845,4 +1125,160 @@ mod tests {Ok(())}++ fn int_split_point(values: impl IntoIterator<Item = i64>) -> SplitPoint {+ SplitPoint::new(+ values+ .into_iter()+ .map(|value| ScalarValue::Int64(Some(value)))+ .collect(),+ )+ }++ fn assert_range_try_new_error(+ ordering: LexOrdering,+ split_points: Vec<SplitPoint>,+ expected: &str,+ ) {+ let error = RangePartitioning::try_new(ordering, split_points)+ .unwrap_err()+ .to_string();+ assert!(error.contains(expected), "{error}");+ }++ #[test]+ fn test_range_partitioning_metadata() -> Result<()> {+ let fixture = PartitioningTestFixture::int64(&["a", "b"])?;++ let range_partitioning =+ fixture.range([0], vec![int_split_point([10]), int_split_point([20])]);+ assert_eq!(range_partitioning.ordering()[0].to_string(), "a@0 ASC");+ assert_eq!(+ range_partitioning.split_points(),+ &[int_split_point([10]), int_split_point([20])]+ );+ let partitioning = Partitioning::Range(range_partitioning);++ assert_eq!(partitioning.partition_count(), 3);+ assert_eq!(+ partitioning.to_string(),+ "Range([a@0 ASC], [(10), (20)], 3)"+ );++ Ok(())+ }++ #[test]+ fn test_range_partitioning_try_new_validates_split_points() -> Result<()> {+ let fixture = PartitioningTestFixture::int64(&["a", "b"])?;+ let asc_a = fixture.range_ordering([0]);+ let ordering_ab = fixture.range_ordering([0, 1]);++ assert_range_try_new_error(+ ordering_ab.clone(),+ vec![int_split_point([10])],+ "split point 0 has width 1, but ordering has width 2",+ );++ RangePartitioning::try_new(+ [fixture.range_sort_expr(0, SortOptions::new(true, false))].into(),+ vec![int_split_point([20]), int_split_point([10])],+ )?;++ assert_range_try_new_error(+ asc_a,+ vec![int_split_point([20]), int_split_point([10])],+ "split points must be strictly ordered",+ );++ assert_range_try_new_error(+ [fixture.range_sort_expr(0, SortOptions::new(false, false))].into(),+ vec![+ SplitPoint::new(vec![ScalarValue::Int64(None)]),+ int_split_point([10]),+ ],+ "split points must be strictly ordered",+ );++ RangePartitioning::try_new(+ ordering_ab.clone(),+ vec![int_split_point([10, 20]), int_split_point([10, 30])],+ )?;++ assert_range_try_new_error(+ ordering_ab,+ vec![int_split_point([10, 30]), int_split_point([10, 20])],+ "split points must be strictly ordered",+ );++ Ok(())+ }++ #[test]+ fn test_range_partitioning_project_preserves_or_degrades() -> Result<()> {+ let fixture = PartitioningTestFixture::int64(&["a", "b"])?;+ let range_partitioning = fixture.range_partitioning_with_ordering(+ [fixture.range_sort_expr(1, SortOptions::new(true, false))].into(),+ vec![int_split_point([10])],+ );++ let keep_b_mapping = ProjectionMapping::from_indices(&[1], &fixture.schema)?;+ let projected =+ range_partitioning.project(&keep_b_mapping, &fixture.eq_properties);+ assert_eq!(+ projected.to_string(),+ "Range([b@0 DESC NULLS LAST], [(10)], 2)"+ );++ let drop_b_mapping = ProjectionMapping::from_indices(&[0], &fixture.schema)?;+ let projected =+ range_partitioning.project(&drop_b_mapping, &fixture.eq_properties);+ let Partitioning::UnknownPartitioning(partition_count) = projected else {+ panic!("expected UnknownPartitioning, got {projected:?}");+ };+ assert_eq!(partition_count, 2);++ Ok(())+ }++ #[test]+ fn test_range_partitioning_project_degrades_if_ordering_collapses() -> Result<()> {+ let fixture = PartitioningTestFixture::int64(&["a", "b"])?;+ let target: Arc<dyn PhysicalExpr> = Arc::new(Column::new("x", 0));+ let range_partitioning =+ fixture.range_partitioning([0, 1], vec![int_split_point([10, 100])]);+ let mapping = ProjectionMapping::from_iter([+ (+ fixture.col(0),+ ProjectionTargets::from(vec![(Arc::clone(&target), 0)]),+ ),+ (+ fixture.col(1),+ ProjectionTargets::from(vec![(Arc::clone(&target), 0)]),+ ),+ ]);++ let projected = range_partitioning.project(&mapping, &fixture.eq_properties);+ let Partitioning::UnknownPartitioning(partition_count) = projected else {+ panic!("expected UnknownPartitioning, got {projected:?}");+ };+ assert_eq!(partition_count, 2);++ Ok(())+ }++ #[test]+ fn test_multi_partition_range_does_not_satisfy_hash_distribution() -> Result<()> {+ let fixture = PartitioningTestFixture::int64(&["a", "b"])?;+ let range_partitioning =+ fixture.range_partitioning([0, 1], vec![int_split_point([10, 100])]);+ let required = fixture.hash_distribution([0, 1]);++ assert_eq!(+ range_partitioning.satisfaction(&required, &fixture.eq_properties, false),+ PartitioningSatisfaction::NotSatisfied+ );++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex b4aa295562..9a6d1e5545 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -144,6 +144,13 @@ pub fn adjust_right_output_partitioning(.collect::<Result<_>>()?;Partitioning::Hash(new_exprs, *size)}+ Partitioning::Range(_) => {+ // Range partitioning optimizer propagation is tracked in+ // https://github.com/apache/datafusion/issues/22395+ return not_impl_err!(+ "Join output partitioning with range partitioning is not implemented"+ );+ }result => result.clone(),};Ok(result)diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rsindex 3005e97542..c7b1d4729e 100644--- a/datafusion/physical-plan/src/lib.rs+++ b/datafusion/physical-plan/src/lib.rs@@ -37,7 +37,7 @@ pub use datafusion_expr::{Accumulator, ColumnarValue};use datafusion_physical_expr::PhysicalSortExpr;pub use datafusion_physical_expr::window::WindowExpr;pub use datafusion_physical_expr::{- Distribution, Partitioning, PhysicalExpr, expressions,+ Distribution, Partitioning, PhysicalExpr, RangePartitioning, SplitPoint, expressions,};pub use crate::display::{DefaultDisplay, DisplayAs, DisplayFormatType, VerboseDisplay};diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex a6363378ed..3d30dd8276 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -737,6 +737,13 @@ impl BatchPartitioner {num_input_partitions,))}+ Partitioning::Range(_) => {+ // Range repartition execution is tracked in+ // https://github.com/apache/datafusion/issues/22397+ not_impl_err!(+ "Range partitioning execution is not implemented by RepartitionExec"+ )+ }other => {not_impl_err!("Unsupported repartitioning scheme {other:?}")}@@ -1430,6 +1437,13 @@ impl ExecutionPlan for RepartitionExec {}Partitioning::Hash(new_partitions, *size)}+ Partitioning::Range(_) => {+ // Range partitioning optimizer propagation is tracked in+ // https://github.com/apache/datafusion/issues/22395+ return not_impl_err!(+ "Projection pushdown through RepartitionExec with range partitioning is not implemented"+ );+ }others => others.clone(),};@@ -1466,6 +1480,18 @@ impl ExecutionPlan for RepartitionExec {if !self.maintains_input_order()[0] {return Ok(SortOrderPushdownResult::Unsupported);}+ match self.partitioning() {+ Partitioning::Range(_) => {+ // Range partitioning optimizer propagation is tracked in+ // https://github.com/apache/datafusion/issues/22395+ return not_impl_err!(+ "Sort pushdown through RepartitionExec with range partitioning is not implemented"+ );+ }+ Partitioning::RoundRobinBatch(_)+ | Partitioning::Hash(_, _)+ | Partitioning::UnknownPartitioning(_) => {}+ }// Delegate to the child and wrap with a new RepartitionExecself.input.try_pushdown_sort(order)?.try_map(|new_input| {@@ -1489,6 +1515,13 @@ impl ExecutionPlan for RepartitionExec {RoundRobinBatch(_) => RoundRobinBatch(target_partitions),Hash(hash, _) => Hash(hash, target_partitions),UnknownPartitioning(_) => UnknownPartitioning(target_partitions),+ Range(_) => {+ // Range repartition execution is tracked in+ // https://github.com/apache/datafusion/issues/22397+ return not_impl_err!(+ "Changing RepartitionExec partition counts with range partitioning is not implemented"+ );+ }};Ok(Some(Arc::new(Self {input: Arc::clone(&self.input),@@ -1617,6 +1650,13 @@ impl RepartitionExec {num_input_partitions,)}+ Partitioning::Range(_) => {+ // Range repartition execution is tracked in+ // https://github.com/apache/datafusion/issues/22397+ return not_impl_err!(+ "Range partitioning execution is not implemented by RepartitionExec"+ );+ }other => {return not_impl_err!("Unsupported repartitioning scheme {other:?}");}@@ -1968,12 +2008,14 @@ mod tests {use arrow::array::{ArrayRef, StringArray, UInt32Array};use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::ScalarValue;use datafusion_common::cast::as_string_array;use datafusion_common::exec_err;use datafusion_common::test_util::batches_to_sort_string;use datafusion_common_runtime::JoinSet;use datafusion_execution::config::SessionConfig;use datafusion_execution::runtime_env::RuntimeEnvBuilder;+ use datafusion_physical_expr::{PhysicalSortExpr, RangePartitioning, SplitPoint};use insta::assert_snapshot;#[test]@@ -2266,6 +2308,40 @@ mod tests {);}+ #[tokio::test]+ async fn unsupported_range_partitioning() -> Result<()> {+ let task_ctx = Arc::new(TaskContext::default());+ let batch = RecordBatch::try_from_iter(vec![(+ "my_awesome_field",+ Arc::new(StringArray::from(vec!["foo", "bar"])) as ArrayRef,+ )])?;++ let schema = batch.schema();+ let expr = col("my_awesome_field", &schema)?;+ let input = MockExec::new(vec![Ok(batch)], Arc::clone(&schema));+ let partitioning = Partitioning::Range(RangePartitioning::new(+ [PhysicalSortExpr::new_default(expr)].into(),+ vec![SplitPoint::new(vec![ScalarValue::Utf8(Some(+ "foo".to_string(),+ ))])],+ ));+ let exec = RepartitionExec::try_new(Arc::new(input), partitioning)?;+ let output_stream = exec.execute(0, task_ctx)?;++ let result_string = crate::common::collect(output_stream)+ .await+ .unwrap_err()+ .to_string();+ assert!(+ result_string.contains(+ "Range partitioning execution is not implemented by RepartitionExec"+ ),+ "actual: {result_string}"+ );++ Ok(())+ }+#[tokio::test]async fn error_for_input_exec() {// This generates an error on a call to execute. The errordiff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex f715de0b59..929ff4f7df 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -1141,7 +1141,8 @@ impl ExecutionPlan for SortExec {vec![Distribution::UnspecifiedDistribution]} else {// global sort- // TODO support RangePartition and OrderedDistribution+ // TODO support range partitioning and OrderedDistribution.+ // See https://github.com/apache/datafusion/issues/22395vec![Distribution::SinglePartition]}}diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rsindex 09570f14ba..eb9b5f09aa 100644--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs@@ -1486,11 +1486,7 @@ mod tests {let task_ctx = Arc::new(TaskContext::default());let schema = Schema::new(vec![Field::new("c1", DataType::UInt64, false)]);let properties = CongestedExec::compute_properties(Arc::new(schema.clone()));- let &partition_count = match properties.output_partitioning() {- Partitioning::RoundRobinBatch(partitions) => partitions,- Partitioning::Hash(_, partitions) => partitions,- Partitioning::UnknownPartitioning(partitions) => partitions,- };+ let partition_count = properties.output_partitioning().partition_count();let source = CongestedExec {schema: schema.clone(),cache: Arc::new(properties),diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex ea6d078366..2185748c70 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1381,13 +1381,22 @@ message PhysicalHashRepartition {uint64 partition_count = 2;}+message PhysicalRangePartitioning {+ repeated PhysicalSortExprNode sort_expr = 1;+ repeated PhysicalRangeSplitPoint split_point = 2;+}++message PhysicalRangeSplitPoint {+ repeated datafusion_common.ScalarValue value = 1;+}+message RepartitionExecNode{PhysicalPlanNode input = 1;- // oneof partition_method {+ // Legacy direct partitioning fields:// uint64 round_robin = 2;// PhysicalHashRepartition hash = 3;// uint64 unknown = 4;- // }+ // New partitioning variants are stored in `partitioning`.Partitioning partitioning = 5;bool preserve_order = 6;}@@ -1397,6 +1406,7 @@ message Partitioning {uint64 round_robin = 1;PhysicalHashRepartition hash = 2;uint64 unknown = 3;+ PhysicalRangePartitioning range = 4;}}diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex 8e6997757f..4136cd2785 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -15768,6 +15768,9 @@ impl serde::Serialize for Partitioning {#[allow(clippy::needless_borrows_for_generic_args)]struct_ser.serialize_field("unknown", ToString::to_string(&v).as_str())?;}+ partitioning::PartitionMethod::Range(v) => {+ struct_ser.serialize_field("range", v)?;+ }}}struct_ser.end()@@ -15784,6 +15787,7 @@ impl<'de> serde::Deserialize<'de> for Partitioning {"roundRobin","hash","unknown",+ "range",];#[allow(clippy::enum_variant_names)]@@ -15791,6 +15795,7 @@ impl<'de> serde::Deserialize<'de> for Partitioning {RoundRobin,Hash,Unknown,+ Range,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -15815,6 +15820,7 @@ impl<'de> serde::Deserialize<'de> for Partitioning {"roundRobin" | "round_robin" => Ok(GeneratedField::RoundRobin),"hash" => Ok(GeneratedField::Hash),"unknown" => Ok(GeneratedField::Unknown),+ "range" => Ok(GeneratedField::Range),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -15856,6 +15862,13 @@ impl<'de> serde::Deserialize<'de> for Partitioning {}partition_method__ = map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| partitioning::PartitionMethod::Unknown(x.0));}+ GeneratedField::Range => {+ if partition_method__.is_some() {+ return Err(serde::de::Error::duplicate_field("range"));+ }+ partition_method__ = map_.next_value::<::std::option::Option<_>>()?.map(partitioning::PartitionMethod::Range)+;+ }}}Ok(Partitioning {@@ -19051,6 +19064,207 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {deserializer.deserialize_struct("datafusion.PhysicalPlanNode", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for PhysicalRangePartitioning {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if !self.sort_expr.is_empty() {+ len += 1;+ }+ if !self.split_point.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalRangePartitioning", len)?;+ if !self.sort_expr.is_empty() {+ struct_ser.serialize_field("sortExpr", &self.sort_expr)?;+ }+ if !self.split_point.is_empty() {+ struct_ser.serialize_field("splitPoint", &self.split_point)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for PhysicalRangePartitioning {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "sort_expr",+ "sortExpr",+ "split_point",+ "splitPoint",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ SortExpr,+ SplitPoint,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "sortExpr" | "sort_expr" => Ok(GeneratedField::SortExpr),+ "splitPoint" | "split_point" => Ok(GeneratedField::SplitPoint),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = PhysicalRangePartitioning;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.PhysicalRangePartitioning")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalRangePartitioning, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut sort_expr__ = None;+ let mut split_point__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::SortExpr => {+ if sort_expr__.is_some() {+ return Err(serde::de::Error::duplicate_field("sortExpr"));+ }+ sort_expr__ = Some(map_.next_value()?);+ }+ GeneratedField::SplitPoint => {+ if split_point__.is_some() {+ return Err(serde::de::Error::duplicate_field("splitPoint"));+ }+ split_point__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(PhysicalRangePartitioning {+ sort_expr: sort_expr__.unwrap_or_default(),+ split_point: split_point__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.PhysicalRangePartitioning", FIELDS, GeneratedVisitor)+ }+}+impl serde::Serialize for PhysicalRangeSplitPoint {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if !self.value.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalRangeSplitPoint", len)?;+ if !self.value.is_empty() {+ struct_ser.serialize_field("value", &self.value)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for PhysicalRangeSplitPoint {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "value",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Value,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "value" => Ok(GeneratedField::Value),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = PhysicalRangeSplitPoint;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.PhysicalRangeSplitPoint")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalRangeSplitPoint, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut value__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Value => {+ if value__.is_some() {+ return Err(serde::de::Error::duplicate_field("value"));+ }+ value__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(PhysicalRangeSplitPoint {+ value: value__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.PhysicalRangeSplitPoint", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for PhysicalScalarSubqueryExprNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex d8187e65a5..4e473668e8 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -2044,14 +2044,26 @@ pub struct PhysicalHashRepartition {pub partition_count: u64,}#[derive(Clone, PartialEq, ::prost::Message)]+pub struct PhysicalRangePartitioning {+ #[prost(message, repeated, tag = "1")]+ pub sort_expr: ::prost::alloc::vec::Vec<PhysicalSortExprNode>,+ #[prost(message, repeated, tag = "2")]+ pub split_point: ::prost::alloc::vec::Vec<PhysicalRangeSplitPoint>,+}+#[derive(Clone, PartialEq, ::prost::Message)]+pub struct PhysicalRangeSplitPoint {+ #[prost(message, repeated, tag = "1")]+ pub value: ::prost::alloc::vec::Vec<super::datafusion_common::ScalarValue>,+}+#[derive(Clone, PartialEq, ::prost::Message)]pub struct RepartitionExecNode {#[prost(message, optional, boxed, tag = "1")]pub input: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,- /// oneof partition_method {+ /// Legacy direct partitioning fields:/// uint64 round_robin = 2;/// PhysicalHashRepartition hash = 3;/// uint64 unknown = 4;- /// }+ /// New partitioning variants are stored in `partitioning`.#[prost(message, optional, tag = "5")]pub partitioning: ::core::option::Option<Partitioning>,#[prost(bool, tag = "6")]@@ -2059,7 +2071,7 @@ pub struct RepartitionExecNode {}#[derive(Clone, PartialEq, ::prost::Message)]pub struct Partitioning {- #[prost(oneof = "partitioning::PartitionMethod", tags = "1, 2, 3")]+ #[prost(oneof = "partitioning::PartitionMethod", tags = "1, 2, 3, 4")]pub partition_method: ::core::option::Option<partitioning::PartitionMethod>,}/// Nested message and enum types in `Partitioning`.@@ -2072,6 +2084,8 @@ pub mod partitioning {Hash(super::PhysicalHashRepartition),#[prost(uint64, tag = "3")]Unknown(u64),+ #[prost(message, tag = "4")]+ Range(super::PhysicalRangePartitioning),}}#[derive(Clone, PartialEq, ::prost::Message)]diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex d2a48aa457..7d2e68d810 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -24,7 +24,9 @@ use arrow::compute::SortOptions;use arrow::datatypes::{Field, Schema};use arrow::ipc::reader::StreamReader;use chrono::{TimeZone, Utc};-use datafusion_common::{DataFusionError, Result, internal_datafusion_err, not_impl_err};+use datafusion_common::{+ DataFusionError, Result, ScalarValue, internal_datafusion_err, not_impl_err,+};use datafusion_datasource::file::FileSource;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};@@ -48,7 +50,9 @@ use datafusion_physical_plan::expressions::{};use datafusion_physical_plan::joins::{HashExpr, SeededRandomState};use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};-use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};+use datafusion_physical_plan::{+ Partitioning, PhysicalExpr, RangePartitioning, SplitPoint, WindowExpr,+};use datafusion_proto_common::common::proto_error;use object_store::ObjectMeta;use object_store::path::Path;@@ -560,6 +564,14 @@ pub fn parse_protobuf_partitioning(proto_converter,)}+ Some(protobuf::partitioning::PartitionMethod::Range(range_partitioning)) => {+ Ok(Some(parse_protobuf_range_partitioning(+ range_partitioning,+ ctx,+ input_schema,+ proto_converter,+ )?))+ }Some(protobuf::partitioning::PartitionMethod::Unknown(partition_count)) => {Ok(Some(Partitioning::UnknownPartitioning(*partition_count as usize,@@ -571,6 +583,49 @@ pub fn parse_protobuf_partitioning(}}+fn parse_protobuf_range_partitioning(+ range_partitioning: &protobuf::PhysicalRangePartitioning,+ ctx: &PhysicalPlanDecodeContext<'_>,+ input_schema: &Schema,+ proto_converter: &dyn PhysicalProtoConverterExtension,+) -> Result<Partitioning> {+ let sort_exprs = parse_physical_sort_exprs(+ &range_partitioning.sort_expr,+ ctx,+ input_schema,+ proto_converter,+ )?;+ let sort_expr_count = sort_exprs.len();+ let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {+ internal_datafusion_err!("Range partitioning requires non-empty ordering")+ })?;+ if ordering.len() != sort_expr_count {+ return Err(internal_datafusion_err!(+ "Range partitioning ordering must not contain duplicate expressions"+ ));+ }+ let split_points = range_partitioning+ .split_point+ .iter()+ .map(parse_protobuf_range_split_point)+ .collect::<Result<_>>()?;+ Ok(Partitioning::Range(RangePartitioning::try_new(+ ordering,+ split_points,+ )?))+}++fn parse_protobuf_range_split_point(+ split_point: &protobuf::PhysicalRangeSplitPoint,+) -> Result<SplitPoint> {+ let values = split_point+ .value+ .iter()+ .map(|value| ScalarValue::try_from(value).map_err(Into::into))+ .collect::<Result<_>>()?;+ Ok(SplitPoint::new(values))+}+pub fn parse_protobuf_file_scan_schema(proto: &protobuf::FileScanExecConf,) -> Result<Arc<Schema>> {diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex 28c0a57e94..9cb9e89760 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -42,7 +42,9 @@ use datafusion_physical_plan::expressions::{use datafusion_physical_plan::joins::HashExpr;use datafusion_physical_plan::udaf::AggregateFunctionExpr;use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};-use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};+use datafusion_physical_plan::{+ Partitioning, PhysicalExpr, RangePartitioning, SplitPoint, WindowExpr,+};use super::{DefaultPhysicalProtoConverter, PhysicalExtensionCodec,@@ -542,6 +544,11 @@ pub fn serialize_partitioning()),}}+ Partitioning::Range(range) => protobuf::Partitioning {+ partition_method: Some(protobuf::partitioning::PartitionMethod::Range(+ serialize_range_partitioning(range, codec, proto_converter)?,+ )),+ },Partitioning::UnknownPartitioning(partition_count) => protobuf::Partitioning {partition_method: Some(protobuf::partitioning::PartitionMethod::Unknown(*partition_count as u64,@@ -551,6 +558,40 @@ pub fn serialize_partitioning(Ok(serialized_partitioning)}+fn serialize_range_partitioning(+ range: &RangePartitioning,+ codec: &dyn PhysicalExtensionCodec,+ proto_converter: &dyn PhysicalProtoConverterExtension,+) -> Result<protobuf::PhysicalRangePartitioning> {+ Ok(protobuf::PhysicalRangePartitioning {+ sort_expr: serialize_physical_sort_exprs(+ range.ordering().iter().cloned(),+ codec,+ proto_converter,+ )?,+ split_point: range+ .split_points()+ .iter()+ .map(serialize_range_split_point)+ .collect::<Result<_>>()?,+ })+}++fn serialize_range_split_point(+ split_point: &SplitPoint,+) -> Result<protobuf::PhysicalRangeSplitPoint> {+ Ok(protobuf::PhysicalRangeSplitPoint {+ value: split_point+ .values()+ .iter()+ .map(|value| {+ TryInto::<datafusion_proto_common::ScalarValue>::try_into(value)+ .map_err(Into::into)+ })+ .collect::<Result<_>>()?,+ })+}+fn serialize_when_then_expr(when_expr: &Arc<dyn PhysicalExpr>,then_expr: &Arc<dyn PhysicalExpr>,diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex d88a360422..bd996eb692 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -90,7 +90,8 @@ use datafusion::physical_plan::windows::{};use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, InputOrderMode, Partitioning,- PhysicalExpr, SendableRecordBatchStream, Statistics, displayable,+ PhysicalExpr, RangePartitioning, SendableRecordBatchStream, SplitPoint, Statistics,+ displayable,};use datafusion::prelude::{ParquetReadOptions, SessionContext};use datafusion::scalar::ScalarValue;@@ -1806,6 +1807,21 @@ fn roundtrip_repartition_preserve_order() -> Result<()> {roundtrip_test(Arc::new(repartition))}+#[test]+fn roundtrip_range_partitioning() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let input = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let range_partitioning = Partitioning::Range(RangePartitioning::new(+ [PhysicalSortExpr::new_default(col("a", &schema)?)].into(),+ vec![SplitPoint::new(vec![ScalarValue::Int64(Some(10))])],+ ));+ // RepartitionExec is used only to carry the partitioning through proto.+ // Executing range repartitioning is intentionally unsupported.+ let repartition = RepartitionExec::try_new(input, range_partitioning)?;++ roundtrip_test(Arc::new(repartition))+}+#[test]fn roundtrip_interleave() -> Result<()> {let field_a = Field::new("col", DataType::Int64, false);