Apache DataFusion SQL Query Engine
fix: handle nested RANGE CURRENT ROW peers (#26067)
Chao Sunb93ef03be4d1
diff --git a/datafusion/core/tests/execution/mod.rs b/datafusion/core/tests/execution/mod.rsindex f33ef87aa3..24a3ff6c7f 100644--- a/datafusion/core/tests/execution/mod.rs+++ b/datafusion/core/tests/execution/mod.rs@@ -19,3 +19,4 @@ mod coop;mod datasource_split;mod logical_plan;mod register_arrow;+mod window_nested_range;diff --git a/datafusion/core/tests/execution/window_nested_range.rs b/datafusion/core/tests/execution/window_nested_range.rsnew file mode 100644index 0000000000..f2343167eb--- /dev/null+++ b/datafusion/core/tests/execution/window_nested_range.rs@@ -0,0 +1,345 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Nested RANGE peers through public physical execution APIs.+//! Construct plans directly to test peer boundaries independently of SQL type admission.++use std::sync::Arc;+use std::time::Duration;++use arrow::array::{ArrayRef, AsArray, Int32Array, Int64Array, ListArray, StructArray};+use arrow::buffer::{NullBuffer, OffsetBuffer};+use arrow::compute::concat_batches;+use arrow::datatypes::{DataType, Field, Int32Type, Int64Type, SchemaRef};+use arrow::record_batch::RecordBatch;+use datafusion::datasource::memory::MemorySourceConfig;+use datafusion::datasource::source::DataSourceExec;+use datafusion::execution::TaskContext;+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;+use datafusion::physical_plan::streaming::{PartitionStream, StreamingTableExec};+use datafusion::physical_plan::windows::{+ BoundedWindowAggExec, WindowAggExec, create_window_expr,+};+use datafusion::physical_plan::{+ ExecutionPlan, ExecutionPlanProperties, InputOrderMode, SendableRecordBatchStream,+ collect,+};+use datafusion::prelude::SessionContext;+use datafusion_common::{Result, ScalarValue};+use datafusion_expr::{+ WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition,+};+use datafusion_functions_aggregate::sum::sum_udaf;+use datafusion_physical_expr::PhysicalSortExpr;+use datafusion_physical_expr::expressions::col;+use datafusion_physical_expr_common::sort_expr::LexOrdering;+use futures::{FutureExt, StreamExt};++fn nested_batches() -> Result<Vec<RecordBatch>> {+ // Three nested keys: a NULL container, a nested NULL, and nested 1.+ // Hidden values under the two NULL containers deliberately differ.+ let values = [Some(10), Some(99), None, None, None, Some(1)];+ let values: ArrayRef = Arc::new(Int32Array::from(values.to_vec()));+ let lists: ArrayRef = Arc::new(ListArray::new(+ Arc::new(Field::new("item", DataType::Int32, true)),+ OffsetBuffer::from_lengths([1; 6]),+ Arc::clone(&values),+ None,+ ));+ let structs: ArrayRef = Arc::new(StructArray::new(+ vec![Arc::new(Field::new("item", DataType::Int32, true))].into(),+ vec![values],+ None,+ ));+ let nulls = NullBuffer::from(vec![false, false, true, true, true, true]);+ let list_field = Arc::new(Field::new("item", lists.data_type().clone(), true));+ let keys: [ArrayRef; 3] = [+ Arc::new(ListArray::new(+ Arc::clone(&list_field),+ OffsetBuffer::from_lengths([1; 6]),+ Arc::clone(&lists),+ Some(nulls.clone()),+ )),+ Arc::new(ListArray::new(+ Arc::new(Field::new("item", structs.data_type().clone(), true)),+ OffsetBuffer::from_lengths([1; 6]),+ structs,+ Some(nulls.clone()),+ )),+ Arc::new(StructArray::new(+ vec![list_field].into(),+ vec![lists],+ Some(nulls),+ )),+ ];+ keys.into_iter()+ .map(|keys| {+ // The tie separates the third nested NULL from the preceding peers.+ let tie: ArrayRef = Arc::new(Int32Array::from(vec![0, 0, 0, 0, 1, 0]));+ Ok(RecordBatch::try_from_iter(vec![+ ("key", keys),+ ("tie", tie),+ // In Linear mode, the final row advances another partition's peers.+ (+ "p",+ Arc::new(Int32Array::from(vec![0, 0, 0, 0, 0, 1])) as ArrayRef,+ ),+ (+ "value",+ Arc::new(Int64Array::from(vec![1, 2, 4, 8, 16, 32])) as ArrayRef,+ ),+ ])?)+ })+ .collect()+}++fn nested_ordering(schema: &SchemaRef) -> Result<LexOrdering> {+ Ok(LexOrdering::from([+ PhysicalSortExpr::new_default(col("key", schema)?),+ PhysicalSortExpr::new_default(col("tie", schema)?),+ ]))+}++fn window_plan(+ input: &Arc<dyn ExecutionPlan>,+ mode: Option<&InputOrderMode>,+ start: WindowFrameBound,+ end: WindowFrameBound,+) -> Result<Arc<dyn ExecutionPlan>> {+ let schema = input.schema();+ let partition_by = if mode == Some(&InputOrderMode::Linear) {+ vec![col("p", &schema)?]+ } else {+ vec![]+ };+ let expr = create_window_expr(+ &WindowFunctionDefinition::AggregateUDF(sum_udaf()),+ "sum".to_string(),+ &[col("value", &schema)?],+ &partition_by,+ input.output_ordering().unwrap(),+ Arc::new(WindowFrame::new_bounds(WindowFrameUnits::Range, start, end)),+ schema,+ false,+ false,+ None,+ )?;+ let plan: Arc<dyn ExecutionPlan> = match mode {+ Some(mode) => Arc::new(BoundedWindowAggExec::try_new(+ vec![expr],+ Arc::clone(input),+ mode.clone(),+ false,+ )?),+ None => Arc::new(WindowAggExec::try_new(+ vec![expr],+ Arc::clone(input),+ false,+ )?),+ };+ assert!(+ input+ .equivalence_properties()+ .ordering_satisfy_requirement(+ plan.required_input_ordering()[0]+ .clone()+ .unwrap()+ .into_single(),+ )?+ );+ Ok(plan)+}++#[tokio::test]+async fn nested_range_current_row_physical_operators() -> Result<()> {+ use WindowFrameBound::{CurrentRow, Following, Preceding};++ for batch in nested_batches()? {+ let schema = batch.schema();+ // Repeated peers cross input batch boundaries.+ let batches = (0..batch.num_rows())+ .map(|row| batch.slice(row, 1))+ .collect();+ let source = MemorySourceConfig::try_new(&[batches], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![nested_ordering(&schema)?])?;+ let input: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(source);+ // window.slt covers list keys in SQL-planned sorted windows.+ let modes: &[Option<InputOrderMode>] = if batch.column(0).data_type().is_list() {+ &[Some(InputOrderMode::Linear)]+ } else {+ &[+ None,+ Some(InputOrderMode::Sorted),+ Some(InputOrderMode::Linear),+ ]+ };+ for (start, end, expected) in [+ (+ Preceding(ScalarValue::UInt64(None)),+ CurrentRow,+ [3, 3, 15, 15, 31, 63],+ ),+ (CurrentRow, CurrentRow, [3, 3, 12, 12, 16, 32]),+ (+ CurrentRow,+ Following(ScalarValue::UInt64(None)),+ [63, 63, 60, 60, 48, 32],+ ),+ ] {+ for mode in modes.iter().map(Option::as_ref) {+ // The planner uses WindowAggExec for UNBOUNDED FOLLOWING.+ if mode.is_some() && end.is_unbounded() {+ continue;+ }+ let plan = window_plan(&input, mode, start.clone(), end.clone())?;+ let output_schema = plan.schema();+ let output = collect(plan, SessionContext::new().task_ctx()).await?;+ let output = concat_batches(&output_schema, &output)?;+ let actual = output+ .column(batch.num_columns())+ .as_primitive::<Int64Type>();+ let mut expected = expected;+ if mode == Some(&InputOrderMode::Linear) {+ // The final row is in its own PARTITION BY group.+ expected[5] = 32;+ }+ assert_eq!(+ actual.iter().collect::<Vec<_>>(),+ expected.map(Some),+ "{}, {start:?} to {end:?}, mode={mode:?}",+ batch.column(0).data_type(),+ );+ }+ }+ }+ Ok(())+}++#[derive(Debug)]+struct OpenEndedPartition {+ batch: RecordBatch,+}++impl PartitionStream for OpenEndedPartition {+ fn schema(&self) -> &SchemaRef {+ self.batch.schema_ref()+ }++ fn execute(&self, _ctx: Arc<TaskContext>) -> SendableRecordBatchStream {+ let batches = (0..self.batch.num_rows())+ .map(|row| Ok(self.batch.slice(row, 1)))+ .collect::<Vec<_>>();+ Box::pin(RecordBatchStreamAdapter::new(+ self.batch.schema(),+ futures::stream::iter(batches).chain(futures::stream::pending()),+ ))+ }+}++async fn assert_emits_before_eof(+ batch: &RecordBatch,+ order_by: LexOrdering,+ mode: InputOrderMode,+ expected: &[Option<i64>],+) -> Result<()> {+ let source: Arc<dyn ExecutionPlan> = Arc::new(StreamingTableExec::try_new(+ batch.schema(),+ vec![Arc::new(OpenEndedPartition {+ batch: batch.clone(),+ })],+ None,+ vec![order_by],+ true,+ None,+ )?);+ let plan = window_plan(+ &source,+ Some(&mode),+ WindowFrameBound::Preceding(ScalarValue::UInt64(None)),+ WindowFrameBound::CurrentRow,+ )?;+ let mut stream = plan.execute(0, SessionContext::new().task_ctx())?;+ let actual = tokio::time::timeout(Duration::from_secs(5), async {+ let mut sums = Vec::new();+ while sums.len() < expected.len() {+ let output = stream.next().await.unwrap()?;+ sums.extend(+ output+ .column(batch.num_columns())+ .as_primitive::<Int64Type>()+ .iter(),+ );+ }+ Ok::<_, datafusion_common::DataFusionError>(sums)+ })+ .await+ .expect("completed peer groups should emit before EOF")?;+ assert_eq!(+ actual,+ expected,+ "{}, mode={mode:?}",+ batch.column(0).data_type()+ );+ // The final peer group cannot finish until another key or EOF arrives.+ assert!(stream.next().now_or_never().is_none());+ Ok(())+}++#[tokio::test]+async fn nested_range_emits_completed_peers_before_eof() -> Result<()> {+ for batch in nested_batches()? {+ for mode in [InputOrderMode::Sorted, InputOrderMode::Linear] {+ assert_emits_before_eof(+ &batch,+ nested_ordering(batch.schema_ref())?,+ mode,+ &[Some(3), Some(3), Some(15), Some(15), Some(31)],+ )+ .await?;+ }+ }+ Ok(())+}++#[tokio::test]+async fn nested_range_desc_nulls_last_emits_before_eof() -> Result<()> {+ // Both [1] and a top-level NULL sort after [2] under DESC NULLS LAST.+ // Each stream stops after that row arrives in a different partition.+ for last_key in [Some([Some(1)]), None] {+ let keys = ListArray::from_iter_primitive::<Int32Type, _, _>([+ Some([Some(2)]),+ Some([Some(2)]),+ last_key,+ ]);+ let batch = RecordBatch::try_from_iter([+ ("key", Arc::new(keys) as ArrayRef),+ ("p", Arc::new(Int32Array::from(vec![0, 0, 1]))),+ ("value", Arc::new(Int64Array::from(vec![1, 2, 4]))),+ ])?;+ let order = PhysicalSortExpr::new_default(col("key", batch.schema_ref())?)+ .desc()+ .nulls_last();+ assert_emits_before_eof(+ &batch,+ [order].into(),+ InputOrderMode::Linear,+ &[Some(3), Some(3)],+ )+ .await?;+ }+ Ok(())+}diff --git a/datafusion/expr/src/window_state.rs b/datafusion/expr/src/window_state.rsindex acba75268b..514fdc839c 100644--- a/datafusion/expr/src/window_state.rs+++ b/datafusion/expr/src/window_state.rs@@ -21,6 +21,7 @@ use std::{collections::VecDeque, ops::Range, sync::Arc};use crate::{WindowFrame, WindowFrameBound, WindowFrameUnits};+use arrow::compute::kernels::sort::{LexicographicalComparator, SortColumn};use arrow::{array::ArrayRef,compute::{SortOptions, concat, concat_batches},@@ -70,7 +71,11 @@ impl WindowAggState {match self.window_frame_ctx.as_mut() {// Rows have no state do nothingSome(WindowFrameContext::Rows(_)) => {}- Some(WindowFrameContext::Range { .. }) => {}+ Some(WindowFrameContext::Range { state, .. }) => {+ // Release the old ORDER BY arrays early. Replacement arrays would+ // also invalidate the cache by identity when it is next used.+ state.peer_comparator = None;+ }Some(WindowFrameContext::Groups { state, .. }) => {let mut n_group_to_del = 0;for (_, end_idx) in &state.group_end_indices {@@ -340,15 +345,95 @@ impl PartitionBatchState {/// ranges of data while processing RANGE frames./// Attribute `sort_options` stores the column ordering specified by the ORDER/// BY clause. This information is used to calculate the range.-#[derive(Debug, Default, Clone)]+#[derive(Debug, Default)]pub struct WindowFrameStateRange {sort_options: Vec<SortOptions>,+ peer_comparator: Option<RangePeerComparator>,+}++/// Reuse comparators to avoid per-row setup work, including logical-null masks+/// that can take O(n) to build for dictionary arrays.+struct RangePeerComparator {+ columns: Vec<ArrayRef>,+ comparator: LexicographicalComparator,+}++impl RangePeerComparator {+ fn matches_columns(&self, columns: &[ArrayRef]) -> bool {+ // Sliced arrays have different row indices even when they share buffers.+ self.columns.len() == columns.len()+ && self+ .columns+ .iter()+ .zip(columns)+ .all(|(old, new)| Arc::ptr_eq(old, new))+ }+}++impl Clone for WindowFrameStateRange {+ fn clone(&self) -> Self {+ // A clone builds its own comparator when it first sees nested keys.+ Self::new(self.sort_options.clone())+ }+}++// Preserve the public WindowFrameStateRange type's existing unwind-safety traits.+// The cached Arrow arrays and comparison closures are immutable, so a comparison+// that unwinds cannot leave this cache partially updated.+impl std::panic::UnwindSafe for RangePeerComparator {}+impl std::panic::RefUnwindSafe for RangePeerComparator {}++impl std::fmt::Debug for RangePeerComparator {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("RangePeerComparator")+ .field("num_columns", &self.columns.len())+ .finish_non_exhaustive()+ }}impl WindowFrameStateRange {/// Create a new object to store the search state.fn new(sort_options: Vec<SortOptions>) -> Self {- Self { sort_options }+ Self {+ sort_options,+ peer_comparator: None,+ }+ }++ fn peer_boundary<const START: bool>(+ &mut self,+ range_columns: &[ArrayRef],+ idx: usize,+ mut boundary: usize,+ length: usize,+ ) -> Result<usize> {+ let cached = match &self.peer_comparator {+ Some(cached) if cached.matches_columns(range_columns) => cached,+ _ => {+ let sort_columns = range_columns+ .iter()+ .zip(&self.sort_options)+ .map(|(values, options)| SortColumn {+ values: Arc::clone(values),+ options: Some(*options),+ })+ .collect::<Vec<_>>();+ self.peer_comparator.insert(RangePeerComparator {+ columns: range_columns.to_vec(),+ comparator: LexicographicalComparator::try_new(&sort_columns)?,+ })+ }+ };+ // ScalarValue equality can distinguish encodings of the same nested value:+ // a list with a null dictionary key and one with a key referencing null.+ while boundary < length {+ let is_peer = cached.comparator.compare(boundary, idx).is_eq();+ if is_peer == START {+ break;+ }+ boundary += 1;+ }+ Ok(boundary)}/// This function calculates beginning/ending indices for the frame of the current row.@@ -364,6 +449,12 @@ impl WindowFrameStateRange {length: usize,idx: usize,) -> Result<Range<usize>> {+ let has_current_row = window_frame.start_bound == WindowFrameBound::CurrentRow+ || window_frame.end_bound == WindowFrameBound::CurrentRow;+ let nested_current_row = has_current_row+ && range_columns+ .iter()+ .any(|column| column.data_type().is_nested());let start = match window_frame.start_bound {WindowFrameBound::Preceding(ref n) => {if n.is_null() {@@ -379,6 +470,9 @@ impl WindowFrameStateRange {)?}}+ WindowFrameBound::CurrentRow if nested_current_row => {+ self.peer_boundary::<true>(range_columns, idx, last_range.start, length)?+ }WindowFrameBound::CurrentRow => self.calculate_index_of_row::<true, true>(range_columns,last_range,@@ -404,6 +498,16 @@ impl WindowFrameStateRange {Some(n),length,)?,+ WindowFrameBound::CurrentRow if nested_current_row => {+ // An equality scan cannot skip preceding non-peers if evaluation+ // starts partway through a partition.+ self.peer_boundary::<false>(+ range_columns,+ idx,+ last_range.end.max(idx),+ length,+ )?+ }WindowFrameBound::CurrentRow => self.calculate_index_of_row::<false, false>(range_columns,last_range,@@ -433,7 +537,7 @@ impl WindowFrameStateRange {/// called twice, in succession, to get window frame start and end indices (with `SIDE`/// supplied as true and false, respectively).fn calculate_index_of_row<const SIDE: bool, const SEARCH_SIDE: bool>(- &mut self,+ &self,range_columns: &[ArrayRef],last_range: &Range<usize>,idx: usize,@@ -729,6 +833,99 @@ mod tests {use arrow::array::{Float64Array, UInt64Array};use arrow::datatypes::{Field, Schema};+ #[test]+ fn nested_range_peers_across_cloning_and_pruning() -> Result<()> {+ use arrow::array::ListArray;+ use arrow::datatypes::Int64Type;++ fn assert_unwind_safe<T: std::panic::UnwindSafe + std::panic::RefUnwindSafe>() {}+ assert_unwind_safe::<WindowFrameStateRange>();++ let frame = Arc::new(WindowFrame::new_bounds(+ WindowFrameUnits::Range,+ WindowFrameBound::CurrentRow,+ WindowFrameBound::CurrentRow,+ ));+ let columns: [ArrayRef; 1] =+ [Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(+ [1, 1, 2, 2].into_iter().map(|v| Some(vec![Some(v)])),+ ))];+ let mut range = WindowFrameStateRange::new(vec![SortOptions::default()]);+ assert_eq!(+ range.calculate_range(&frame, &(0..0), &columns, 4, 0)?,+ 0..2+ );+ assert_eq!(+ range+ .clone()+ .calculate_range(&frame, &(0..2), &columns, 4, 1)?,+ 0..2+ );++ let columns = [columns[0].slice(1, 3)];+ assert_eq!(+ range.calculate_range(&frame, &(0..0), &columns, 3, 0)?,+ 0..1+ );+ let retained = Arc::downgrade(&columns[0]);+ let mut state = WindowAggState::new(&DataType::Int64)?;+ state.window_frame_range =+ range.calculate_range(&frame, &(0..1), &columns, 3, 1)?;+ assert_eq!(state.window_frame_range, 1..3);+ state.last_calculated_index = 1;+ state.window_frame_ctx = Some(WindowFrameContext::Range {+ window_frame: frame,+ state: range,+ });+ let pruned = [columns[0].slice(1, 2)];+ drop(columns);+ assert!(retained.upgrade().is_some());+ state.prune_state(1);+ assert!(retained.upgrade().is_none());+ assert_eq!(+ state.window_frame_ctx.as_mut().unwrap().calculate_range(+ &pruned,+ &state.window_frame_range,+ 2,+ 0,+ )?,+ 0..2+ );+ Ok(())+ }++ #[test]+ fn range_peers_compare_nested_dictionary_nulls_logically() -> Result<()> {+ use arrow::array::{DictionaryArray, Int8Array, Int32Array, ListArray};+ use arrow::buffer::OffsetBuffer;+ use arrow::datatypes::Int8Type;++ // Both lists contain NULL: one dictionary key is null, the other+ // references a null value. ScalarValue equality distinguishes them.+ let values: ArrayRef = Arc::new(DictionaryArray::<Int8Type>::try_new(+ Int8Array::from(vec![None, Some(0)]),+ Arc::new(Int32Array::from(vec![None])),+ )?);+ let columns: [ArrayRef; 1] = [Arc::new(ListArray::new(+ Arc::new(Field::new("item", values.data_type().clone(), true)),+ OffsetBuffer::from_lengths([1, 1]),+ values,+ None,+ ))];+ let frame = Arc::new(WindowFrame::new_bounds(+ WindowFrameUnits::Range,+ WindowFrameBound::CurrentRow,+ WindowFrameBound::CurrentRow,+ ));+ let mut state = WindowFrameContext::new(frame, vec![SortOptions::default()]);+ let mut range = 0..0;+ for idx in 0..2 {+ range = state.calculate_range(&columns, &range, 2, idx)?;+ assert_eq!(range, 0..2);+ }+ Ok(())+ }+fn get_test_data() -> (Vec<ArrayRef>, Vec<SortOptions>) {let range_columns: Vec<ArrayRef> = vec![Arc::new(Float64Array::from(vec![5.0, 7.0, 8.0, 8.0, 9., 10., 10., 10., 11.,diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex e31dc65173..bd3aea11ae 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -1325,11 +1325,9 @@ fn extract_window_frame_target_type(col_type: &DataType) -> Option<DataType> {/// run over an ORDER BY column of `col_type` even though the type has no/// arithmetic for finite offsets.///-/// Such a frame only compares rows to find peers, so the type must compare-/// the same way in the RANGE peer check (`ScalarValue::partial_cmp`) as in-/// the sort that produced the input order. That holds for durations and-/// intervals; it does not for structs and maps, whose `ScalarValue`-/// comparison differs from the sorter's, so they stay unsupported.+/// This fallback admits durations and intervals, including encoded values.+/// Admission remains conservative: Arrow peer comparison also supports nested+/// types, but admitting structs and maps requires separate validation.fn supports_free_range_frame(col_type: &DataType) -> bool {match col_type {DataType::Duration(_) | DataType::Interval(_) => true,@@ -1342,16 +1340,8 @@ fn supports_free_range_frame(col_type: &DataType) -> bool {}/// Whether `col_type` is a list, possibly behind dictionary or run-end-/// encoding.-///-/// Lists are kept out of the free-range fallback because their peer-/// comparison and the sort do not agree on element NULLs: `compare_rows`-/// applies the NULLS FIRST / NULLS LAST option to the top-level value only and-/// then calls `ScalarValue::partial_cmp`, whose `partial_cmp_list` always-/// orders a NULL element after a non-NULL one (Postgres semantics), while the-/// sorter's `make_comparator` applies the option to the elements as well. Under-/// `ORDER BY d, l NULLS FIRST` with tied `d`, the sort puts `[NULL]` before-/// `[1]` and the peer check orders them the other way round.+/// encoding. Free-range fallback still excludes lists conservatively, although+/// Arrow can now find peers for supported list keys.fn is_list_type(col_type: &DataType) -> bool {match col_type {DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) => {@@ -1364,8 +1354,9 @@ fn is_list_type(col_type: &DataType) -> bool {}/// Errors if any ORDER BY expression has a type not supported in a free RANGE-/// frame: a type with neither an offset target nor a sound peer comparison, or-/// a list (see `is_list_type`).+/// frame: a type with neither an offset target nor support in the free-range+/// fallback, or a list (see `is_list_type`). These conservative admission rules+/// can be relaxed separately from the nested peer-comparison fix.fn check_free_range_order_by_types(expressions: &[Sort],schema: &DFSchema,@@ -1407,9 +1398,8 @@ fn coerce_window_frame(}target_type}- // A free range frame has no offsets to coerce, so ORDER BY- // types without arithmetic are fine as long as their peer- // comparison is sound (see `supports_free_range_frame`).+ // A free range frame has no offsets to coerce. Check whether+ // its ORDER BY types are supported by the free-range fallback.None if window_frame.free_range() => {check_free_range_order_by_types(expressions, schema)?;return Ok(window_frame);diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rsindex 5f6bbc46c2..4b2e8258fd 100644--- a/datafusion/physical-expr/src/window/window_expr.rs+++ b/datafusion/physical-expr/src/window/window_expr.rs@@ -23,7 +23,7 @@ use std::sync::Arc;use crate::PhysicalExpr;use arrow::array::BooleanArray;-use arrow::array::{Array, ArrayRef, new_empty_array};+use arrow::array::{Array, ArrayRef, make_comparator, new_empty_array};use arrow::compute::SortOptions;use arrow::compute::filter as arrow_filter;use arrow::compute::kernels::sort::SortColumn;@@ -31,7 +31,6 @@ use arrow::datatypes::FieldRef;use arrow::record_batch::RecordBatch;use datafusion_common::cast::as_boolean_array;use datafusion_common::hash_utils::RandomState;-use datafusion_common::utils::compare_rows;use datafusion_common::{Result, ScalarValue, arrow_datafusion_err, exec_datafusion_err, exec_err,internal_err,@@ -353,6 +352,10 @@ pub trait AggregateWindowExpr: WindowExpr {let mut row_wise_results: Vec<ScalarValue> =Vec::with_capacity(length.saturating_sub(idx));let is_causal = self.get_window_frame().is_causal();+ // These ordering arrays do not change during this evaluation. Compare+ // their last/most-recent rows lazily once, since comparator setup can+ // scan entire dictionary arrays to compute logical nulls.+ let mut row_ahead = None;while idx < length {// Start search from the last_range. This squeezes searched range.let cur_range =@@ -367,6 +370,7 @@ pub trait AggregateWindowExpr: WindowExpr {most_recent_row_order_bys,self.order_by(),idx,+ &mut row_ahead,)?{break;@@ -422,6 +426,7 @@ pub(crate) fn filter_arrays(/// columns./// * `sort_exprs`: Defines the lexicographical ordering in question./// * `idx`: The current index in the window frame.+/// * `row_ahead`: The last/most-recent row comparison, reused within one evaluation.////// # Returns///@@ -432,6 +437,7 @@ pub(crate) fn is_end_bound_safe(most_recent_order_bys: Option<&[ArrayRef]>,sort_exprs: &[PhysicalSortExpr],idx: usize,+ row_ahead: &mut Option<bool>,) -> Result<bool> {if sort_exprs.is_empty() {// Early return if no sort expressions are present:@@ -448,6 +454,7 @@ pub(crate) fn is_end_bound_safe(most_recent_order_bys.map(|items| &items[0]),&sort_exprs[0].options,idx,+ row_ahead,),WindowFrameContext::Groups {window_frame,@@ -458,6 +465,7 @@ pub(crate) fn is_end_bound_safe(&order_bys[0],most_recent_order_bys.map(|items| &items[0]),&sort_exprs[0].options,+ row_ahead,),}}@@ -505,13 +513,14 @@ fn is_end_bound_safe_for_range(most_recent_ob_col: Option<&ArrayRef>,sort_options: &SortOptions,idx: usize,+ row_ahead: &mut Option<bool>,) -> Result<bool> {match end_bound {// A frame ending in `N PRECEDING` can only reach the end of the buffer// when the current row's frame ends at its last peer, as for a NULL or// NaN ORDER BY value, so it is safe once a row past its peers arrives.WindowFrameBound::Preceding(_) | WindowFrameBound::CurrentRow => {- is_row_ahead(orderby_col, most_recent_ob_col, sort_options)+ is_row_ahead(orderby_col, most_recent_ob_col, sort_options, row_ahead)}WindowFrameBound::Following(delta) => {let Some(most_recent_ob_col) = most_recent_ob_col else {@@ -556,23 +565,24 @@ fn is_end_bound_safe_for_groups(orderby_col: &ArrayRef,most_recent_ob_col: Option<&ArrayRef>,sort_options: &SortOptions,+ row_ahead: &mut Option<bool>,) -> Result<bool> {match end_bound {WindowFrameBound::Preceding(value) => {let zero = ScalarValue::new_zero(&value.data_type())?;if value.eq(&zero) {- is_row_ahead(orderby_col, most_recent_ob_col, sort_options)+ is_row_ahead(orderby_col, most_recent_ob_col, sort_options, row_ahead)} else {Ok(true)}}WindowFrameBound::CurrentRow => {- is_row_ahead(orderby_col, most_recent_ob_col, sort_options)+ is_row_ahead(orderby_col, most_recent_ob_col, sort_options, row_ahead)}WindowFrameBound::Following(ScalarValue::UInt64(Some(offset))) => {let delta = state.group_end_indices.len() - state.current_group_idx;if delta == (*offset as usize) + 1 {- is_row_ahead(orderby_col, most_recent_ob_col, sort_options)+ is_row_ahead(orderby_col, most_recent_ob_col, sort_options, row_ahead)} else {Ok(false)}@@ -581,23 +591,27 @@ fn is_end_bound_safe_for_groups(}}-/// This utility function checks whether `current_cols` is ahead of the `old_cols`-/// in terms of `sort_options`.+/// Checks whether the most recent input row is ahead of the partition's last row.+/// Both arrays stay unchanged during one evaluation, so reuse the comparison.fn is_row_ahead(old_col: &ArrayRef,current_col: Option<&ArrayRef>,sort_options: &SortOptions,+ row_ahead: &mut Option<bool>,) -> Result<bool> {+ if let Some(ahead) = *row_ahead {+ return Ok(ahead);+ }let Some(current_col) = current_col else {return Ok(false);};if old_col.is_empty() || current_col.is_empty() {return Ok(false);}- let last_value = ScalarValue::try_from_array(old_col, old_col.len() - 1)?;- let current_value = ScalarValue::try_from_array(current_col, 0)?;- let cmp = compare_rows(&[current_value], &[last_value], &[*sort_options])?;- Ok(cmp.is_gt())+ let cmp = make_comparator(old_col.as_ref(), current_col.as_ref(), *sort_options)?;+ let ahead = cmp(old_col.len() - 1, 0).is_lt();+ *row_ahead = Some(ahead);+ Ok(ahead)}/// Get order by expression results inside `order_by_columns`.@@ -783,7 +797,8 @@ mod tests {&SortOptions {descending: false,nulls_first: false- }+ },+ &mut None,)?);assert!(!is_row_ahead(&old_values,@@ -791,7 +806,8 @@ mod tests {&SortOptions {descending: false,nulls_first: false- }+ },+ &mut None,)?);Ok(())diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.sltindex 7d050ec4d3..1422875db3 100644--- a/datafusion/sqllogictest/test_files/window.slt+++ b/datafusion/sqllogictest/test_files/window.slt@@ -7547,8 +7547,7 @@ statement error Error during planning: RANGE window frames are not supported forselect count(*) over (order by d, m) from (values (arrow_cast(1, 'Duration(Second)'), map(['a'], [1])), (arrow_cast(1, 'Duration(Second)'), map(['a'], [2]))) t(d, m)# the trailing key check also runs when the first ORDER BY type supports offset-# arithmetic; otherwise these maps are treated as peers because their keys are-# equal even though their values differ+# arithmetic, so a map after an integer is rejected as wellstatement error Error during planning: RANGE window frames are not supported for ORDER BY type Mapselect count(*) over (order by i, m) from (values (1, map(['a'], [1])), (1, map(['a'], [2]))) t(i, m)@@ -7567,23 +7566,64 @@ select count(*) over (order by l) from (values ([1]), ([2])) t(l)12-# but a list whose elements are nested (struct, list) plans and then fails at-# execution: the peer comparison in the RANGE frame goes through-# `ScalarValue::partial_cmp`, which cannot compare nested list elements, while-# the sort that produced the input uses `make_comparator` and can. Documented-# here so the fix to the scalar comparison can flip these expectations.-query error DataFusion error: Internal error: Uncomparable values: List\(\[\{c0: 1\}\]\), List\(\[\{c0: 1\}\]\)-select count(*) over (order by l) from (values ([struct(1)]), ([struct(2)])) t(l)--query error DataFusion error: Internal error: Uncomparable values: List\(\[\[1\]\]\), List\(\[\[1\]\]\)-select count(*) over (order by l) from (values ([[1]]), ([[2]])) t(l)--# after a duration key the free-range check looks at every ORDER BY type, and-# a list is rejected at planning even though it sorts fine: the RANGE peer-# check (`compare_rows`, then `ScalarValue::partial_cmp`) always orders a NULL-# list element after a non-NULL one, while the sort applies NULLS FIRST to the-# elements as well, so with tied durations `[NULL]` and `[1]` would be sorted-# one way and compared the other+# NULL elements must form the same peers as Arrow's list sort order.+query I+select count(*) over (order by l nulls first) from (values ([NULL::int]), ([1]), ([NULL::int])) t(l)+----+2+2+3++# With one row per batch, nested peers cross batch boundaries. The tie column+# separates rows with the same nested key.+statement ok+set datafusion.execution.batch_size = 1;++statement ok+create table nested_range_peers as+select * from (values+ (NULL, NULL, 0, 1),+ (NULL, NULL, 0, 2),+ ([[NULL::int]], [struct(NULL::int)], 0, 4),+ ([[NULL::int]], [struct(NULL::int)], 0, 8),+ ([[NULL::int]], [struct(NULL::int)], 1, 16),+ ([[1]], [struct(1::int)], 0, 32)+) t(lists, structs, tie, v);++query IIII+select count(*) over (order by lists nulls first, tie),+ count(*) over (order by structs nulls first, tie),+ sum(v) over (order by lists nulls first, tie range between current row and current row),+ sum(v) over (order by structs nulls first, tie range between current row and current row)+from nested_range_peers order by v+----+2 2 3 3+2 2 3 3+4 4 12 12+4 4 12 12+5 5 16 16+6 6 32 32++query II+select sum(v) over (order by lists nulls first, tie range between current row and unbounded following),+ sum(v) over (order by structs nulls first, tie range between current row and unbounded following)+from nested_range_peers order by v+----+63 63+63 63+60 60+60 60+48 48+32 32++statement ok+drop table nested_range_peers;++statement ok+reset datafusion.execution.batch_size;++# after a duration key the free-range check looks at every ORDER BY type.+# Trailing list keys remain rejected by this check.query error Error during planning: RANGE window frames are not supported for ORDER BY type Listselect count(*) over (order by d, l nulls first) from (values (arrow_cast(1, 'Duration(Second)'), [NULL]), (arrow_cast(1, 'Duration(Second)'), [1])) t(d, l)