Apache DataFusion SQL Query Engine
Expose accumulator state to allow prefix scanning (#24035)
Brent Gardnera253a6a426a6
diff --git a/datafusion/physical-expr/src/window/standard.rs b/datafusion/physical-expr/src/window/standard.rsindex 2de080ec9a..278b66c373 100644--- a/datafusion/physical-expr/src/window/standard.rs+++ b/datafusion/physical-expr/src/window/standard.rs@@ -176,6 +176,7 @@ impl WindowExpr for StandardWindowExpr {.or_insert(WindowState {state: new_state.clone(),window_fn: WindowFn::Builtin(evaluator),+ published: false,})};let evaluator = match &mut window_state.window_fn {diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rsindex 47147b909d..3f8f0dc158 100644--- a/datafusion/physical-expr/src/window/window_expr.rs+++ b/datafusion/physical-expr/src/window/window_expr.rs@@ -33,7 +33,8 @@ use datafusion_common::cast::as_boolean_array;use datafusion_common::hash_utils::RandomState;use datafusion_common::utils::compare_rows;use datafusion_common::{- Result, ScalarValue, arrow_datafusion_err, exec_datafusion_err, internal_err,+ Result, ScalarValue, arrow_datafusion_err, exec_datafusion_err, exec_err,+ internal_err,};use datafusion_expr::window_state::{PartitionBatchState, WindowAggState, WindowFrameContext, WindowFrameStateGroups,@@ -250,6 +251,7 @@ pub trait AggregateWindowExpr: WindowExpr {WindowState {state: WindowAggState::new(out_type)?,window_fn: WindowFn::Aggregate(accumulator),+ published: false,},);};@@ -646,7 +648,49 @@ impl<'a> WindowEvalContext<'a> {pub struct WindowState {pub state: WindowAggState,pub window_fn: WindowFn,+ /// True once [`Self::aggregate_state`] has been called on this entry.+ /// Guards against a second destructive [`Accumulator::state`] read: the+ /// method itself errors on second call, and the observer loop in+ /// `BoundedWindowAggStream::publish_finalized_states` uses this as an+ /// early-skip so it doesn't attempt one. Independent of `state.is_end`,+ /// which is a group-closed signal that the pruning path also reads.+ pub published: bool,}++impl WindowState {+ /// [`Accumulator::state`] if this window function is an aggregate, `None`+ /// otherwise (built-in functions like `row_number`, `rank`, `lead`/`lag`+ /// have no serializable accumulator state).+ ///+ /// [`Accumulator::state`] takes `&mut self` and its trait doc calls out+ /// that "this function should not be called twice, otherwise it will+ /// result in potentially non-deterministic behavior." Several built-in+ /// impls (`median`, `percentile_cont`, `string_agg`,+ /// `min_max_bytes`/`min_max_struct`) `std::mem::take` their internal+ /// buffers on call — a second call returns *empty* state, not the same+ /// state, so a downstream prefix-merge would silently lose every value+ /// the accumulator had ingested.+ ///+ /// Enforced at this layer: on first call we set [`Self::published`] and+ /// return the state; any later call errors rather than performing a+ /// destructive re-read.+ pub fn aggregate_state(&mut self) -> Result<Option<Vec<ScalarValue>>> {+ if self.published {+ return exec_err!(+ "WindowState::aggregate_state called more than once; \+ Accumulator::state is a destructive read for several \+ built-in aggregates and a second call would silently lose data"+ );+ }+ let state = match &mut self.window_fn {+ WindowFn::Aggregate(accumulator) => Some(accumulator.state()?),+ WindowFn::Builtin(_) => None,+ };+ self.published = true;+ Ok(state)+ }+}+pub type PartitionWindowAggStates = IndexMap<PartitionKey, WindowState, RandomState>;/// The IndexMap (i.e. an ordered HashMap) where record batches are separated for each partition.@@ -656,11 +700,66 @@ pub type PartitionBatches = IndexMap<PartitionKey, PartitionBatchState, RandomStmod tests {use std::sync::Arc;- use crate::window::window_expr::is_row_ahead;+ use crate::window::window_expr::{WindowFn, WindowState, is_row_ahead};use arrow::array::{ArrayRef, Float64Array};use arrow::compute::SortOptions;- use datafusion_common::Result;+ use arrow::datatypes::DataType;+ use datafusion_common::{Result, ScalarValue};+ use datafusion_expr::{Accumulator, window_state::WindowAggState};++ /// Minimal [`Accumulator`] whose `state()` records how many times it was+ /// called by returning the count as its single state element. Any second+ /// call would surface (were it allowed to happen) as `[UInt64(2)]`+ /// instead of `[UInt64(1)]`.+ #[derive(Debug)]+ struct CallCountingAccumulator {+ calls: usize,+ }++ impl Accumulator for CallCountingAccumulator {+ fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {+ Ok(())+ }+ fn evaluate(&mut self) -> Result<ScalarValue> {+ Ok(ScalarValue::Null)+ }+ fn size(&self) -> usize {+ size_of::<Self>()+ }+ fn state(&mut self) -> Result<Vec<ScalarValue>> {+ self.calls += 1;+ Ok(vec![ScalarValue::UInt64(Some(self.calls as u64))])+ }+ fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {+ Ok(())+ }+ }++ #[test]+ fn aggregate_state_errors_on_second_call() -> Result<()> {+ // `Accumulator::state()` is a destructive read for several built-in+ // aggregates (median, percentile_cont, string_agg, min_max_bytes/+ // min_max_struct all `mem::take` their internal buffers). Its trait+ // doc says "should not be called twice"; `WindowState::aggregate_state`+ // enforces that at this layer by returning an error rather than+ // performing the second read.+ let acc: Box<dyn Accumulator> = Box::new(CallCountingAccumulator { calls: 0 });+ let mut ws = WindowState {+ state: WindowAggState::new(&DataType::UInt64)?,+ window_fn: WindowFn::Aggregate(acc),+ published: false,+ };+ let first = ws.aggregate_state()?;+ assert_eq!(first, Some(vec![ScalarValue::UInt64(Some(1))]));+ assert!(ws.published, "published must flip on successful publish");+ let err = ws.aggregate_state().unwrap_err().to_string();+ assert!(+ err.contains("called more than once"),+ "expected second-call error, got: {err}"+ );+ Ok(())+ }#[test]fn test_is_row_ahead() -> Result<()> {diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rsindex 952aae9846..adbc3dde7a 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs@@ -1178,6 +1178,7 @@ pub fn ensure_distribution(exec.window_expr(),exec.input(),&exec.partition_keys(),+ None,)? {plan = updated_window;}@@ -1186,6 +1187,7 @@ pub fn ensure_distribution(exec.window_expr(),exec.input(),&exec.partition_keys(),+ exec.state_observer().cloned(),)?{plan = updated_window;diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rsindex c66d5310a1..90b19ca95b 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/mod.rs@@ -556,17 +556,33 @@ fn adjust_window_sort_removal(window_tree.children.push(child_node);let child_plan = &window_tree.children[0].plan;+ // Captured up-front so the fallback `BoundedWindowAggExec::try_new` below+ // can reinstall the observer that was on the source exec. `None` when+ // the source is a `WindowAggExec` (no observer) or when no observer was+ // installed on the source `BoundedWindowAggExec`.+ let state_observer = window_tree+ .plan+ .downcast_ref::<BoundedWindowAggExec>()+ .and_then(|exec| exec.state_observer().cloned());let (window_expr, new_window) = if let Some(exec) =window_tree.plan.downcast_ref::<WindowAggExec>(){let window_expr = exec.window_expr();- let new_window =- get_best_fitting_window(window_expr, child_plan, &exec.partition_keys())?;+ let new_window = get_best_fitting_window(+ window_expr,+ child_plan,+ &exec.partition_keys(),+ None,+ )?;(window_expr, new_window)} else if let Some(exec) = window_tree.plan.downcast_ref::<BoundedWindowAggExec>() {let window_expr = exec.window_expr();- let new_window =- get_best_fitting_window(window_expr, child_plan, &exec.partition_keys())?;+ let new_window = get_best_fitting_window(+ window_expr,+ child_plan,+ &exec.partition_keys(),+ state_observer.clone(),+ )?;(window_expr, new_window)} else {return plan_err!("Expected WindowAggExec or BoundedWindowAggExec");@@ -589,12 +605,15 @@ fn adjust_window_sort_removal(window_tree.children.push(child_node);if window_expr.iter().all(|e| e.uses_bounded_memory()) {- Arc::new(BoundedWindowAggExec::try_new(- window_expr.to_vec(),- child_plan,- InputOrderMode::Sorted,- !window_expr[0].partition_by().is_empty(),- )?) as _+ Arc::new(+ BoundedWindowAggExec::try_new(+ window_expr.to_vec(),+ child_plan,+ InputOrderMode::Sorted,+ !window_expr[0].partition_by().is_empty(),+ )?+ .with_state_observer(state_observer)?,+ ) as _} else {Arc::new(WindowAggExec::try_new(window_expr.to_vec(),diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rsindex e9a0d9f47c..6ee4d1d305 100644--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs@@ -55,7 +55,7 @@ use datafusion_common::utils::{evaluate_partition_ranges, get_at_indices, get_row_at_idx,};use datafusion_common::{- HashMap, Result, arrow_datafusion_err, exec_datafusion_err, exec_err,+ HashMap, Result, ScalarValue, arrow_datafusion_err, exec_datafusion_err, exec_err,};use datafusion_execution::TaskContext;use datafusion_expr::ColumnarValue;@@ -77,8 +77,47 @@ use hashbrown::hash_table::HashTable;use indexmap::IndexMap;use log::debug;+/// Callback receiver for per-partition window state.+///+/// `state` is the result of [`Accumulator::state`], which is a `&mut self`+/// call whose trait doc states "this function should not be called twice."+/// Several built-in aggregates (`median`, `percentile_cont`, `string_agg`,+/// `min_max_bytes`/`min_max_struct`) `std::mem::take` their internal+/// buffers to build that state — so `state` is a destructive read, not a+/// snapshot. The exec fires this at most once per group; a callee that+/// needs the value beyond the callback must retain it (e.g. clone into+/// owned storage).+///+/// [`Accumulator::state`]: datafusion_expr::Accumulator::state+pub trait WindowStateObserver: Send + Sync {+ /// Invoked once per (output-partition-index, window-expression,+ /// PARTITION BY tuple) as each PARTITION BY group closes, for every+ /// aggregate window expression on the exec. Non-aggregate window+ /// functions (e.g. `row_number`, `rank`, `lead`/`lag`) do not fire this+ /// callback.+ ///+ /// # Arguments+ ///+ /// * `partition_idx` - Output partition index of the [`BoundedWindowAggExec`]+ /// stream firing this callback.+ /// * `window_expr` - The window expression whose state just closed.+ /// * `partition_key` - The PARTITION BY tuple that just closed.+ /// * `state` - [`Accumulator::state`] for the closed group of+ /// `window_expr`. See the trait-level doc for the destructive-read+ /// contract.+ ///+ /// [`Accumulator::state`]: datafusion_expr::Accumulator::state+ fn finalize_window_aggregate(+ &self,+ partition_idx: usize,+ window_expr: &Arc<dyn WindowExpr>,+ partition_key: &PartitionKey,+ state: Vec<ScalarValue>,+ ) -> Result<()>;+}+/// Window execution plan-#[derive(Debug, Clone)]+#[derive(Clone)]pub struct BoundedWindowAggExec {/// Input planinput: Arc<dyn ExecutionPlan>,@@ -101,6 +140,32 @@ pub struct BoundedWindowAggExec {cache: Arc<PlanProperties>,/// If `can_rerepartition` is false, partition_keys is always empty.can_repartition: bool,+ /// Invoked at partition-close to publish finalized per-partition window+ /// state. Storage and multi-group handling are the caller's; the exec is+ /// a pure event source.+ state_observer: Option<Arc<dyn WindowStateObserver>>,+}++impl std::fmt::Debug for BoundedWindowAggExec {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("BoundedWindowAggExec")+ .field("input", &self.input)+ .field("window_expr", &self.window_expr)+ .field("schema", &self.schema)+ .field("metrics", &self.metrics)+ .field("input_order_mode", &self.input_order_mode)+ .field(+ "ordered_partition_by_indices",+ &self.ordered_partition_by_indices,+ )+ .field("cache", &self.cache)+ .field("can_repartition", &self.can_repartition)+ .field(+ "state_observer",+ &self.state_observer.as_ref().map(|_| "..."),+ )+ .finish()+ }}impl BoundedWindowAggExec {@@ -141,9 +206,50 @@ impl BoundedWindowAggExec {ordered_partition_by_indices,cache: Arc::new(cache),can_repartition,+ state_observer: None,})}+ /// Install (or clear) a [`WindowStateObserver`] that receives each+ /// PARTITION BY group's finalized window state at partition close.+ ///+ /// Errors when `observer` is `Some` and any window expression on this+ /// exec has a non-ever-expanding frame (i.e. its start bound is not+ /// `UNBOUNDED PRECEDING`). Those frames use `SlidingAggregateWindowExpr`+ /// under the hood, whose accumulator calls `retract_batch` — at+ /// partition close the accumulator holds only the last frame's rows,+ /// not the partition aggregate, so the observed state would silently+ /// misrepresent the group.+ pub fn with_state_observer(+ mut self,+ observer: Option<Arc<dyn WindowStateObserver>>,+ ) -> Result<Self> {+ if observer.is_some() {+ for expr in &self.window_expr {+ if !expr.get_window_frame().is_ever_expanding() {+ return exec_err!(+ "cannot install WindowStateObserver on BoundedWindowAggExec \+ with a sliding aggregate window frame (start != \+ UNBOUNDED PRECEDING) for `{}`; sliding accumulator state \+ is frame-only, not the partition aggregate",+ expr.name()+ );+ }+ }+ }+ self.state_observer = observer;+ Ok(self)+ }++ /// The currently-installed [`WindowStateObserver`], if any. Optimizer+ /// rules that rebuild this exec via+ /// [`crate::windows::get_best_fitting_window`] or a direct `try_new`+ /// call must read this and reinstall it on the new exec, otherwise a+ /// caller-installed observer is silently dropped by the rewrite.+ pub fn state_observer(&self) -> Option<&Arc<dyn WindowStateObserver>> {+ self.state_observer.as_ref()+ }+/// Window expressionspub fn window_expr(&self) -> &[Arc<dyn WindowExpr>] {&self.window_expr@@ -362,12 +468,14 @@ impl ExecutionPlan for BoundedWindowAggExec {children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {check_if_same_properties!(self, children);- Ok(Arc::new(BoundedWindowAggExec::try_new(+ let new = BoundedWindowAggExec::try_new(self.window_expr.clone(),Arc::clone(&children[0]),self.input_order_mode.clone(),self.can_repartition,- )?))+ )?+ .with_state_observer(self.state_observer.clone())?;+ Ok(Arc::new(new))}fn with_new_children_and_same_properties(@@ -394,6 +502,8 @@ impl ExecutionPlan for BoundedWindowAggExec {input,BaselineMetrics::new(&self.metrics, partition),search_mode,+ partition,+ self.state_observer.clone(),)?);Ok(stream)}@@ -450,6 +560,10 @@ impl ExecutionPlan for BoundedWindowAggExec {// below, since `partition_keys()` returns an empty vec when this is// false and the decoder recovers it as `!partition_keys.is_empty()`.can_repartition: _,+ // Runtime callback installed after planning; not part of the wire+ // format. Any decoder that needs it must reinstall via+ // `with_state_observer`.+ state_observer: _,} = self;let input = ctx.encode_child(input)?;@@ -1080,9 +1194,48 @@ pub struct BoundedWindowAggStream {/// partitions, so finished partitions are pruned eagerly instead and no/// such bound is needed.most_recent_row: Option<RecordBatch>,+ /// Output partition index this stream serves; passed as the first+ /// argument to [`WindowStateObserver::finalize_window_aggregate`].+ partition_idx: usize,+ /// If set, invoked from [`Self::publish_finalized_states`] with the+ /// finalized per-window-expression state for every partition key that is+ /// about to be dropped.+ state_observer: Option<Arc<dyn WindowStateObserver>>,}impl BoundedWindowAggStream {+ /// Fire `observer` once per (window expression, partition key) for every+ /// group whose [`WindowAggState::is_end`] is true. Always mutates when+ /// called: [`datafusion_expr::Accumulator::state`] requires `&mut`, which+ /// propagates up here. The caller is responsible for deciding whether to+ /// fire (i.e. checking whether an observer is installed).+ ///+ /// Exactly-once per group is enforced by [`WindowState::aggregate_state`],+ /// which errors on second call; the `published` early-skip below avoids reaching the error.+ fn publish_finalized_states(+ &mut self,+ observer: &dyn WindowStateObserver,+ ) -> Result<()> {+ let partition_idx = self.partition_idx;+ for (expr_idx, per_expr) in self.window_agg_states.iter_mut().enumerate() {+ let window_expr = &self.window_expr[expr_idx];+ for (key, ws) in per_expr.iter_mut() {+ if ws.published || !ws.state.is_end {+ continue;+ }+ if let Some(state) = ws.aggregate_state()? {+ observer.finalize_window_aggregate(+ partition_idx,+ window_expr,+ key,+ state,+ )?;+ }+ }+ }+ Ok(())+ }+/// Prunes sections of the state that are no longer needed when calculating/// results (as determined by window frame boundaries and number of results generated).// For instance, if first `n` (not necessarily same with `n_out`) elements are no longer needed to@@ -1123,6 +1276,8 @@ impl BoundedWindowAggStream {input: SendableRecordBatchStream,baseline_metrics: BaselineMetrics,search_mode: Box<dyn PartitionSearcher>,+ partition_idx: usize,+ state_observer: Option<Arc<dyn WindowStateObserver>>,) -> Result<Self> {let state = window_expr.iter().map(|_| IndexMap::default()).collect();let empty_batch = RecordBatch::new_empty(Arc::clone(&schema));@@ -1137,6 +1292,8 @@ impl BoundedWindowAggStream {baseline_metrics,search_mode,most_recent_row: None,+ partition_idx,+ state_observer,})}@@ -1154,6 +1311,14 @@ impl BoundedWindowAggStream {)?;}+ // Fire before `calculate_out_columns`: on causal frames every row+ // already streamed out, so at EOS that call returns `None` and the+ // prune path is skipped — the final partition would otherwise be+ // dropped unobserved.+ if let Some(observer) = self.state_observer.clone() {+ self.publish_finalized_states(observer.as_ref())?;+ }+let schema = Arc::clone(&self.schema);let window_expr_out = self.search_mode.calculate_out_columns(&self.input_buffer,@@ -1412,10 +1577,11 @@ mod tests {use crate::projection::{ProjectionExec, ProjectionExpr};use crate::streaming::{PartitionStream, StreamingTableExec};use crate::test::TestMemoryExec;+ use crate::windows::bounded_window_agg_exec::WindowStateObserver;use crate::windows::{BoundedWindowAggExec, InputOrderMode, create_udwf_window_expr, create_window_expr,};- use crate::{ExecutionPlan, displayable, execute_stream};+ use crate::{ExecutionPlan, WindowExpr, displayable, execute_stream};use arrow::array::{RecordBatch,@@ -1436,7 +1602,7 @@ mod tests {use datafusion_functions_window::nth_value::last_value_udwf;use datafusion_functions_window::nth_value::nth_value_udwf;use datafusion_physical_expr::expressions::{Column, Literal, col};- use datafusion_physical_expr::window::StandardWindowExpr;+ use datafusion_physical_expr::window::{PartitionKey, StandardWindowExpr};use datafusion_physical_expr::{LexOrdering, PhysicalExpr};use futures::future::Shared;@@ -1993,6 +2159,629 @@ mod tests {Ok(())}+ type Observation = (usize, PartitionKey, Vec<ScalarValue>);++ /// Test [`WindowStateObserver`] that records every callback into a shared+ /// `Vec` for later assertion.+ struct RecordingObserver {+ sink: Arc<std::sync::Mutex<Vec<Observation>>>,+ }++ impl WindowStateObserver for RecordingObserver {+ fn finalize_window_aggregate(+ &self,+ partition_idx: usize,+ _window_expr: &Arc<dyn WindowExpr>,+ partition_key: &PartitionKey,+ state: Vec<ScalarValue>,+ ) -> Result<()> {+ self.sink+ .lock()+ .unwrap()+ .push((partition_idx, partition_key.clone(), state));+ Ok(())+ }+ }++ /// Build a `BoundedWindowAggExec` for `count(sn) OVER (PARTITION BY hash+ /// ORDER BY sn <frame>)` over a fixed two-group source (hash=1 × 3,+ /// hash=2 × 3, sorted by (hash, sn)). Returns the plan pre-observer so+ /// callers can decide how to install it.+ fn build_partition_close_plan(frame: WindowFrame) -> Result<BoundedWindowAggExec> {+ let schema = test_schema();++ let mut sn_b = UInt64Builder::with_capacity(6);+ let mut hash_b = Int64Builder::with_capacity(6);+ for (sn, hash) in [(1u64, 1i64), (2, 1), (3, 1), (4, 2), (5, 2), (6, 2)] {+ sn_b.append_value(sn);+ hash_b.append_value(hash);+ }+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(sn_b.finish()), Arc::new(hash_b.finish())],+ )?;+ let ordering: LexOrdering = [+ PhysicalSortExpr {+ expr: col("hash", &schema)?,+ options: SortOptions::default(),+ },+ PhysicalSortExpr {+ expr: col("sn", &schema)?,+ options: SortOptions::default(),+ },+ ]+ .into();+ let source_raw =+ TestMemoryExec::try_new(&[vec![batch]], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![ordering])?;+ let source: Arc<dyn ExecutionPlan> =+ Arc::new(TestMemoryExec::update_cache(&Arc::new(source_raw)));++ let expr = create_window_expr(+ &WindowFunctionDefinition::AggregateUDF(count_udaf()),+ "cnt".to_string(),+ &[col("sn", &schema)?],+ &[col("hash", &schema)?],+ &[PhysicalSortExpr {+ expr: col("sn", &schema)?,+ options: SortOptions::default(),+ }],+ Arc::new(frame),+ source.schema(),+ false,+ false,+ None,+ )?;++ BoundedWindowAggExec::try_new(vec![expr], source, InputOrderMode::Sorted, false)+ }++ // Two PARTITION BY groups: hash=1 [sn=1,2,3] then hash=2 [sn=4,5,6].+ // Input is sorted by (hash, sn) so we can run in Sorted mode; in that+ // mode `mark_partition_end` closes the leading group mid-stream and+ // EOS closes the tail — both fire the observer for an ever-expanding+ // frame. Sliding frames are rejected at install time.++ #[tokio::test]+ async fn test_state_observer_rejects_sliding_frame() -> Result<()> {+ // `CURRENT ROW → UNBOUNDED FOLLOWING` is not ever-expanding, so this+ // maps to `SlidingAggregateWindowExpr` whose accumulator retracts as+ // rows leave the frame — at partition close the accumulator holds+ // only the last frame's rows, not the partition aggregate.+ // `with_state_observer` refuses this configuration.+ use std::sync::Mutex;++ let plan = build_partition_close_plan(WindowFrame::new_bounds(+ WindowFrameUnits::Rows,+ WindowFrameBound::CurrentRow,+ WindowFrameBound::Following(ScalarValue::UInt64(None)),+ ))?;+ let observer: Arc<dyn WindowStateObserver> = Arc::new(RecordingObserver {+ sink: Arc::new(Mutex::new(vec![])),+ });+ let err = plan.with_state_observer(Some(observer)).unwrap_err();+ let msg = err.to_string();+ assert!(+ msg.contains("sliding aggregate window frame"),+ "expected sliding-frame rejection, got: {msg}"+ );+ Ok(())+ }++ #[tokio::test]+ async fn test_finalized_state_observer_fires_on_causal_frame() -> Result<()> {+ // `ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW` — ever-expanding,+ // `PlainAggregateWindowExpr` under the hood. At partition close the+ // accumulator holds the partition aggregate. Both mid-stream close+ // (hash=1 as hash=2 rows arrive) and EOS (hash=2 at drain) fire.+ use std::sync::Mutex;++ let task_ctx = Arc::new(TaskContext::default());+ let plan = build_partition_close_plan(WindowFrame::new_bounds(+ WindowFrameUnits::Rows,+ WindowFrameBound::Preceding(ScalarValue::UInt64(None)),+ WindowFrameBound::CurrentRow,+ ))?;++ let observations: Arc<Mutex<Vec<Observation>>> = Arc::new(Mutex::new(vec![]));+ let observer: Arc<dyn WindowStateObserver> = Arc::new(RecordingObserver {+ sink: Arc::clone(&observations),+ });+ let plan = plan.with_state_observer(Some(observer))?;++ let _ = collect(Arc::new(plan).execute(0, task_ctx)?).await?;++ // count(sn) over each of hash=1 (3 rows) and hash=2 (3 rows), in+ // close order — hash=1 first (mid-stream close), hash=2 second (EOS).+ let observed: Vec<(usize, i64, Vec<ScalarValue>)> = observations+ .lock()+ .unwrap()+ .iter()+ .map(|(idx, key, state)| {+ let hash = match &key[0] {+ ScalarValue::Int64(Some(v)) => *v,+ other => panic!("unexpected partition-key element: {other:?}"),+ };+ (*idx, hash, state.clone())+ })+ .collect();+ assert_eq!(+ observed,+ vec![+ (0, 1, vec![ScalarValue::Int64(Some(3))]),+ (0, 2, vec![ScalarValue::Int64(Some(3))]),+ ]+ );+ Ok(())+ }++ #[tokio::test]+ async fn test_finalized_state_observer_fires_exactly_once_across_batches()+ -> Result<()> {+ // Regression guard for the exactly-once observer contract when+ // partition close and pruning happen on different `compute_aggregates`+ // calls.+ //+ // The observer fires from `publish_finalized_states`, called at the+ // top of every `compute_aggregates`. Entries are only cleared by+ // `prune_state`, which runs only when `calculate_out_columns` returns+ // `Some`. Nothing in the type system ties the two together, so a+ // group whose state was published on batch N must not be re-published+ // on batch N+1 or at EOS.+ //+ // Layout: three PARTITION BY groups streamed across two input+ // batches, so each group closes on a distinct `compute_aggregates`+ // call:+ // batch 1 = [hash=1 × 2] — no close (single group).+ // batch 2 = [hash=2 × 2, hash=3 × 2] — `mark_partition_end`+ // closes hash=1 and hash=2.+ // EOS — closes hash=3.+ //+ // Assertion: each key appears exactly once across all observations.+ use std::sync::Mutex;++ let task_ctx = Arc::new(TaskContext::default());+ let schema = test_schema();++ // Two batches, same output partition.+ let make_batch = |rows: &[(u64, i64)]| -> Result<RecordBatch> {+ let mut sn_b = UInt64Builder::with_capacity(rows.len());+ let mut hash_b = Int64Builder::with_capacity(rows.len());+ for &(sn, hash) in rows {+ sn_b.append_value(sn);+ hash_b.append_value(hash);+ }+ Ok(RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(sn_b.finish()), Arc::new(hash_b.finish())],+ )?)+ };+ let batch1 = make_batch(&[(1, 1), (2, 1)])?;+ let batch2 = make_batch(&[(3, 2), (4, 2), (5, 3), (6, 3)])?;++ let ordering: LexOrdering = [+ PhysicalSortExpr {+ expr: col("hash", &schema)?,+ options: SortOptions::default(),+ },+ PhysicalSortExpr {+ expr: col("sn", &schema)?,+ options: SortOptions::default(),+ },+ ]+ .into();+ let source_raw =+ TestMemoryExec::try_new(&[vec![batch1, batch2]], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![ordering])?;+ let source: Arc<dyn ExecutionPlan> =+ Arc::new(TestMemoryExec::update_cache(&Arc::new(source_raw)));++ let expr = create_window_expr(+ &WindowFunctionDefinition::AggregateUDF(count_udaf()),+ "cnt".to_string(),+ &[col("sn", &schema)?],+ &[col("hash", &schema)?],+ &[PhysicalSortExpr {+ expr: col("sn", &schema)?,+ options: SortOptions::default(),+ }],+ Arc::new(WindowFrame::new_bounds(+ WindowFrameUnits::Rows,+ WindowFrameBound::Preceding(ScalarValue::UInt64(None)),+ WindowFrameBound::CurrentRow,+ )),+ source.schema(),+ false,+ false,+ None,+ )?;++ let observations: Arc<Mutex<Vec<Observation>>> = Arc::new(Mutex::new(vec![]));+ let observer: Arc<dyn WindowStateObserver> = Arc::new(RecordingObserver {+ sink: Arc::clone(&observations),+ });++ let plan = BoundedWindowAggExec::try_new(+ vec![expr],+ source,+ InputOrderMode::Sorted,+ false,+ )?+ .with_state_observer(Some(observer))?;++ let _ = collect(Arc::new(plan).execute(0, task_ctx)?).await?;++ let fired: Vec<i64> = observations+ .lock()+ .unwrap()+ .iter()+ .map(|(_, key, _)| match &key[0] {+ ScalarValue::Int64(Some(v)) => *v,+ other => panic!("unexpected partition-key element: {other:?}"),+ })+ .collect();+ // Each group closes on a distinct `compute_aggregates` call — hash=1+ // and hash=2 on batch 2's `mark_partition_end`, hash=3 at EOS — and+ // each appears exactly once, in close order.+ assert_eq!(fired, vec![1, 2, 3]);+ Ok(())+ }++ /// Run one task's local BWAG for `SUM(sn) OVER (ORDER BY sn ROWS+ /// UNBOUNDED PRECEDING TO CURRENT ROW)` with no PARTITION BY, over+ /// `input` sorted ascending. Returns the per-row output values and the+ /// observed finalized state total (which the caller uses as a carry-in+ /// for the next task).+ async fn run_running_sum_task(+ input: &[u64],+ task_ctx: Arc<TaskContext>,+ ) -> Result<(Vec<u64>, u64)> {+ use arrow::array::UInt64Array;+ use datafusion_functions_aggregate::sum::sum_udaf;+ use std::sync::Mutex;++ /// Observer for `run_running_sum_task`: captures the single running+ /// SUM total published at EOS. Asserts exactly-one fire and rejects+ /// non-empty partition keys (this helper is no-PARTITION-BY only).+ struct RunningSumObserver {+ sink: Arc<Mutex<Option<u64>>>,+ }++ impl WindowStateObserver for RunningSumObserver {+ fn finalize_window_aggregate(+ &self,+ _partition_idx: usize,+ _window_expr: &Arc<dyn WindowExpr>,+ partition_key: &PartitionKey,+ state: Vec<ScalarValue>,+ ) -> Result<()> {+ assert!(+ partition_key.is_empty(),+ "empty PartitionKey for no-PARTITION-BY plan"+ );+ let total = match &state[0] {+ ScalarValue::UInt64(Some(v)) => *v,+ ScalarValue::Int64(Some(v)) => *v as u64,+ other => panic!("unexpected sum state element: {other:?}"),+ };+ let prev = self.sink.lock().unwrap().replace(total);+ assert!(prev.is_none(), "observer must fire exactly once per task");+ Ok(())+ }+ }++ let schema = test_schema();+ let mut sn_b = UInt64Builder::with_capacity(input.len());+ let mut hash_b = Int64Builder::with_capacity(input.len());+ for &sn in input {+ sn_b.append_value(sn);+ hash_b.append_value(0);+ }+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(sn_b.finish()), Arc::new(hash_b.finish())],+ )?;+ let ordering: LexOrdering = [PhysicalSortExpr {+ expr: col("sn", &schema)?,+ options: SortOptions::default(),+ }]+ .into();+ let source_raw =+ TestMemoryExec::try_new(&[vec![batch]], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![ordering])?;+ let source: Arc<dyn ExecutionPlan> =+ Arc::new(TestMemoryExec::update_cache(&Arc::new(source_raw)));++ let window_fn = WindowFunctionDefinition::AggregateUDF(sum_udaf());+ let args = vec![col("sn", &schema)?];+ let partition_by: Vec<Arc<dyn PhysicalExpr>> = vec![];+ let order_by = vec![PhysicalSortExpr {+ expr: col("sn", &schema)?,+ options: SortOptions::default(),+ }];+ let frame = WindowFrame::new_bounds(+ WindowFrameUnits::Rows,+ WindowFrameBound::Preceding(ScalarValue::UInt64(None)),+ WindowFrameBound::CurrentRow,+ );+ let expr = create_window_expr(+ &window_fn,+ "running_sum".to_string(),+ &args,+ &partition_by,+ &order_by,+ Arc::new(frame),+ source.schema(),+ false,+ false,+ None,+ )?;++ let total_sink: Arc<Mutex<Option<u64>>> = Arc::new(Mutex::new(None));+ let observer: Arc<dyn WindowStateObserver> = Arc::new(RunningSumObserver {+ sink: Arc::clone(&total_sink),+ });++ let plan = BoundedWindowAggExec::try_new(+ vec![expr],+ source,+ InputOrderMode::Sorted,+ false,+ )?+ .with_state_observer(Some(observer))?;+ let batches = collect(Arc::new(plan).execute(0, task_ctx)?).await?;++ let mut out = Vec::with_capacity(input.len());+ for batch in &batches {+ let col = batch+ .column_by_name("running_sum")+ .expect("running_sum column present");+ let arr = col+ .as_any()+ .downcast_ref::<UInt64Array>()+ .expect("SUM(UInt64) → UInt64Array");+ for i in 0..arr.len() {+ out.push(arr.value(i));+ }+ }+ let total = total_sink+ .lock()+ .unwrap()+ .expect("observer must have fired at EOS");+ Ok((out, total))+ }++ /// Run one task's local BWAG for `approx_distinct(sn) OVER (ORDER BY sn+ /// ROWS UNBOUNDED PRECEDING TO CURRENT ROW)` with no PARTITION BY, and+ /// return the single EOS-observed [`Accumulator::state`] Vec.+ async fn run_approx_distinct_task(+ input: &[u64],+ task_ctx: Arc<TaskContext>,+ ) -> Result<Vec<ScalarValue>> {+ use datafusion_functions_aggregate::approx_distinct::approx_distinct_udaf;+ use std::sync::Mutex;++ /// Observer for `run_approx_distinct_task`: capture the single EOS+ /// state. Asserts exactly-one fire and rejects non-empty partition+ /// keys (helper is no-PARTITION-BY only).+ struct ApproxDistinctObserver {+ sink: Arc<Mutex<Option<Vec<ScalarValue>>>>,+ }++ impl WindowStateObserver for ApproxDistinctObserver {+ fn finalize_window_aggregate(+ &self,+ _partition_idx: usize,+ _window_expr: &Arc<dyn WindowExpr>,+ partition_key: &PartitionKey,+ state: Vec<ScalarValue>,+ ) -> Result<()> {+ assert!(+ partition_key.is_empty(),+ "empty PartitionKey for no-PARTITION-BY plan"+ );+ let prev = self.sink.lock().unwrap().replace(state);+ assert!(prev.is_none(), "observer must fire exactly once per task");+ Ok(())+ }+ }++ let schema = test_schema();+ let mut sn_b = UInt64Builder::with_capacity(input.len());+ let mut hash_b = Int64Builder::with_capacity(input.len());+ for &sn in input {+ sn_b.append_value(sn);+ hash_b.append_value(0);+ }+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(sn_b.finish()), Arc::new(hash_b.finish())],+ )?;+ let ordering: LexOrdering = [PhysicalSortExpr {+ expr: col("sn", &schema)?,+ options: SortOptions::default(),+ }]+ .into();+ let source_raw =+ TestMemoryExec::try_new(&[vec![batch]], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![ordering])?;+ let source: Arc<dyn ExecutionPlan> =+ Arc::new(TestMemoryExec::update_cache(&Arc::new(source_raw)));++ let expr = create_window_expr(+ &WindowFunctionDefinition::AggregateUDF(approx_distinct_udaf()),+ "approx_distinct_sn".to_string(),+ &[col("sn", &schema)?],+ &[],+ &[PhysicalSortExpr {+ expr: col("sn", &schema)?,+ options: SortOptions::default(),+ }],+ Arc::new(WindowFrame::new_bounds(+ WindowFrameUnits::Rows,+ WindowFrameBound::Preceding(ScalarValue::UInt64(None)),+ WindowFrameBound::CurrentRow,+ )),+ source.schema(),+ false,+ false,+ None,+ )?;++ let state_sink: Arc<Mutex<Option<Vec<ScalarValue>>>> = Arc::new(Mutex::new(None));+ let observer: Arc<dyn WindowStateObserver> = Arc::new(ApproxDistinctObserver {+ sink: Arc::clone(&state_sink),+ });++ let plan = BoundedWindowAggExec::try_new(+ vec![expr],+ source,+ InputOrderMode::Sorted,+ false,+ )?+ .with_state_observer(Some(observer))?;+ let _ = collect(Arc::new(plan).execute(0, task_ctx)?).await?;++ state_sink+ .lock()+ .unwrap()+ .take()+ .ok_or_else(|| exec_datafusion_err!("observer never fired"))+ }++ #[tokio::test]+ async fn test_prefix_scan_across_tasks_matches_single_bwag() -> Result<()> {+ // Demonstrates the parallel-window shape reviewers asked about:+ // range-shuffle `SUM(sn) OVER (ORDER BY sn UNBOUNDED PRECEDING TO+ // CURRENT ROW)` across two tasks, then prefix-scan each task's+ // finalized state (from the observer) to carry-in the next task's+ // rows. Result must match a single BWAG over the concatenated input.+ let task_ctx = Arc::new(TaskContext::default());++ // Two tasks under range partition on sn:+ let (task1_out, task1_total) =+ run_running_sum_task(&[1, 1, 2, 2, 3, 3, 4, 4], Arc::clone(&task_ctx))+ .await?;+ let (task2_out, task2_total) =+ run_running_sum_task(&[5, 5, 6, 6, 7, 7, 8, 8], Arc::clone(&task_ctx))+ .await?;++ // Local (uncorrected) outputs and totals — first pass.+ assert_eq!(task1_out, vec![1, 2, 4, 6, 9, 12, 16, 20]);+ assert_eq!(task1_total, 20);+ assert_eq!(task2_out, vec![5, 10, 16, 22, 29, 36, 44, 52]);+ assert_eq!(task2_total, 52);++ // Prefix scan over per-task totals → carry-in for each task. Task 0's+ // carry-in is 0; task N's carry-in is the sum of tasks [0, N).+ let carry_ins = [0u64, task1_total];++ // Second pass: shift each task's local values by its carry-in.+ let task1_final: Vec<u64> = task1_out.iter().map(|v| v + carry_ins[0]).collect();+ let task2_final: Vec<u64> = task2_out.iter().map(|v| v + carry_ins[1]).collect();+ let parallel_result: Vec<u64> = task1_final+ .iter()+ .chain(task2_final.iter())+ .copied()+ .collect();++ // Oracle: single BWAG over the full concatenated input.+ let (single_result, single_total) = run_running_sum_task(+ &[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8],+ task_ctx,+ )+ .await?;++ assert_eq!(+ parallel_result, single_result,+ "two-task prefix-scan must match single-BWAG oracle"+ );+ // And matches the sequence in the design discussion.+ assert_eq!(+ single_result,+ vec![1, 2, 4, 6, 9, 12, 16, 20, 25, 30, 36, 42, 49, 56, 64, 72]+ );+ assert_eq!(single_total, 72);+ Ok(())+ }++ #[tokio::test]+ async fn test_prefix_merge_across_tasks_approx_distinct() -> Result<()> {+ // Load-bearing contract for the parallel-window use case: the state+ // exposed by `WindowStateObserver::finalize_window_aggregate` must be+ // compatible with `Accumulator::merge_batch` on a fresh accumulator+ // of the same UDAF. This is what allows non-decomposable aggregates+ // like `approx_distinct` (HLL sketch state) to be prefix-merged+ // across shard tasks — the reason we exposed accumulator state at+ // all. If this ever breaks, downstream parallel-window work has to+ // wait for a public API change.+ use arrow::array::{ArrayRef, BinaryArray};+ use arrow::datatypes::FieldRef;+ use datafusion_expr::function::AccumulatorArgs;+ use datafusion_functions_aggregate::approx_distinct::approx_distinct_udaf;++ let task_ctx = Arc::new(TaskContext::default());++ // Two tasks with overlapping inputs; concatenated distinct universe+ // is {1,2,3,4,5}.+ let state1 =+ run_approx_distinct_task(&[1, 1, 2, 3], Arc::clone(&task_ctx)).await?;+ let state2 = run_approx_distinct_task(&[3, 4, 5], Arc::clone(&task_ctx)).await?;+ let state_single =+ run_approx_distinct_task(&[1, 1, 2, 3, 3, 4, 5], Arc::clone(&task_ctx))+ .await?;++ // approx_distinct state is a single serialized-HLL Binary field.+ assert_eq!(state1.len(), 1, "single state field");+ assert_eq!(state2.len(), 1, "single state field");+ assert_eq!(state_single.len(), 1, "single state field");++ // Seed a fresh accumulator with the given serialized HLL states via+ // `merge_batch` and return its distinct-count evaluation.+ fn evaluate_merged(states: &[&ScalarValue]) -> Result<ScalarValue> {+ let udaf = approx_distinct_udaf();+ let input_schema =+ Arc::new(Schema::new(vec![Field::new("sn", DataType::UInt64, true)]));+ let return_field: FieldRef =+ Arc::new(Field::new("approx_distinct_sn", DataType::UInt64, true));+ let expr_field: FieldRef = Arc::new(Field::new("sn", DataType::UInt64, true));+ let physical_col: Arc<dyn PhysicalExpr> = col("sn", &input_schema)?;+ let args = AccumulatorArgs {+ return_field: Arc::clone(&return_field),+ schema: &input_schema,+ ignore_nulls: false,+ order_bys: &[],+ is_reversed: false,+ name: "approx_distinct",+ is_distinct: false,+ exprs: std::slice::from_ref(&physical_col),+ expr_fields: std::slice::from_ref(&expr_field),+ };+ let mut acc = udaf.accumulator(args)?;+ let byte_slices: Vec<&[u8]> = states+ .iter()+ .map(|s| match s {+ ScalarValue::Binary(Some(v)) => v.as_slice(),+ other => panic!("expected Binary state, got {other:?}"),+ })+ .collect();+ let bin: ArrayRef = Arc::new(BinaryArray::from_iter_values(byte_slices));+ acc.merge_batch(std::slice::from_ref(&bin))?;+ acc.evaluate()+ }++ let merged = evaluate_merged(&[&state1[0], &state2[0]])?;+ let oracle = evaluate_merged(&[&state_single[0]])?;++ assert_eq!(+ merged, oracle,+ "merged task states must match single-BWAG oracle — parallel prefix-merge contract"+ );+ // HLL is approximate but exact for a 5-element universe.+ assert_eq!(merged, ScalarValue::UInt64(Some(5)));+ Ok(())+ }+#[test]fn test_bounded_window_agg_cardinality_effect() -> Result<()> {let schema = test_schema();diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rsindex baa6abd839..089bdc23ee 100644--- a/datafusion/physical-plan/src/windows/mod.rs+++ b/datafusion/physical-plan/src/windows/mod.rs@@ -54,7 +54,7 @@ use datafusion_physical_expr_common::sort_expr::{use itertools::Itertools;// Public interface:-pub use bounded_window_agg_exec::BoundedWindowAggExec;+pub use bounded_window_agg_exec::{BoundedWindowAggExec, WindowStateObserver};pub use datafusion_physical_expr::window::{PlainAggregateWindowExpr, StandardWindowExpr, WindowExpr,};@@ -596,6 +596,12 @@ pub fn get_best_fitting_window(// They are either the same with `window_expr`'s PARTITION BY columns,// or it is empty if partitioning is not desirable for this windowing operator.physical_partition_keys: &[Arc<dyn PhysicalExpr>],+ // A [`WindowStateObserver`] installed on the source+ // [`BoundedWindowAggExec`] (via [`BoundedWindowAggExec::with_state_observer`])+ // that must survive the rebuild. Ignored when the rebuilt exec is a+ // [`WindowAggExec`], which does not carry an observer. `None` when the+ // source is a [`WindowAggExec`] or has no observer installed.+ state_observer: Option<Arc<dyn WindowStateObserver>>,) -> Result<Option<Arc<dyn ExecutionPlan>>> {// Contains at least one window expr and all of the partition by and order by sections// of the window_exprs are same.@@ -635,12 +641,15 @@ pub fn get_best_fitting_window(// If all window expressions can run with bounded memory, choose the// bounded window variant:if window_expr.iter().all(|e| e.uses_bounded_memory()) {- Ok(Some(Arc::new(BoundedWindowAggExec::try_new(- window_expr,- Arc::clone(input),- input_order_mode,- !physical_partition_keys.is_empty(),- )?) as _))+ Ok(Some(Arc::new(+ BoundedWindowAggExec::try_new(+ window_expr,+ Arc::clone(input),+ input_order_mode,+ !physical_partition_keys.is_empty(),+ )?+ .with_state_observer(state_observer)?,+ ) as _))} else if input_order_mode != InputOrderMode::Sorted {// For `WindowAggExec` to work correctly PARTITION BY columns should be sorted.// Hence, if `input_order_mode` is not `Sorted` we should convert@@ -939,6 +948,79 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn get_best_fitting_window_preserves_state_observer() -> Result<()> {+ // `EnforceSorting`/`EnforceDistribution` call `get_best_fitting_window`+ // on a source `BoundedWindowAggExec` and replace it with the returned+ // exec. Without observer propagation, a `WindowStateObserver`+ // installed on the source is silently dropped by the rebuild.+ use datafusion_common::ScalarValue;+ use datafusion_expr::{WindowFrameBound, WindowFrameUnits};++ struct NoopObserver;+ impl WindowStateObserver for NoopObserver {+ fn finalize_window_aggregate(+ &self,+ _partition_idx: usize,+ _window_expr: &Arc<dyn WindowExpr>,+ _partition_key: &datafusion_physical_expr::window::PartitionKey,+ _state: Vec<ScalarValue>,+ ) -> Result<()> {+ Ok(())+ }+ }++ let schema = create_test_schema()?;+ let sort = sort_expr("nullable_col", &schema);+ let ordering: LexOrdering = [sort.clone()].into();+ let source = streaming_table_exec(&schema, ordering, false)?;++ let expr = create_window_expr(+ &WindowFunctionDefinition::AggregateUDF(count_udaf()),+ "cnt".to_string(),+ &[col("nullable_col", &schema)?],+ &[],+ &[sort],+ Arc::new(WindowFrame::new_bounds(+ WindowFrameUnits::Rows,+ WindowFrameBound::Preceding(ScalarValue::UInt64(None)),+ WindowFrameBound::CurrentRow,+ )),+ source.schema(),+ false,+ false,+ None,+ )?;++ let observer: Arc<dyn WindowStateObserver> = Arc::new(NoopObserver);+ let bounded = BoundedWindowAggExec::try_new(+ vec![expr],+ Arc::clone(&source),+ Sorted,+ false,+ )?+ .with_state_observer(Some(Arc::clone(&observer)))?;++ let rebuilt = get_best_fitting_window(+ bounded.window_expr(),+ bounded.input(),+ &bounded.partition_keys(),+ bounded.state_observer().cloned(),+ )?+ .expect("rebuild should produce a plan");+ let bwag = rebuilt+ .downcast_ref::<BoundedWindowAggExec>()+ .expect("rebuild yielded BoundedWindowAggExec");+ let installed = bwag+ .state_observer()+ .expect("observer preserved through rebuild");+ assert!(+ Arc::ptr_eq(installed, &observer),+ "observer identity preserved through rebuild",+ );+ Ok(())+ }+#[tokio::test]async fn test_satisfy_nullable() -> Result<()> {let schema = create_test_schema()?;