Apache DataFusion SQL Query Engine
refactor: Simplify NLJ re-scans with `ReplayableStreamSource` (#21742)
Yongting Youeca2e007a68d
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex 030fa5bbf9..c686d9d5a0 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -79,7 +79,7 @@ use log::debug;use parking_lot::Mutex;use crate::metrics::SpillMetrics;-use crate::spill::in_progress_spill_file::InProgressSpillFile;+use crate::spill::replayable_spill_input::ReplayableStreamSource;use crate::spill::spill_manager::SpillManager;#[expect(rustdoc::private_intra_doc_links)]@@ -931,16 +931,8 @@ pub(crate) struct SpillStateActive {reservation: MemoryReservation,/// Accumulated left batches for the current chunkpending_batches: Vec<RecordBatch>,- /// SpillManager for right-side spilling- right_spill_manager: SpillManager,- /// In-progress spill file for writing right batches during first pass- right_spill_in_progress: Option<InProgressSpillFile>,- /// Completed right-side spill file (available after first pass)- right_spill_file: Option<RefCountedTempFile>,- /// Max right batch memory size (for read_spill_as_stream)- right_max_batch_memory: usize,- /// Whether this is the first right-side pass (need to spill while reading)- is_first_right_pass: bool,+ /// Right input that spills on the first pass and replays from spill later.+ right_input: ReplayableStreamSource,}pub(crate) struct NestedLoopJoinStream {@@ -960,7 +952,8 @@ pub(crate) struct NestedLoopJoinStream {/// type of the joinpub(crate) join_type: JoinType,/// the probe-side(right) table data of the nested loop join- pub(crate) right_data: SendableRecordBatchStream,+ /// `Option` is used because memory-limited path requires resetting it.+ pub(crate) right_data: Option<SendableRecordBatchStream>,/// the build-side table data of the nested loop joinpub(crate) left_data: OnceFut<JoinLeftData>,/// Projection to construct the output schema from the left and right tables.@@ -1258,7 +1251,7 @@ impl NestedLoopJoinStream {output_schema: Arc::clone(&schema),join_filter: filter,join_type,- right_data,+ right_data: Some(right_data),column_indices,left_data,metrics,@@ -1356,7 +1349,15 @@ impl NestedLoopJoinStream {.register(context.memory_pool());// Create SpillManager for right-side spilling- let right_schema = self.right_data.schema();+ let right_schema = self+ .right_data+ .as_ref()+ .expect("right_data must be present before fallback")+ .schema();+ let right_data = self+ .right_data+ .take()+ .expect("right_data must be present before fallback");let right_spill_manager = SpillManager::new(context.runtime_env(),self.metrics.spill_metrics.clone(),@@ -1370,11 +1371,11 @@ impl NestedLoopJoinStream {left_schema: None,reservation,pending_batches: Vec::new(),- right_spill_manager,- right_spill_in_progress: None,- right_spill_file: None,- right_max_batch_memory: 0,- is_first_right_pass: true,+ right_input: ReplayableStreamSource::new(+ right_data,+ right_spill_manager,+ "NestedLoopJoin right spill",+ ),}));// State stays BufferingLeft — next poll will enter@@ -1571,33 +1572,12 @@ impl NestedLoopJoinStream {self.buffered_left_data = Some(Arc::new(left_data));- // Set up right-side stream for this pass- if !active.is_first_right_pass {- if let Some(file) = active.right_spill_file.as_ref() {- match active.right_spill_manager.read_spill_as_stream(- file.clone(),- Some(active.right_max_batch_memory),- ) {- Ok(stream) => {- self.right_data = stream;- }- Err(e) => {- return ControlFlow::Break(Poll::Ready(Some(Err(e))));- }- }+ match active.right_input.open_pass() {+ Ok(stream) => {+ self.right_data = Some(stream);}- } else {- // First pass: create InProgressSpillFile for right side- match active- .right_spill_manager- .create_in_progress_file("NestedLoopJoin right spill")- {- Ok(file) => {- active.right_spill_in_progress = Some(file);- }- Err(e) => {- return ControlFlow::Break(Poll::Ready(Some(Err(e))));- }+ Err(e) => {+ return ControlFlow::Break(Poll::Ready(Some(Err(e))));}}@@ -1613,7 +1593,12 @@ impl NestedLoopJoinStream {&mut self,cx: &mut std::task::Context<'_>,) -> ControlFlow<Poll<Option<Result<RecordBatch>>>> {- match self.right_data.poll_next_unpin(cx) {+ match self+ .right_data+ .as_mut()+ .expect("right_data must be present while fetching right")+ .poll_next_unpin(cx)+ {Poll::Ready(result) => match result {Some(Ok(right_batch)) => {// Update metrics@@ -1626,19 +1611,6 @@ impl NestedLoopJoinStream {return ControlFlow::Continue(());}- // In memory-limited mode, spill right batch to disk on first pass- if let SpillState::Active(ref mut active) = self.spill_state- && active.is_first_right_pass- && let Some(ref mut spill_file) = active.right_spill_in_progress- {- if let Err(e) = spill_file.append_batch(&right_batch) {- return ControlFlow::Break(Poll::Ready(Some(Err(e))));- }- active.right_max_batch_memory = active- .right_max_batch_memory- .max(right_batch.get_array_memory_size());- }-self.current_right_batch = Some(right_batch);// Prepare right bitmap@@ -1654,29 +1626,6 @@ impl NestedLoopJoinStream {}Some(Err(e)) => ControlFlow::Break(Poll::Ready(Some(Err(e)))),None => {- // Right stream exhausted.- // In memory-limited mode, finalize the spill file after first pass.- if let SpillState::Active(ref mut active) = self.spill_state- && active.is_first_right_pass- {- if let Some(mut spill_in_progress) =- active.right_spill_in_progress.take()- {- match spill_in_progress.finish() {- Ok(Some(file)) => {- active.right_spill_file = Some(file);- }- Ok(None) => {- // No data was spilled (right side was empty)- }- Err(e) => {- return ControlFlow::Break(Poll::Ready(Some(Err(e))));- }- }- }- active.is_first_right_pass = false;- }-self.state = NLJState::EmitLeftUnmatched;ControlFlow::Continue(())}@@ -2257,7 +2206,11 @@ impl NestedLoopJoinStream {}let bitmap_sliced = BooleanArray::new(bitmap_sliced.finish(), None);- let right_schema = self.right_data.schema();+ let right_schema = self+ .right_data+ .as_ref()+ .expect("right_data must be present when building unmatched batch")+ .schema();build_unmatched_batch(&self.output_schema,&left_batch_sliced,diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex 5954da8f59..51e59318e2 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -18,6 +18,7 @@//! Defines the spilling functionspub(crate) mod in_progress_spill_file;+pub(crate) mod replayable_spill_input;pub(crate) mod spill_manager;pub mod spill_pool;diff --git a/datafusion/physical-plan/src/spill/replayable_spill_input.rs b/datafusion/physical-plan/src/spill/replayable_spill_input.rsnew file mode 100644index 0000000000..ddef15a639--- /dev/null+++ b/datafusion/physical-plan/src/spill/replayable_spill_input.rs@@ -0,0 +1,445 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Utility for replaying a one-shot input `RecordBatchStream` through spill.+//!+//! See comments in [`ReplayableStreamSource`] for details.++use std::pin::Pin;+use std::sync::Arc;+use std::task::{Context, Poll};++use arrow::datatypes::SchemaRef;+use arrow::record_batch::RecordBatch;+use datafusion_common::{Result, internal_err};+use datafusion_execution::RecordBatchStream;+use datafusion_execution::SendableRecordBatchStream;+use datafusion_execution::disk_manager::RefCountedTempFile;+use futures::Stream;+use parking_lot::Mutex;++use crate::EmptyRecordBatchStream;+use crate::spill::in_progress_spill_file::InProgressSpillFile;+use crate::spill::spill_manager::SpillManager;++/// Spill-backed replayable stream source.+///+/// [`ReplayableStreamSource`] is constructed from an input stream, usually produced+/// by executing an input `ExecutionPlan`.+///+/// - On the first pass, it evaluates the input stream, produces `RecordBatch`es,+/// caches those batches to a local spill file, and also forwards them to the+/// output.+/// - On subsequent passes, it reads directly from the spill file.+///+/// ```text+/// first pass:+///+/// RecordBatch stream+/// |+/// v+/// [batch] -> output+/// |+/// +----> spill file+///+///+/// later passes:+///+/// spill file+/// |+/// v+/// [batch] -> output+/// ```+///+/// This is useful when an input stream must be replayed and:+/// - Re-evaluation is expensive because the input stream may come from a long+/// and complex pipeline.+/// - The parent operator is under memory pressure and cannot cache the input in+/// memory for replay.+///+/// # Concurrency assumption+/// Passes must be opened and consumed sequentially.+/// Opening another pass before exhausting the current one returns an error.+pub(crate) struct ReplayableStreamSource {+ schema: SchemaRef,+ input: Option<SendableRecordBatchStream>,+ spill_manager: SpillManager,+ request_description: String,+ /// Inner state is owned by either the source or one active stream to ensure+ /// sequential access; see struct docs for the concurrency contract.+ ///+ /// Ownership model:+ /// - No active stream: source owns the state (`source.state = Some(state)`).+ /// - Active stream: the stream owns the state (`source.state = None`).+ state: Arc<Mutex<Option<StateInner>>>,+}++/// Inner state exclusively owned by either [`ReplayableStreamSource`] or one [`ReplayableSpillStream`]+enum StateInner {+ Unopened,+ Replayable(Option<RefCountedTempFile>),+ Poisoned,+}++impl ReplayableStreamSource {+ /// Creates a replayable stream producer over a one-shot input stream.+ ///+ /// It caches the input into a local spill file on the first pass, then+ /// reads directly from that spill file on subsequent passes.+ pub(crate) fn new(+ input: SendableRecordBatchStream,+ spill_manager: SpillManager,+ request_description: impl Into<String>,+ ) -> Self {+ let schema = input.schema();+ Self {+ schema,+ input: Some(input),+ spill_manager,+ request_description: request_description.into(),+ state: Arc::new(Mutex::new(Some(StateInner::Unopened))),+ }+ }++ fn set_state(&self, state: StateInner) {+ *self.state.lock() = Some(state);+ }++ /// Opens the next pass over this input.+ ///+ /// The first call returns a stream that forwards upstream batches while+ /// caching them to spill. Later calls return streams that read directly+ /// from the completed spill file.+ ///+ /// # Note+ /// Subsequent passes MUST be opened only after the previous pass is fully+ /// consumed; otherwise, an error is returned.+ pub(crate) fn open_pass(&mut self) -> Result<SendableRecordBatchStream> {+ let state = self.state.lock().take();+ let Some(state) = state else {+ return internal_err!("ReplayableStreamSource pass is still active");+ };++ match state {+ StateInner::Unopened => {+ let Some(input) = self.input.take() else {+ self.set_state(StateInner::Poisoned);+ return internal_err!(+ "ReplayableStreamSource missing first-pass input"+ );+ };+ let spill_file = match self+ .spill_manager+ .create_in_progress_file(&self.request_description)+ {+ Ok(spill_file) => spill_file,+ Err(e) => {+ self.input = Some(input);+ self.set_state(StateInner::Unopened);+ return Err(e);+ }+ };++ Ok(Box::pin(ReplayableSpillStream::new_first(+ Arc::clone(&self.schema),+ input,+ Arc::clone(&self.state),+ spill_file,+ )))+ }+ StateInner::Poisoned => {+ internal_err!(+ "ReplayableStreamSource first pass did not complete successfully"+ )+ }+ StateInner::Replayable(spill_file) => {+ let replay_state = spill_file.clone();+ match ReplayableSpillStream::new_replay(+ Arc::clone(&self.schema),+ &self.spill_manager,+ Arc::clone(&self.state),+ spill_file,+ ) {+ Ok(stream) => Ok(Box::pin(stream)),+ Err(e) => {+ self.set_state(StateInner::Replayable(replay_state));+ Err(e)+ }+ }+ }+ }+ }+}++/// Makes a one-shot stream replayable using spill caching, keeping replays fast+/// and memory efficient.+///+/// On the first pass, it evaluates and forwards output from `inner` while+/// caching it to a spill file for future replays.+///+/// On later passes, it replays directly from the cached spill file.+///+/// See also [`ReplayableStreamSource`] for details.+struct ReplayableSpillStream {+ schema: SchemaRef,+ shared_state: Arc<Mutex<Option<StateInner>>>,+ held_state: Option<StateInner>,+ spill_file: Option<InProgressSpillFile>,+ inner: SendableRecordBatchStream,+}++impl ReplayableSpillStream {+ fn new_first(+ schema: SchemaRef,+ inner: SendableRecordBatchStream,+ shared_state: Arc<Mutex<Option<StateInner>>>,+ spill_file: InProgressSpillFile,+ ) -> Self {+ Self {+ schema,+ shared_state,+ held_state: Some(StateInner::Unopened),+ spill_file: Some(spill_file),+ inner,+ }+ }++ fn new_replay(+ schema: SchemaRef,+ spill_manager: &SpillManager,+ shared_state: Arc<Mutex<Option<StateInner>>>,+ spill_file: Option<RefCountedTempFile>,+ ) -> Result<Self> {+ let inner = if let Some(file) = spill_file.as_ref() {+ spill_manager.read_spill_as_stream(file.clone(), None)?+ } else {+ Box::pin(EmptyRecordBatchStream::new(Arc::clone(&schema)))+ };++ Ok(Self {+ schema,+ shared_state,+ held_state: Some(StateInner::Replayable(spill_file)),+ spill_file: None,+ inner,+ })+ }++ fn restore_held_state(&mut self) {+ if let Some(state) = self.held_state.take() {+ *self.shared_state.lock() = Some(state);+ }+ }++ fn set_state(&mut self, state: StateInner) {+ if self.held_state.take().is_some() {+ *self.shared_state.lock() = Some(state);+ }+ }++ fn poison(&mut self) {+ self.set_state(StateInner::Poisoned);+ }+}++impl Stream for ReplayableSpillStream {+ type Item = Result<RecordBatch>;++ fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {+ let this = self.get_mut();++ match this.inner.as_mut().poll_next(cx) {+ Poll::Ready(Some(Ok(batch))) => {+ if batch.num_rows() > 0+ && let Some(spill_file) = this.spill_file.as_mut()+ && let Err(e) = spill_file.append_batch(&batch)+ {+ this.spill_file.take();+ this.poison();+ return Poll::Ready(Some(Err(e)));+ }++ Poll::Ready(Some(Ok(batch)))+ }+ Poll::Ready(Some(Err(e))) => {+ this.spill_file.take();+ this.poison();+ Poll::Ready(Some(Err(e)))+ }+ // The stream is exhausted, give the inner state ownership back to `ReplayableStreamSource`+ Poll::Ready(None) => {+ if let Some(spill_file) = this.spill_file.as_mut() {+ match spill_file.finish() {+ Ok(file) => {+ this.spill_file.take();+ this.set_state(StateInner::Replayable(file));+ Poll::Ready(None)+ }+ Err(e) => {+ this.spill_file.take();+ this.poison();+ Poll::Ready(Some(Err(e)))+ }+ }+ } else {+ this.restore_held_state();+ Poll::Ready(None)+ }+ }+ Poll::Pending => Poll::Pending,+ }+ }+}++impl RecordBatchStream for ReplayableSpillStream {+ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.schema)+ }+}++impl Drop for ReplayableSpillStream {+ /// If a stream is dropped before it finishes, poison the state so later+ /// replay attempts fail.+ ///+ /// A partial first pass leaves the spill file incomplete, so replaying it+ /// would be unsafe.+ fn drop(&mut self) {+ if self.held_state.is_some() {+ self.poison();+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::Int64Array;+ use arrow_schema::{DataType, Field, Schema};+ use datafusion_execution::runtime_env::RuntimeEnvBuilder;+ use datafusion_physical_expr_common::metrics::{+ ExecutionPlanMetricsSet, SpillMetrics,+ };+ use futures::{StreamExt, TryStreamExt};++ use crate::stream::RecordBatchStreamAdapter;++ fn build_spill_manager(schema: SchemaRef) -> Result<SpillManager> {+ let runtime = Arc::new(RuntimeEnvBuilder::new().build()?);+ let metrics_set = ExecutionPlanMetricsSet::new();+ let spill_metrics = SpillMetrics::new(&metrics_set, 0);+ Ok(SpillManager::new(runtime, spill_metrics, schema))+ }++ fn build_batch(schema: SchemaRef, values: Vec<i64>) -> Result<RecordBatch> {+ RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(values))])+ .map_err(Into::into)+ }++ #[tokio::test]+ async fn test_replayable_spill_input_replays_completed_first_pass() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let batch1 = build_batch(Arc::clone(&schema), vec![1, 2])?;+ let batch2 = build_batch(Arc::clone(&schema), vec![3, 4])?;++ let input = Box::pin(RecordBatchStreamAdapter::new(+ Arc::clone(&schema),+ futures::stream::iter(vec![Ok(batch1.clone()), Ok(batch2.clone())]),+ ));+ let spill_manager = build_spill_manager(Arc::clone(&schema))?;+ let mut replayable =+ ReplayableStreamSource::new(input, spill_manager, "test replayable spill");++ let pass1 = replayable.open_pass()?;+ let pass1_batches = pass1.try_collect::<Vec<_>>().await?;+ assert_eq!(pass1_batches, vec![batch1.clone(), batch2.clone()]);++ let pass2 = replayable.open_pass()?;+ let pass2_batches = pass2.try_collect::<Vec<_>>().await?;+ assert_eq!(pass2_batches, vec![batch1, batch2]);++ Ok(())+ }++ // Try to open a new pass, when the first pass has not finished.+ // The spill file is only partially written, so an error will be returned.+ #[tokio::test]+ async fn test_replayable_spill_input_poisoned_when_first_pass_dropped() -> Result<()>+ {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let batch1 = build_batch(Arc::clone(&schema), vec![1, 2])?;+ let batch2 = build_batch(Arc::clone(&schema), vec![3, 4])?;++ let input = Box::pin(RecordBatchStreamAdapter::new(+ Arc::clone(&schema),+ futures::stream::iter(vec![Ok(batch1), Ok(batch2)]),+ ));+ let spill_manager = build_spill_manager(Arc::clone(&schema))?;+ let mut replayable =+ ReplayableStreamSource::new(input, spill_manager, "test replayable spill");++ let mut pass1 = replayable.open_pass()?;+ let first = pass1.next().await.transpose()?;+ assert!(first.is_some());+ drop(pass1);++ let err = match replayable.open_pass() {+ Ok(_) => panic!("expected first pass to poison replayable spill input"),+ Err(err) => err.strip_backtrace(),+ };+ assert!(+ err.to_string().contains(+ "ReplayableStreamSource first pass did not complete successfully"+ )+ );++ Ok(())+ }++ // Open a new pass, when the previous pass from spill is still in progress.+ // An error is expected, since it requires sequential access.+ #[tokio::test]+ async fn test_replayable_spill_input_errors_when_replay_pass_in_progress()+ -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let batch1 = build_batch(Arc::clone(&schema), vec![1, 2])?;+ let batch2 = build_batch(Arc::clone(&schema), vec![3, 4])?;++ let input = Box::pin(RecordBatchStreamAdapter::new(+ Arc::clone(&schema),+ futures::stream::iter(vec![Ok(batch1.clone()), Ok(batch2.clone())]),+ ));+ let spill_manager = build_spill_manager(Arc::clone(&schema))?;+ let mut replayable =+ ReplayableStreamSource::new(input, spill_manager, "test replayable spill");++ let pass1 = replayable.open_pass()?;+ let _ = pass1.try_collect::<Vec<_>>().await?;++ let pass2 = replayable.open_pass()?;+ let err = match replayable.open_pass() {+ Ok(_) => panic!("expected open_pass to fail while replay pass is active"),+ Err(err) => err.strip_backtrace(),+ };+ assert!(+ err.to_string()+ .contains("ReplayableStreamSource pass is still active")+ );+ drop(pass2);++ Ok(())+ }+}