Skip to content

Apache DataFusion SQL Query Engine

Dynamic work scheduling in FileStream (#21351)

Andrew Lamb466c3eaecfd9
diff --git a/Cargo.lock b/Cargo.lock
index ca2554e99d..37b18c3048 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1968,6 +1968,7 @@ dependencies = [
"liblzma",
"log",
"object_store",
+ "parking_lot",
"rand 0.9.2",
"tempfile",
"tokio",
diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.toml
index 4027521658..40e2271f45 100644
--- a/datafusion/datasource/Cargo.toml
+++ b/datafusion/datasource/Cargo.toml
@@ -64,6 +64,7 @@ itertools = { workspace = true }
liblzma = { workspace = true, optional = true }
log = { workspace = true }
object_store = { workspace = true }
+parking_lot = { workspace = true }
rand = { workspace = true }
tempfile = { workspace = true, optional = true }
tokio = { workspace = true }
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 3a3277be08..04b74528d5 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -24,7 +24,8 @@ use crate::file_groups::FileGroup;
use crate::{
PartitionedFile, display::FileGroupsDisplay, file::FileSource,
file_compression_type::FileCompressionType, file_stream::FileStreamBuilder,
- source::DataSource, statistics::MinMaxStatistics,
+ file_stream::work_source::SharedWorkSource, source::DataSource,
+ statistics::MinMaxStatistics,
};
use arrow::datatypes::FieldRef;
use arrow::datatypes::{DataType, Schema, SchemaRef};
@@ -38,6 +39,7 @@ use datafusion_execution::{
};
use datafusion_expr::Operator;
+use crate::source::OpenArgs;
use datafusion_physical_expr::expressions::{BinaryExpr, Column};
use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_expr::utils::reassign_expr_columns;
@@ -55,6 +57,7 @@ use datafusion_physical_plan::{
metrics::ExecutionPlanMetricsSet,
};
use log::{debug, warn};
+use std::any::Any;
use std::{fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::Arc};
/// [`FileScanConfig`] represents scanning data from a group of files
@@ -578,6 +581,15 @@ impl DataSource for FileScanConfig {
partition: usize,
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
+ self.open_with_args(OpenArgs::new(partition, context))
+ }
+
+ fn open_with_args(&self, args: OpenArgs) -> Result<SendableRecordBatchStream> {
+ let OpenArgs {
+ partition,
+ context,
+ sibling_state,
+ } = args;
let object_store = context.runtime_env().object_store(&self.object_store_url)?;
let batch_size = self
.batch_size
@@ -587,8 +599,17 @@ impl DataSource for FileScanConfig {
let morselizer = source.create_morselizer(object_store, self, partition)?;
+ // Extract the shared work source from the sibling state if it exists.
+ // This allows multiple sibling streams to steal work from a single
+ // shared queue of unopened files.
+ let shared_work_source = sibling_state
+ .as_ref()
+ .and_then(|state| state.downcast_ref::<SharedWorkSource>())
+ .cloned();
+
let stream = FileStreamBuilder::new(self)
.with_partition(partition)
+ .with_shared_work_source(shared_work_source)
.with_morselizer(morselizer)
.with_metrics(source.metrics())
.build()?;
@@ -985,6 +1006,20 @@ impl DataSource for FileScanConfig {
// Delegate to the file source
self.file_source.apply_expressions(f)
}
+
+ /// Create any shared state that should be passed between sibling streams
+ /// during one execution.
+ ///
+ /// This returns `None` when sibling streams must not share work, such as
+ /// when file order must be preserved or the file groups define the output
+ /// partitioning needed for the rest of the plan
+ fn create_sibling_state(&self) -> Option<Arc<dyn Any + Send + Sync>> {
+ if self.preserve_order || self.partitioned_by_file_group {
+ return None;
+ }
+
+ Some(Arc::new(SharedWorkSource::from_config(self)) as Arc<dyn Any + Send + Sync>)
+ }
}
impl FileScanConfig {
@@ -1362,19 +1397,33 @@ mod tests {
use super::*;
use crate::TableSchema;
+ use crate::source::DataSourceExec;
use crate::test_util::col;
use crate::{
generate_test_files, test_util::MockSource, tests::aggr_test_schema,
verify_sort_integrity,
};
+ use arrow::array::{Int32Array, RecordBatch};
use arrow::datatypes::Field;
use datafusion_common::ColumnStatistics;
use datafusion_common::stats::Precision;
+ use datafusion_common::tree_node::TreeNodeRecursion;
+ use datafusion_common::{Result, assert_batches_eq, internal_err};
+ use datafusion_execution::TaskContext;
use datafusion_expr::SortExpr;
+ use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::create_physical_sort_expr;
use datafusion_physical_expr::expressions::Literal;
use datafusion_physical_expr::projection::ProjectionExpr;
+ use datafusion_physical_expr::projection::ProjectionExprs;
+ use datafusion_physical_plan::ExecutionPlan;
+ use datafusion_physical_plan::execution_plan::collect;
+ use futures::FutureExt as _;
+ use futures::StreamExt as _;
+ use futures::stream;
+ use object_store::ObjectStore;
+ use std::fmt::Debug;
#[derive(Clone)]
struct InexactSortPushdownSource {
@@ -1394,7 +1443,7 @@ mod tests {
impl FileSource for InexactSortPushdownSource {
fn create_file_opener(
&self,
- _object_store: Arc<dyn object_store::ObjectStore>,
+ _object_store: Arc<dyn ObjectStore>,
_base_config: &FileScanConfig,
_partition: usize,
) -> Result<Arc<dyn crate::file_stream::FileOpener>> {
@@ -2278,6 +2327,88 @@ mod tests {
assert_eq!(partition_stats.total_byte_size, Precision::Exact(800));
}
+ /// Regression test for reusing a `DataSourceExec` after its execution-local
+ /// shared work queue has been drained.
+ ///
+ /// This test uses a single file group with two files so the scan creates a
+ /// shared unopened-file queue. Executing after `reset_state` must recreate
+ /// the shared queue and return the same rows again.
+ #[tokio::test]
+ async fn reset_state_recreates_shared_work_source() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "value",
+ DataType::Int32,
+ false,
+ )]));
+ let file_source = Arc::new(
+ MockSource::new(Arc::clone(&schema))
+ .with_file_opener(Arc::new(ResetStateTestFileOpener { schema })),
+ );
+
+ let config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_group(FileGroup::new(vec![
+ PartitionedFile::new("file1.parquet", 100),
+ PartitionedFile::new("file2.parquet", 100),
+ ]))
+ .build();
+
+ let exec: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(config);
+ let task_ctx = Arc::new(TaskContext::default());
+
+ // Running the same scan after resetting the state, should
+ // produce the same answer.
+ let first_run = collect(Arc::clone(&exec), Arc::clone(&task_ctx)).await?;
+ let reset_exec = exec.reset_state()?;
+ let second_run = collect(reset_exec, task_ctx).await?;
+
+ let expected = [
+ "+-------+",
+ "| value |",
+ "+-------+",
+ "| 1 |",
+ "| 2 |",
+ "+-------+",
+ ];
+ assert_batches_eq!(expected, &first_run);
+ assert_batches_eq!(expected, &second_run);
+
+ Ok(())
+ }
+
+ /// Test-only `FileOpener` that turns file names like `file1.parquet` into a
+ /// single-batch stream containing that numeric value
+ #[derive(Debug)]
+ struct ResetStateTestFileOpener {
+ schema: SchemaRef,
+ }
+
+ impl crate::file_stream::FileOpener for ResetStateTestFileOpener {
+ fn open(
+ &self,
+ file: PartitionedFile,
+ ) -> Result<crate::file_stream::FileOpenFuture> {
+ let value = file
+ .object_meta
+ .location
+ .as_ref()
+ .trim_start_matches("file")
+ .trim_end_matches(".parquet")
+ .parse::<i32>()
+ .expect("invalid test file name");
+ let schema = Arc::clone(&self.schema);
+ Ok(async move {
+ let batch = RecordBatch::try_new(
+ schema,
+ vec![Arc::new(Int32Array::from(vec![value]))],
+ )
+ .expect("test batch should be valid");
+ Ok(stream::iter(vec![Ok(batch)]).boxed())
+ }
+ .boxed())
+ }
+ }
+
#[test]
fn test_output_partitioning_not_partitioned_by_file_group() {
let file_schema = aggr_test_schema();
@@ -2461,7 +2592,7 @@ mod tests {
impl FileSource for ExactSortPushdownSource {
fn create_file_opener(
&self,
- _object_store: Arc<dyn object_store::ObjectStore>,
+ _object_store: Arc<dyn ObjectStore>,
_base_config: &FileScanConfig,
_partition: usize,
) -> Result<Arc<dyn crate::file_stream::FileOpener>> {
diff --git a/datafusion/datasource/src/file_stream/builder.rs b/datafusion/datasource/src/file_stream/builder.rs
index efe9c39ce3..7034e90255 100644
--- a/datafusion/datasource/src/file_stream/builder.rs
+++ b/datafusion/datasource/src/file_stream/builder.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
use crate::file_scan_config::FileScanConfig;
use crate::file_stream::scan_state::ScanState;
+use crate::file_stream::work_source::{SharedWorkSource, WorkSource};
use crate::morsel::{FileOpenerMorselizer, Morselizer};
use datafusion_common::{Result, internal_err};
use datafusion_physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet};
@@ -33,10 +34,11 @@ pub struct FileStreamBuilder<'a> {
morselizer: Option<Box<dyn Morselizer>>,
metrics: Option<&'a ExecutionPlanMetricsSet>,
on_error: OnError,
+ shared_work_source: Option<SharedWorkSource>,
}
impl<'a> FileStreamBuilder<'a> {
- /// Create a new builder.
+ /// Create a new builder for [`FileStream`].
pub fn new(config: &'a FileScanConfig) -> Self {
Self {
config,
@@ -44,6 +46,7 @@ impl<'a> FileStreamBuilder<'a> {
morselizer: None,
metrics: None,
on_error: OnError::Fail,
+ shared_work_source: None,
}
}
@@ -81,6 +84,15 @@ impl<'a> FileStreamBuilder<'a> {
self
}
+ /// Configure the [`SharedWorkSource`] for sibling work stealing.
+ pub(crate) fn with_shared_work_source(
+ mut self,
+ shared_work_source: Option<SharedWorkSource>,
+ ) -> Self {
+ self.shared_work_source = shared_work_source;
+ self
+ }
+
/// Build the configured [`FileStream`].
pub fn build(self) -> Result<FileStream> {
let Self {
@@ -89,6 +101,7 @@ impl<'a> FileStreamBuilder<'a> {
morselizer,
metrics,
on_error,
+ shared_work_source,
} = self;
let Some(partition) = partition else {
@@ -106,10 +119,14 @@ impl<'a> FileStreamBuilder<'a> {
"FileStreamBuilder invalid partition index: {partition}"
);
};
+ let work_source = match shared_work_source {
+ Some(shared) => WorkSource::Shared(shared),
+ None => WorkSource::Local(file_group.into_inner().into()),
+ };
let file_stream_metrics = FileStreamMetrics::new(metrics, partition);
let scan_state = Box::new(ScanState::new(
- file_group.into_inner(),
+ work_source,
config.limit,
morselizer,
on_error,
diff --git a/datafusion/datasource/src/file_stream/mod.rs b/datafusion/datasource/src/file_stream/mod.rs
index ff71f16023..e277690cff 100644
--- a/datafusion/datasource/src/file_stream/mod.rs
+++ b/datafusion/datasource/src/file_stream/mod.rs
@@ -24,6 +24,7 @@
mod builder;
mod metrics;
mod scan_state;
+pub(crate) mod work_source;
use std::pin::Pin;
use std::sync::Arc;
@@ -175,6 +176,7 @@ mod tests {
IoFutureId, MockMorselizer, MockPlanBuilder, MockPlanner, MorselId,
PendingPlannerBuilder, PollsToResolve,
};
+ use crate::source::DataSource;
use crate::tests::make_partition;
use crate::{PartitionedFile, TableSchema};
use arrow::array::{AsArray, RecordBatch};
@@ -184,14 +186,22 @@ mod tests {
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
use futures::{FutureExt as _, StreamExt as _};
+ use std::collections::{BTreeMap, VecDeque};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
- use crate::file_stream::{FileOpenFuture, FileOpener, FileStreamBuilder, OnError};
+ use crate::file_stream::{
+ FileOpenFuture, FileOpener, FileStream, FileStreamBuilder, OnError,
+ work_source::SharedWorkSource,
+ };
use crate::test_util::MockSource;
use datafusion_common::{assert_batches_eq, exec_err, internal_err};
+ /// Test identifier for one `FileStream` partition.
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+ struct PartitionId(usize);
+
/// Test `FileOpener` which will simulate errors during file opening or scanning
#[derive(Default)]
struct TestOpener {
@@ -758,8 +768,8 @@ mod tests {
async fn morsel_two_ios_one_batch() -> Result<()> {
let test = FileStreamMorselTest::new().with_file(
MockPlanner::builder("file1.parquet")
- .add_plan(PendingPlannerBuilder::new(IoFutureId(1)).build())
- .add_plan(PendingPlannerBuilder::new(IoFutureId(2)).build())
+ .add_plan(PendingPlannerBuilder::new(IoFutureId(1)))
+ .add_plan(PendingPlannerBuilder::new(IoFutureId(2)))
.add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 42))
.return_none(),
);
@@ -871,8 +881,7 @@ mod tests {
async fn morsel_ready_child_planner() -> Result<()> {
let child_planner = MockPlanner::builder("child planner")
.add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 42))
- .return_none()
- .build();
+ .return_none();
let test = FileStreamMorselTest::new().with_file(
MockPlanner::builder("file1.parquet")
@@ -1001,11 +1010,331 @@ mod tests {
Ok(())
}
- /// Tests how FileStream opens and processes files.
+ /// Return a morsel test with two partitions:
+ /// Partition 0: file1, file2, file3
+ /// Partition 1: file4
+ ///
+ /// Partition 1 has only 1 file but it polled first 4 times
+ fn two_partition_morsel_test() -> FileStreamMorselTest {
+ FileStreamMorselTest::new()
+ // Partition 0 has three files
+ .with_file_in_partition(
+ PartitionId(0),
+ MockPlanner::builder("file1.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 101))
+ .return_none(),
+ )
+ .with_file_in_partition(
+ PartitionId(0),
+ MockPlanner::builder("file2.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(11), 102))
+ .return_none(),
+ )
+ .with_file_in_partition(
+ PartitionId(0),
+ MockPlanner::builder("file3.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(12), 103))
+ .return_none(),
+ )
+ // Partition 1 has only one file, but is polled first
+ .with_file_in_partition(
+ PartitionId(1),
+ MockPlanner::builder("file4.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(13), 201))
+ .return_none(),
+ )
+ .with_reads(vec![
+ PartitionId(1),
+ PartitionId(1),
+ PartitionId(1),
+ PartitionId(1),
+ PartitionId(1),
+ ])
+ }
+
+ /// Verifies that an idle sibling stream can steal shared files from
+ /// another stream once it exhausts its own local work.
+ #[tokio::test]
+ async fn morsel_shared_files_can_be_stolen() -> Result<()> {
+ let test = two_partition_morsel_test().with_file_stream_events(false);
+
+ // Partition 0 starts with 3 files, but Partition 1 is polled first.
+ // Since Partition 1 is polled first, it will run all the files even those
+ // that were assigned to Partition 0.
+ insta::assert_snapshot!(test.run().await.unwrap(), @r"
+ ----- Partition 0 -----
+ Done
+ ----- Partition 1 -----
+ Batch: 101
+ Batch: 102
+ Batch: 103
+ Batch: 201
+ Done
+ ----- File Stream Events -----
+ (omitted due to with_file_stream_events(false))
+ ");
+
+ Ok(())
+ }
+
+ /// Verifies that a stream that must preserve order keeps its files local
+ /// and therefore cannot steal from a sibling shared queue.
+ #[tokio::test]
+ async fn morsel_preserve_order_keeps_files_local() -> Result<()> {
+ // same fixture as `morsel_shared_files_can_be_stolen` but marked as
+ // preserve-order
+ let test = two_partition_morsel_test()
+ .with_preserve_order(true)
+ .with_file_stream_events(false);
+
+ // Even though that Partition 1 is polled first, it can not steal files
+ // from partition 0. The three files originally assigned to Partition 0
+ // must be evaluated by Partition 0.
+ insta::assert_snapshot!(test.run().await.unwrap(), @r"
+ ----- Partition 0 -----
+ Batch: 101
+ Batch: 102
+ Batch: 103
+ Done
+ ----- Partition 1 -----
+ Batch: 201
+ Done
+ ----- File Stream Events -----
+ (omitted due to with_file_stream_events(false))
+ ");
+
+ Ok(())
+ }
+
+ /// Verifies that `partitioned_by_file_group` disables shared work stealing.
+ #[tokio::test]
+ async fn morsel_partitioned_by_file_group_keeps_files_local() -> Result<()> {
+ // same fixture as `morsel_shared_files_can_be_stolen` but marked as
+ // preserve-partitioned
+ let test = two_partition_morsel_test()
+ .with_partitioned_by_file_group(true)
+ .with_file_stream_events(false);
+
+ insta::assert_snapshot!(test.run().await.unwrap(), @r"
+ ----- Partition 0 -----
+ Batch: 101
+ Batch: 102
+ Batch: 103
+ Done
+ ----- Partition 1 -----
+ Batch: 201
+ Done
+ ----- File Stream Events -----
+ (omitted due to with_file_stream_events(false))
+ ");
+
+ Ok(())
+ }
+
+ /// Verifies that an empty sibling can immediately steal shared files when
+ /// it is polled before the stream that originally owned them.
+ #[tokio::test]
+ async fn morsel_empty_sibling_can_steal() -> Result<()> {
+ let test = FileStreamMorselTest::new()
+ .with_file_in_partition(
+ PartitionId(0),
+ MockPlanner::builder("file1.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 101))
+ .return_none(),
+ )
+ .with_file_in_partition(
+ PartitionId(0),
+ MockPlanner::builder("file2.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(11), 102))
+ .return_none(),
+ )
+ // Poll the empty sibling first so it steals both files.
+ .with_reads(vec![PartitionId(1), PartitionId(1), PartitionId(1)])
+ .with_file_stream_events(false);
+
+ insta::assert_snapshot!(test.run().await.unwrap(), @r"
+ ----- Partition 0 -----
+ Done
+ ----- Partition 1 -----
+ Batch: 101
+ Batch: 102
+ Done
+ ----- File Stream Events -----
+ (omitted due to with_file_stream_events(false))
+ ");
+
+ Ok(())
+ }
+
+ /// Ensures that if a sibling is built and polled
+ /// before another sibling has been built and contributed its files to the
+ /// shared queue, the first sibling does not finish prematurely.
+ #[tokio::test]
+ async fn morsel_empty_sibling_can_finish_before_shared_work_exists() -> Result<()> {
+ let test = FileStreamMorselTest::new()
+ .with_file_in_partition(
+ PartitionId(0),
+ MockPlanner::builder("file1.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 101))
+ .return_none(),
+ )
+ .with_file_in_partition(
+ PartitionId(0),
+ MockPlanner::builder("file2.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(11), 102))
+ .return_none(),
+ )
+ // Build streams lazily so partition 1 can poll the shared queue
+ // before partition 0 has contributed its files. Once partition 0
+ // is built, a later poll of partition 1 can still steal one of
+ // them from the shared queue.
+ .with_build_streams_on_first_read(true)
+ .with_reads(vec![PartitionId(1), PartitionId(0), PartitionId(1)])
+ .with_file_stream_events(false);
+
+ // Partition 1 polls too early once, then later steals one file after
+ // partition 0 has populated the shared queue.
+ insta::assert_snapshot!(test.run().await.unwrap(), @r"
+ ----- Partition 0 -----
+ Batch: 102
+ Done
+ ----- Partition 1 -----
+ Batch: 101
+ Done
+ ----- File Stream Events -----
+ (omitted due to with_file_stream_events(false))
+ ");
+
+ Ok(())
+ }
+
+ /// Verifies that a sibling hitting its limit does not count shared files
+ /// left in the queue as already processed by that stream.
+ #[tokio::test]
+ async fn morsel_shared_limit_does_not_double_count_files_processed() -> Result<()> {
+ let test = two_partition_morsel_test();
+ let unlimited_config = test.test_config();
+ let limited_config = test.clone().with_limit(1).test_config();
+ let shared_work_source = limited_config
+ .create_sibling_state()
+ .and_then(|state| state.as_ref().downcast_ref::<SharedWorkSource>().cloned())
+ .expect("shared work source");
+ let limited_metrics = ExecutionPlanMetricsSet::new();
+ let unlimited_metrics = ExecutionPlanMetricsSet::new();
+
+ let limited_stream = FileStreamBuilder::new(&limited_config)
+ .with_partition(1)
+ .with_shared_work_source(Some(shared_work_source.clone()))
+ .with_morselizer(Box::new(test.morselizer.clone()))
+ .with_metrics(&limited_metrics)
+ .build()?;
+
+ let unlimited_stream = FileStreamBuilder::new(&unlimited_config)
+ .with_partition(0)
+ .with_shared_work_source(Some(shared_work_source))
+ .with_morselizer(Box::new(test.morselizer))
+ .with_metrics(&unlimited_metrics)
+ .build()?;
+
+ let limited_output = drain_stream_output(limited_stream).await?;
+ let unlimited_output = drain_stream_output(unlimited_stream).await?;
+
+ insta::assert_snapshot!(format!(
+ "----- Limited Stream -----\n{limited_output}\n----- Unlimited Stream -----\n{unlimited_output}"
+ ), @r"
+ ----- Limited Stream -----
+ Batch: 101
+ ----- Unlimited Stream -----
+ Batch: 102
+ Batch: 103
+ Batch: 201
+ ");
+
+ assert_eq!(
+ metric_count(&limited_metrics, "files_opened"),
+ 1,
+ "the limited stream should only open the file that produced its output"
+ );
+ assert_eq!(
+ metric_count(&limited_metrics, "files_processed"),
+ 1,
+ "the limited stream should only mark its own file as processed"
+ );
+ assert_eq!(
+ metric_count(&unlimited_metrics, "files_opened"),
+ 3,
+ "the draining stream should open the remaining shared files"
+ );
+ assert_eq!(
+ metric_count(&unlimited_metrics, "files_processed"),
+ 3,
+ "the draining stream should process exactly the files it opened"
+ );
+
+ Ok(())
+ }
+
+ /// Verifies that one fast sibling can drain shared files that originated
+ /// in more than one other partition.
+ #[tokio::test]
+ async fn morsel_one_sibling_can_drain_multiple_siblings() -> Result<()> {
+ let test = FileStreamMorselTest::new()
+ .with_file_in_partition(
+ PartitionId(0),
+ MockPlanner::builder("file1.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 101))
+ .return_none(),
+ )
+ // Partition 1 has two files
+ .with_file_in_partition(
+ PartitionId(1),
+ MockPlanner::builder("file2.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(11), 102))
+ .return_none(),
+ )
+ .with_file_in_partition(
+ PartitionId(1),
+ MockPlanner::builder("file3.parquet")
+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(12), 103))
+ .return_none(),
+ )
+ // Partition 2 starts empty but is polled first, so it should drain
+ // the shared queue across both sibling partitions.
+ .with_reads(vec![
+ PartitionId(2),
+ PartitionId(2),
+ PartitionId(1),
+ PartitionId(2),
+ ])
+ .with_file_stream_events(false);
+
+ insta::assert_snapshot!(test.run().await.unwrap(), @r"
+ ----- Partition 0 -----
+ Done
+ ----- Partition 1 -----
+ Batch: 103
+ Done
+ ----- Partition 2 -----
+ Batch: 101
+ Batch: 102
+ Done
+ ----- File Stream Events -----
+ (omitted due to with_file_stream_events(false))
+ ");
+
+ Ok(())
+ }
+
+ /// Tests how one or more `FileStream`s consume morselized file work.
#[derive(Clone)]
struct FileStreamMorselTest {
morselizer: MockMorselizer,
- file_names: Vec<String>,
+ partition_files: BTreeMap<PartitionId, Vec<String>>,
+ preserve_order: bool,
+ partitioned_by_file_group: bool,
+ file_stream_events: bool,
+ build_streams_on_first_read: bool,
+ reads: Vec<PartitionId>,
limit: Option<usize>,
}
@@ -1014,75 +1343,238 @@ mod tests {
fn new() -> Self {
Self {
morselizer: MockMorselizer::new(),
- file_names: vec![],
+ partition_files: BTreeMap::new(),
+ preserve_order: false,
+ partitioned_by_file_group: false,
+ file_stream_events: true,
+ build_streams_on_first_read: false,
+ reads: vec![],
limit: None,
}
}
- /// Adds one file and its root planner to the test input.
- fn with_file(mut self, planner: impl Into<MockPlanner>) -> Self {
+ /// Adds one file and its root planner to partition 0.
+ fn with_file(self, planner: impl Into<MockPlanner>) -> Self {
+ self.with_file_in_partition(PartitionId(0), planner)
+ }
+
+ /// Adds one file and its root planner to the specified input partition.
+ fn with_file_in_partition(
+ mut self,
+ partition: PartitionId,
+ planner: impl Into<MockPlanner>,
+ ) -> Self {
let planner = planner.into();
- self.file_names.push(planner.file_path().to_string());
- self.morselizer = self.morselizer.with_file(planner);
+ let file_path = planner.file_path().to_string();
+ self.morselizer = self.morselizer.with_planner(planner);
+ self.partition_files
+ .entry(partition)
+ .or_default()
+ .push(file_path);
self
}
- /// Sets a global output limit for the stream.
+ /// Marks the stream (and all partitions) to preserve the specified file
+ /// order.
+ fn with_preserve_order(mut self, preserve_order: bool) -> Self {
+ self.preserve_order = preserve_order;
+ self
+ }
+
+ /// Marks the test scan as pre-partitioned by file group, which should
+ /// force each stream to keep its own files local.
+ fn with_partitioned_by_file_group(
+ mut self,
+ partitioned_by_file_group: bool,
+ ) -> Self {
+ self.partitioned_by_file_group = partitioned_by_file_group;
+ self
+ }
+
+ /// Controls whether scheduler events are included in the snapshot.
+ ///
+ /// When disabled, `run()` still includes the event section header but
+ /// replaces the trace with a fixed placeholder so tests can focus only
+ /// on the output batches.
+ fn with_file_stream_events(mut self, file_stream_events: bool) -> Self {
+ self.file_stream_events = file_stream_events;
+ self
+ }
+
+ /// Controls whether streams are all built up front or lazily on their
+ /// first read.
+ ///
+ /// The default builds all streams before polling begins, which matches
+ /// normal execution. Tests may enable lazy creation to model races
+ /// where one sibling polls before another has contributed its files to
+ /// the shared queue.
+ fn with_build_streams_on_first_read(
+ mut self,
+ build_streams_on_first_read: bool,
+ ) -> Self {
+ self.build_streams_on_first_read = build_streams_on_first_read;
+ self
+ }
+
+ /// Sets the partition polling order.
+ ///
+ /// `run()` polls these partitions in the listed order first. After
+ /// those explicit reads are exhausted, it completes to round
+ /// robin across all configured partitions, skipping any streams that
+ /// have already finished.
+ ///
+ /// This allows testing early scheduling decisions explicit in a test
+ /// while avoiding a fully scripted poll trace for the remainder.
+ fn with_reads(mut self, reads: Vec<PartitionId>) -> Self {
+ self.reads = reads;
+ self
+ }
+
+ /// Sets a global output limit for all streams created by this test.
fn with_limit(mut self, limit: usize) -> Self {
self.limit = Some(limit);
self
}
- /// Runs the test returns combined output and scheduler trace text as a String.
+ /// Runs the test and returns combined stream output and scheduler
+ /// trace text.
async fn run(self) -> Result<String> {
let observer = self.morselizer.observer().clone();
observer.clear();
- let config = self.test_config();
let metrics_set = ExecutionPlanMetricsSet::new();
- let mut stream = FileStreamBuilder::new(&config)
- .with_partition(0)
- .with_morselizer(Box::new(self.morselizer))
- .with_metrics(&metrics_set)
- .build()?;
+ let partition_count = self.num_partitions();
- let mut stream_contents = Vec::new();
- while let Some(result) = stream.next().await {
- match result {
- Ok(batch) => {
- let col = batch.column(0).as_primitive::<Int32Type>();
- let batch_id = col.value(0);
- stream_contents.push(format!("Batch: {batch_id}"));
- }
- Err(e) => {
- // Pull the actual message for external errors rather than
- // relying on DataFusionError formatting, which changes
- // if backtraces are enabled, etc
- let message = if let DataFusionError::External(generic) = e {
- generic.to_string()
- } else {
- e.to_string()
- };
- stream_contents.push(format!("Error: {message}"));
- }
+ let mut partitions = (0..partition_count)
+ .map(|_| PartitionState::new())
+ .collect::<Vec<_>>();
+
+ let mut build_order = Vec::new();
+ for partition in self.reads.iter().map(|partition| partition.0) {
+ if !build_order.contains(&partition) {
+ build_order.push(partition);
+ }
+ }
+ for partition in 0..partition_count {
+ if !build_order.contains(&partition) {
+ build_order.push(partition);
}
}
- stream_contents.push("Done".to_string());
- Ok(format!(
- "----- Output Stream -----\n{}\n----- File Stream Events -----\n{}",
- stream_contents.join("\n"),
+ let config = self.test_config();
+ // `DataSourceExec::execute` creates one execution-local shared
+ // state object via `create_sibling_state()` and then passes it
+ // to `open_with_sibling_state(...)`. These tests build
+ // `FileStream`s directly, bypassing `DataSourceExec`, so they must
+ // perform the same setup explicitly when exercising sibling-stream
+ // work stealing.
+ let shared_work_source = config.create_sibling_state().and_then(|state| {
+ state.as_ref().downcast_ref::<SharedWorkSource>().cloned()
+ });
+ if !self.build_streams_on_first_read {
+ for partition in build_order {
+ let stream = FileStreamBuilder::new(&config)
+ .with_partition(partition)
+ .with_shared_work_source(shared_work_source.clone())
+ .with_morselizer(Box::new(self.morselizer.clone()))
+ .with_metrics(&metrics_set)
+ .build()?;
+ partitions[partition].set_stream(stream);
+ }
+ }
+
+ let mut initial_reads: VecDeque<_> = self.reads.into();
+ let mut next_round_robin = 0;
+
+ while !initial_reads.is_empty()
+ || partitions.iter().any(PartitionState::is_active)
+ {
+ let partition = if let Some(partition) = initial_reads.pop_front() {
+ partition.0
+ } else {
+ let partition = next_round_robin;
+ next_round_robin = (next_round_robin + 1) % partition_count.max(1);
+ partition
+ };
+
+ let partition_state = &mut partitions[partition];
+
+ if self.build_streams_on_first_read && !partition_state.built {
+ let stream = FileStreamBuilder::new(&config)
+ .with_partition(partition)
+ .with_shared_work_source(shared_work_source.clone())
+ .with_morselizer(Box::new(self.morselizer.clone()))
+ .with_metrics(&metrics_set)
+ .build()?;
+ partition_state.set_stream(stream);
+ }
+
+ let Some(stream) = partition_state.stream.as_mut() else {
+ continue;
+ };
+
+ match stream.next().await {
+ Some(result) => partition_state.push_output(format_result(result)),
+ None => partition_state.finish(),
+ }
+ }
+
+ let output_text = if partition_count == 1 {
+ format!(
+ "----- Output Stream -----\n{}",
+ partitions[0].output.join("\n")
+ )
+ } else {
+ partitions
+ .into_iter()
+ .enumerate()
+ .map(|(partition, state)| {
+ format!(
+ "----- Partition {} -----\n{}",
+ partition,
+ state.output.join("\n")
+ )
+ })
+ .collect::<Vec<_>>()
+ .join("\n")
+ };
+
+ let file_stream_events = if self.file_stream_events {
observer.format_events()
+ } else {
+ "(omitted due to with_file_stream_events(false))".to_string()
+ };
+
+ Ok(format!(
+ "{output_text}\n----- File Stream Events -----\n{file_stream_events}",
))
}
- /// Builds the `FileScanConfig` for the configured mock file set.
+ /// Returns the number of configured partitions, including empty ones
+ /// that appear only in the explicit read schedule.
+ fn num_partitions(&self) -> usize {
+ self.partition_files
+ .keys()
+ .map(|partition| partition.0 + 1)
+ .chain(self.reads.iter().map(|partition| partition.0 + 1))
+ .max()
+ .unwrap_or(1)
+ }
+
+ /// Builds a `FileScanConfig` covering every configured partition.
fn test_config(&self) -> FileScanConfig {
- let file_group = self
- .file_names
- .iter()
- .map(|name| PartitionedFile::new(name, 10))
- .collect();
+ let file_groups = (0..self.num_partitions())
+ .map(|partition| {
+ self.partition_files
+ .get(&PartitionId(partition))
+ .into_iter()
+ .flat_map(|files| files.iter())
+ .map(|name| PartitionedFile::new(name, 10))
+ .collect::<Vec<_>>()
+ .into()
+ })
+ .collect::<Vec<_>>();
+
let table_schema = TableSchema::new(
Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)])),
vec![],
@@ -1091,9 +1583,94 @@ mod tests {
ObjectStoreUrl::parse("test:///").unwrap(),
Arc::new(MockSource::new(table_schema)),
)
- .with_file_group(file_group)
+ .with_file_groups(file_groups)
.with_limit(self.limit)
+ .with_preserve_order(self.preserve_order)
+ .with_partitioned_by_file_group(self.partitioned_by_file_group)
.build()
}
}
+
+ /// Formats one stream poll result into a stable snapshot line.
+ fn format_result(result: Result<RecordBatch>) -> String {
+ match result {
+ Ok(batch) => {
+ let col = batch.column(0).as_primitive::<Int32Type>();
+ let batch_id = col.value(0);
+ format!("Batch: {batch_id}")
+ }
+ Err(e) => {
+ // Pull the actual message for external errors rather than
+ // relying on DataFusionError formatting, which changes if
+ // backtraces are enabled, etc.
+ let message = if let DataFusionError::External(generic) = e {
+ generic.to_string()
+ } else {
+ e.to_string()
+ };
+ format!("Error: {message}")
+ }
+ }
+ }
+
+ async fn drain_stream_output(stream: FileStream) -> Result<String> {
+ let output = stream
+ .collect::<Vec<_>>()
+ .await
+ .into_iter()
+ .map(|result| result.map(|batch| format_result(Ok(batch))))
+ .collect::<Result<Vec<_>>>()?;
+ Ok(output.join("\n"))
+ }
+
+ fn metric_count(metrics: &ExecutionPlanMetricsSet, name: &str) -> usize {
+ metrics
+ .clone_inner()
+ .sum_by_name(name)
+ .unwrap_or_else(|| panic!("missing metric: {name}"))
+ .as_usize()
+ }
+
+ /// Test-only state for one stream partition in [`FileStreamMorselTest`].
+ struct PartitionState {
+ /// Whether the `FileStream` for this partition has been built yet.
+ built: bool,
+ /// The live stream, if this partition has not finished yet.
+ stream: Option<FileStream>,
+ /// Snapshot lines produced by this partition.
+ output: Vec<String>,
+ }
+
+ impl PartitionState {
+ /// Create an unbuilt partition with no output yet.
+ fn new() -> Self {
+ Self {
+ built: false,
+ stream: None,
+ output: vec![],
+ }
+ }
+
+ /// Returns true if this partition might still produce output.
+ fn is_active(&self) -> bool {
+ !self.built || self.stream.is_some()
+ }
+
+ /// Records that this partition's stream has been built.
+ fn set_stream(&mut self, stream: FileStream) {
+ self.stream = Some(stream);
+ self.built = true;
+ }
+
+ /// Records one formatted output line for this partition.
+ fn push_output(&mut self, line: String) {
+ self.output.push(line);
+ }
+
+ /// Marks this partition as finished.
+ fn finish(&mut self) {
+ self.push_output("Done".to_string());
+ self.stream = None;
+ }
+ }
}
diff --git a/datafusion/datasource/src/file_stream/scan_state.rs b/datafusion/datasource/src/file_stream/scan_state.rs
index 025164c29c..21125cd088 100644
--- a/datafusion/datasource/src/file_stream/scan_state.rs
+++ b/datafusion/datasource/src/file_stream/scan_state.rs
@@ -19,7 +19,6 @@ use datafusion_common::internal_datafusion_err;
use std::collections::VecDeque;
use std::task::{Context, Poll};
-use crate::PartitionedFile;
use crate::morsel::{Morsel, MorselPlanner, Morselizer, PendingMorselPlanner};
use arrow::record_batch::RecordBatch;
use datafusion_common::{DataFusionError, Result};
@@ -27,6 +26,7 @@ use datafusion_physical_plan::metrics::ScopedTimerGuard;
use futures::stream::BoxStream;
use futures::{FutureExt as _, StreamExt as _};
+use super::work_source::WorkSource;
use super::{FileStreamMetrics, OnError};
/// State [`FileStreamState::Scan`].
@@ -45,7 +45,7 @@ use super::{FileStreamMetrics, OnError};
/// # State Transitions
///
/// ```text
-/// file_iter
+/// work_source
/// |
/// v
/// morselizer.plan_file(file)
@@ -62,8 +62,8 @@ use super::{FileStreamMetrics, OnError};
///
/// [`FileStreamState::Scan`]: super::FileStreamState::Scan
pub(super) struct ScanState {
- /// Files that still need to be planned.
- file_iter: VecDeque<PartitionedFile>,
+ /// Unopened files that still need to be planned for this stream.
+ work_source: WorkSource,
/// Remaining row limit, if any.
remain: Option<usize>,
/// The morselizer used to plan files.
@@ -76,7 +76,10 @@ pub(super) struct ScanState {
ready_morsels: VecDeque<Box<dyn Morsel>>,
/// The active reader, if any.
reader: Option<BoxStream<'static, Result<RecordBatch>>>,
- /// The currently outstanding I/O, if any.
+ /// The single planner currently blocked on I/O, if any.
+ ///
+ /// Once the I/O completes, yields the next planner and is pushed back
+ /// onto `ready_planners`.
pending_planner: Option<PendingMorselPlanner>,
/// Metrics for the active scan queues.
metrics: FileStreamMetrics,
@@ -84,15 +87,14 @@ pub(super) struct ScanState {
impl ScanState {
pub(super) fn new(
- file_iter: impl Into<VecDeque<PartitionedFile>>,
+ work_source: WorkSource,
remain: Option<usize>,
morselizer: Box<dyn Morselizer>,
on_error: OnError,
metrics: FileStreamMetrics,
) -> Self {
- let file_iter = file_iter.into();
Self {
- file_iter,
+ work_source,
remain,
morselizer,
on_error,
@@ -170,7 +172,7 @@ impl ScanState {
(batch, false)
} else {
let batch = batch.slice(0, *remain);
- let done = 1 + self.file_iter.len();
+ let done = 1 + self.work_source.skipped_on_limit();
self.metrics.files_processed.add(done);
*remain = 0;
(batch, true)
@@ -263,8 +265,8 @@ impl ScanState {
};
}
- // No outstanding work remains, so morselize the next unopened file.
- let part_file = match self.file_iter.pop_front() {
+ // No outstanding work remains, so begin planning the next unopened file.
+ let part_file = match self.work_source.pop_front() {
Some(part_file) => part_file,
None => return ScanAndReturn::Done(None),
};
diff --git a/datafusion/datasource/src/file_stream/work_source.rs b/datafusion/datasource/src/file_stream/work_source.rs
new file mode 100644
index 0000000000..7f31dacca9
--- /dev/null
+++ b/datafusion/datasource/src/file_stream/work_source.rs
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::VecDeque;
+use std::sync::Arc;
+
+use crate::PartitionedFile;
+use crate::file_groups::FileGroup;
+use crate::file_scan_config::FileScanConfig;
+use parking_lot::Mutex;
+
+/// Source of work for `ScanState`.
+///
+/// Streams that may share work across siblings use [`WorkSource::Shared`],
+/// while streams that can not share work (e.g. because they must preserve file
+/// order) use [`WorkSource::Local`].
+#[derive(Debug, Clone)]
+pub(super) enum WorkSource {
+ /// Files this stream will plan locally without sharing them.
+ Local(VecDeque<PartitionedFile>),
+ /// Files shared with sibling streams.
+ Shared(SharedWorkSource),
+}
+
+impl WorkSource {
+ /// Pop the next file to plan from this work source.
+ pub(super) fn pop_front(&mut self) -> Option<PartitionedFile> {
+ match self {
+ Self::Local(files) => files.pop_front(),
+ Self::Shared(shared) => shared.pop_front(),
+ }
+ }
+
+ /// Return how many queued files should be counted as already processed
+ /// when this stream stops early after hitting a global limit.
+ pub(super) fn skipped_on_limit(&self) -> usize {
+ match self {
+ Self::Local(files) => files.len(),
+ Self::Shared(_) => 0,
+ }
+ }
+}
+
+/// Shared source of work for sibling `FileStream`s
+///
+/// The queue is created once per execution and shared by all reorderable
+/// sibling streams for that execution. Whichever stream becomes idle first may
+/// take the next unopened file from the front of the queue.
+///
+/// It uses a [`Mutex`] internally to provide thread-safe access
+/// to the shared file queue.
+#[derive(Debug, Clone)]
+pub(crate) struct SharedWorkSource {
+ inner: Arc<SharedWorkSourceInner>,
+}
+
+#[derive(Debug, Default)]
+pub(super) struct SharedWorkSourceInner {
+ files: Mutex<VecDeque<PartitionedFile>>,
+}
+
+impl SharedWorkSource {
+ /// Create a shared work source containing the provided unopened files.
+ pub(crate) fn new(files: impl IntoIterator<Item = PartitionedFile>) -> Self {
+ let files = files.into_iter().collect();
+ Self {
+ inner: Arc::new(SharedWorkSourceInner {
+ files: Mutex::new(files),
+ }),
+ }
+ }
+
+ /// Create a shared work source for the unopened files in `config`.
+ pub(crate) fn from_config(config: &FileScanConfig) -> Self {
+ Self::new(config.file_groups.iter().flat_map(FileGroup::iter).cloned())
+ }
+
+ /// Pop the next file from the shared work queue.
+ ///
+ /// Returns `None` if the queue is empty
+ fn pop_front(&self) -> Option<PartitionedFile> {
+ self.inner.files.lock().pop_front()
+ }
+}
diff --git a/datafusion/datasource/src/morsel/mocks.rs b/datafusion/datasource/src/morsel/mocks.rs
index cd1fa3732f..ceb0e72069 100644
--- a/datafusion/datasource/src/morsel/mocks.rs
+++ b/datafusion/datasource/src/morsel/mocks.rs
@@ -295,8 +295,11 @@ impl MockPlanBuilder {
}
/// Add a ready child planner
- pub(crate) fn with_ready_planner(self, ready_planners: MockPlanner) -> Self {
- self.with_ready_planners(vec![ready_planners])
+ pub(crate) fn with_ready_planner(
+ self,
+ ready_planner: impl Into<MockPlanner>,
+ ) -> Self {
+ self.with_ready_planners(vec![ready_planner.into()])
}
/// Add ready child planners produced by this planning step.
@@ -430,8 +433,9 @@ impl MockMorselizer {
&self.observer
}
- /// Associates a file path with the planner spec used to open it.
- pub(crate) fn with_file(mut self, planner: MockPlanner) -> Self {
+ /// Specify the return planner for the specified file_path
+ pub(crate) fn with_planner(mut self, planner: impl Into<MockPlanner>) -> Self {
+ let planner = planner.into();
self.files.insert(planner.file_path.clone(), planner);
self
}
diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs
index 02a15502da..420c6b508c 100644
--- a/datafusion/datasource/src/source.rs
+++ b/datafusion/datasource/src/source.rs
@@ -20,7 +20,7 @@
use std::any::Any;
use std::fmt;
use std::fmt::{Debug, Formatter};
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_plan::execution_plan::{
@@ -124,11 +124,21 @@ use datafusion_physical_plan::filter_pushdown::{
/// └─────────────────────┘
/// ```
pub trait DataSource: Any + Send + Sync + Debug {
+ /// Open the specified output partition and return its stream of
+ /// [`RecordBatch`]es.
+ ///
+ /// This should be used by data sources that do not need any sibling
+ /// coordination. Data sources that want to use per-execution shared state
+ /// (for example, to reorder work across partitions at runtime) should
+ /// implement [`Self::open_with_args`] instead.
+ ///
+ /// [`RecordBatch`]: arrow::record_batch::RecordBatch
fn open(
&self,
partition: usize,
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream>;
+
/// Format this source for display in explain plans
fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> fmt::Result;
@@ -246,6 +256,55 @@ pub trait DataSource: Any + Send + Sync + Debug {
) -> Option<Arc<dyn DataSource>> {
None
}
+
+ /// Create per execution state to share across sibling instances of this
+ /// data source during one execution.
+ ///
+ /// Returns `None` (the default) if this data source has
+ /// no sibling-shared execution state.
+ fn create_sibling_state(&self) -> Option<Arc<dyn Any + Send + Sync>> {
+ None
+ }
+
+ /// Open a partition using optional sibling-shared execution state.
+ ///
+ /// The default implementation ignores the additional state and delegates to
+ /// [`Self::open`].
+ fn open_with_args(&self, args: OpenArgs) -> Result<SendableRecordBatchStream> {
+ self.open(args.partition, args.context)
+ }
+}
+
+/// Arguments for [`DataSource::open_with_args`]
+#[derive(Debug, Clone)]
+pub struct OpenArgs {
+ /// Which partition to open
+ pub partition: usize,
+ /// The task context for execution
+ pub context: Arc<TaskContext>,
+ /// Optional sibling-shared execution state, see
+ /// [`DataSource::create_sibling_state`] for details.
+ pub sibling_state: Option<Arc<dyn Any + Send + Sync>>,
+}
+
+impl OpenArgs {
+ /// Create a new OpenArgs with required arguments
+ pub fn new(partition: usize, context: Arc<TaskContext>) -> Self {
+ Self {
+ partition,
+ context,
+ sibling_state: None,
+ }
+ }
+
+ /// Set sibling shared state
+ pub fn with_shared_state(
+ mut self,
+ sibling_state: Option<Arc<dyn Any + Send + Sync>>,
+ ) -> Self {
+ self.sibling_state = sibling_state;
+ self
+ }
}
impl dyn DataSource {
@@ -276,6 +335,12 @@ pub struct DataSourceExec {
data_source: Arc<dyn DataSource>,
/// Cached plan properties such as sort order
cache: Arc<PlanProperties>,
+ /// Per execution state shared across partitions of this plan.
+ ///
+ /// Created by [`DataSource::create_sibling_state`]
+ /// and then passed to
+ /// [`DataSource::open_with_args`].
+ execution_state: Arc<OnceLock<Option<Arc<dyn Any + Send + Sync>>>>,
}
impl DisplayAs for DataSourceExec {
@@ -349,8 +414,15 @@ impl ExecutionPlan for DataSourceExec {
partition: usize,
context: Arc<TaskContext>,
) -> Result<SendableRecordBatchStream> {
- let stream = self.data_source.open(partition, Arc::clone(&context))?;
+ let shared_state = self
+ .execution_state
+ .get_or_init(|| self.data_source.create_sibling_state())
+ .clone();
+ let args = OpenArgs::new(partition, Arc::clone(&context))
+ .with_shared_state(shared_state);
+ let stream = self.data_source.open_with_args(args)?;
let batch_size = context.session_config().batch_size();
+
log::debug!(
"Batch splitting enabled for partition {partition}: batch_size={batch_size}"
);
@@ -386,8 +458,13 @@ impl ExecutionPlan for DataSourceExec {
fn with_fetch(&self, limit: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {
let data_source = self.data_source.with_fetch(limit)?;
let cache = Arc::clone(&self.cache);
+ let execution_state = Arc::new(OnceLock::new());
- Some(Arc::new(Self { data_source, cache }))
+ Some(Arc::new(Self {
+ data_source,
+ cache,
+ execution_state,
+ }))
}
fn fetch(&self) -> Option<usize> {
@@ -480,6 +557,12 @@ impl ExecutionPlan for DataSourceExec {
as Arc<dyn ExecutionPlan>
})
}
+
+ fn reset_state(self: Arc<Self>) -> Result<Arc<dyn ExecutionPlan>> {
+ let mut new_exec = Arc::unwrap_or_clone(self);
+ new_exec.execution_state = Arc::new(OnceLock::new());
+ Ok(Arc::new(new_exec))
+ }
}
impl DataSourceExec {
@@ -493,6 +576,7 @@ impl DataSourceExec {
Self {
data_source,
cache: Arc::new(cache),
+ execution_state: Arc::new(OnceLock::new()),
}
}
@@ -504,6 +588,7 @@ impl DataSourceExec {
pub fn with_data_source(mut self, data_source: Arc<dyn DataSource>) -> Self {
self.cache = Arc::new(Self::compute_properties(&data_source));
self.data_source = data_source;
+ self.execution_state = Arc::new(OnceLock::new());
self
}
diff --git a/datafusion/datasource/src/test_util.rs b/datafusion/datasource/src/test_util.rs
index ffd4b6123e..b59ce58a42 100644
--- a/datafusion/datasource/src/test_util.rs
+++ b/datafusion/datasource/src/test_util.rs
@@ -34,6 +34,7 @@ pub(crate) struct MockSource {
filter: Option<Arc<dyn PhysicalExpr>>,
table_schema: crate::table_schema::TableSchema,
projection: crate::projection::SplitProjection,
+ file_opener: Option<Arc<dyn FileOpener>>,
}
impl Default for MockSource {
@@ -45,6 +46,7 @@ impl Default for MockSource {
filter: None,
projection: crate::projection::SplitProjection::unprojected(&table_schema),
table_schema,
+ file_opener: None,
}
}
}
@@ -57,6 +59,7 @@ impl MockSource {
filter: None,
projection: crate::projection::SplitProjection::unprojected(&table_schema),
table_schema,
+ file_opener: None,
}
}
@@ -64,6 +67,11 @@ impl MockSource {
self.filter = Some(filter);
self
}
+
+ pub fn with_file_opener(mut self, file_opener: Arc<dyn FileOpener>) -> Self {
+ self.file_opener = Some(file_opener);
+ self
+ }
}
impl FileSource for MockSource {
@@ -73,7 +81,9 @@ impl FileSource for MockSource {
_base_config: &FileScanConfig,
_partition: usize,
) -> Result<Arc<dyn FileOpener>> {
- unimplemented!()
+ self.file_opener.clone().ok_or_else(|| {
+ datafusion_common::internal_datafusion_err!("MockSource missing FileOpener")
+ })
}
fn filter(&self) -> Option<Arc<dyn PhysicalExpr>> {