Apache DataFusion SQL Query Engine
Rewrite FileStream in terms of Morsel API (#21342)
Andrew Lamb776b72325aa3
diff --git a/Cargo.lock b/Cargo.lockindex 0a2e1b7b04..a67cffe08a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1964,6 +1964,7 @@ dependencies = ["flate2","futures","glob",+ "insta","itertools 0.14.0","liblzma","log",diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 5307cb15ee..1db055880d 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-//! [`ParquetOpener`] and [`ParquetMorselizer`] state machines for opening Parquet files+//! [`ParquetMorselizer`] state machines for opening Parquet filesuse crate::page_filter::PagePruningAccessPlanFilter;use crate::row_filter::build_projection_read_plan;@@ -26,15 +26,11 @@ use crate::{};use arrow::array::{RecordBatch, RecordBatchOptions};use arrow::datatypes::DataType;-use datafusion_common::internal_err;-use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};-use datafusion_datasource::morsel::{- Morsel, MorselPlan, MorselPlanner, Morselizer, PendingMorselPlanner,-};+use datafusion_datasource::morsel::{Morsel, MorselPlan, MorselPlanner, Morselizer};use datafusion_physical_expr::projection::{ProjectionExprs, Projector};use datafusion_physical_expr::utils::reassign_expr_columns;use datafusion_physical_expr_adapter::replace_columns_with_literals;-use std::collections::{HashMap, VecDeque};+use std::collections::HashMap;use std::fmt;use std::future::Future;use std::mem;@@ -82,19 +78,6 @@ use parquet::bloom_filter::Sbbf;use parquet::errors::ParquetError;use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader};-/// Implements [`FileOpener`] for Parquet-#[derive(Clone)]-pub(super) struct ParquetOpener {- pub(super) morselizer: ParquetMorselizer,-}--impl FileOpener for ParquetOpener {- fn open(&self, partitioned_file: PartitionedFile) -> Result<FileOpenFuture> {- let future = ParquetOpenFuture::new(&self.morselizer, partitioned_file)?;- Ok(Box::pin(future))- }-}-/// Stateless Parquet morselizer implementation.////// Reading a Parquet file is a multi-stage process, with multiple CPU-intensive@@ -173,7 +156,7 @@ impl Morselizer for ParquetMorselizer {}}-/// States for [`ParquetOpenFuture`]+/// States for [`ParquetMorselPlanner`]////// These states correspond to the steps required to read and apply various/// filter operations.@@ -425,85 +408,6 @@ impl ParquetOpenState {}}-/// Adapter for a [`MorselPlanner`] to the [`FileOpener`] API-///-/// Compatibility adapter that drives a morsel planner through the-/// [`FileOpener`] API.-struct ParquetOpenFuture {- planner: Option<Box<dyn MorselPlanner>>,- pending_io: Option<PendingMorselPlanner>,- ready_morsels: VecDeque<Box<dyn Morsel>>,-}--impl ParquetOpenFuture {- fn new(- morselizer: &ParquetMorselizer,- partitioned_file: PartitionedFile,- ) -> Result<Self> {- Ok(Self {- planner: Some(morselizer.plan_file(partitioned_file)?),- pending_io: None,- ready_morsels: VecDeque::new(),- })- }-}--impl Future for ParquetOpenFuture {- type Output = Result<BoxStream<'static, Result<RecordBatch>>>;-- fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {- loop {- // If planner I/O completed, resume with the returned planner.- if let Some(io_future) = self.pending_io.as_mut() {- let maybe_planner = ready!(io_future.poll_unpin(cx));- // Clear `pending_io` before handling the result so an error- // cannot leave both continuation paths populated.- self.pending_io = None;- if self.planner.is_some() {- return Poll::Ready(internal_err!(- "ParquetOpenFuture does not support concurrent planners"- ));- }- self.planner = Some(maybe_planner?);- }-- // If a stream morsel is ready, return it.- if let Some(morsel) = self.ready_morsels.pop_front() {- return Poll::Ready(Ok(morsel.into_stream()));- }-- // This shim must always own either a planner, a pending planner- // future, or a ready morsel. Reaching this branch means the- // continuation was lost.- let Some(planner) = self.planner.take() else {- return Poll::Ready(internal_err!(- "ParquetOpenFuture polled after completion"- ));- };-- // Planner completed without producing a stream morsel.- // (e.g. all row groups were pruned)- let Some(mut plan) = planner.plan()? else {- return Poll::Ready(Ok(futures::stream::empty().boxed()));- };-- let mut child_planners = plan.take_ready_planners();- if child_planners.len() > 1 {- return Poll::Ready(internal_err!(- "Parquet FileOpener adapter does not support child morsel planners"- ));- }- self.planner = child_planners.pop();-- self.ready_morsels = plan.take_morsels().into();-- if let Some(io_future) = plan.take_pending_planner() {- self.pending_io = Some(io_future);- }- }- }-}-/// Implements the Morsel APIstruct ParquetStreamMorsel {stream: BoxStream<'static, Result<RecordBatch>>,@@ -1710,17 +1614,18 @@ async fn load_page_index<T: AsyncFileReader>(#[cfg(test)]mod test {- use std::sync::Arc;-+ use super::*;use super::{ConstantColumns, ParquetMorselizer, constant_columns_from_stats};- use crate::{DefaultParquetFileReaderFactory, RowGroupAccess, opener::ParquetOpener};+ use crate::{DefaultParquetFileReaderFactory, RowGroupAccess};+ use arrow::array::RecordBatch;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use bytes::{BufMut, BytesMut};use datafusion_common::{- ColumnStatistics, DataFusionError, ScalarValue, Statistics, record_batch,+ ColumnStatistics, ScalarValue, Statistics, internal_err, record_batch,stats::Precision,};- use datafusion_datasource::{PartitionedFile, TableSchema, file_stream::FileOpener};+ use datafusion_datasource::morsel::{Morsel, Morselizer};+ use datafusion_datasource::{PartitionedFile, TableSchema};use datafusion_expr::{col, lit};use datafusion_physical_expr::{PhysicalExpr,@@ -1732,14 +1637,17 @@ mod test {DefaultPhysicalExprAdapterFactory, replace_columns_with_literals,};use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;- use futures::{Stream, StreamExt};+ use futures::StreamExt;+ use futures::stream::BoxStream;use object_store::{ObjectStore, ObjectStoreExt, memory::InMemory, path::Path};use parquet::arrow::ArrowWriter;use parquet::file::properties::WriterProperties;+ use std::collections::VecDeque;+ use std::sync::Arc;- /// Builder for creating [`ParquetOpener`] instances with sensible defaults for tests.+ /// Builder for creating [`ParquetMorselizer`] instances with sensible defaults for tests./// This helps reduce code duplication and makes it clear what differs between test cases.- struct ParquetOpenerBuilder {+ struct ParquetMorselizerBuilder {store: Option<Arc<dyn ObjectStore>>,table_schema: Option<TableSchema>,partition_index: usize,@@ -1756,13 +1664,13 @@ mod test {enable_page_index: bool,enable_bloom_filter: bool,enable_row_group_stats_pruning: bool,- coerce_int96: Option<arrow::datatypes::TimeUnit>,+ coerce_int96: Option<TimeUnit>,max_predicate_cache_size: Option<usize>,reverse_row_groups: bool,preserve_order: bool,}- impl ParquetOpenerBuilder {+ impl ParquetMorselizerBuilder {/// Create a new builder with sensible defaults for tests.fn new() -> Self {Self {@@ -1849,17 +1757,17 @@ mod test {self}- /// Build the ParquetOpener instance.+ /// Build the ParquetMorselizer instance.////// # Panics////// Panics if required fields (store, schema/table_schema) are not set.- fn build(self) -> ParquetOpener {+ fn build(self) -> ParquetMorselizer {let store = self.store- .expect("ParquetOpenerBuilder: store must be set via with_store()");+ .expect("ParquetMorselizerBuilder: store must be set via with_store()");let table_schema = self.table_schema.expect(- "ParquetOpenerBuilder: table_schema must be set via with_schema() or with_table_schema()",+ "ParquetMorselizerBuilder: table_schema must be set via with_schema() or with_table_schema()",);let file_schema = Arc::clone(table_schema.file_schema());@@ -1873,7 +1781,7 @@ mod test {ProjectionExprs::from_indices(&all_indices, &file_schema)};- let morselizer = ParquetMorselizer {+ ParquetMorselizer {partition_index: self.partition_index,projection,batch_size: self.batch_size,@@ -1900,8 +1808,45 @@ mod test {encryption_factory: None,max_predicate_cache_size: self.max_predicate_cache_size,reverse_row_groups: self.reverse_row_groups,+ }+ }+ }++ /// Test helper that drives a [`ParquetMorselizer`] to completion and returns+ /// the first stream morsel it produces.+ ///+ /// This mirrors how `FileStream` consumes the morsel APIs: it repeatedly+ /// plans CPU work, awaits any discovered I/O futures, and feeds the planner+ /// back into the ready queue until a stream morsel is ready.+ async fn open_file(+ morselizer: &ParquetMorselizer,+ file: PartitionedFile,+ ) -> Result<BoxStream<'static, Result<RecordBatch>>> {+ let mut planners = VecDeque::from([morselizer.plan_file(file)?]);+ let mut morsels: VecDeque<Box<dyn Morsel>> = VecDeque::new();++ loop {+ if let Some(morsel) = morsels.pop_front() {+ return Ok(Box::pin(morsel.into_stream()));+ }++ let Some(planner) = planners.pop_front() else {+ return Ok(Box::pin(futures::stream::empty()));};- ParquetOpener { morselizer }++ if let Some(mut plan) = planner.plan()? {+ morsels.extend(plan.take_morsels());+ planners.extend(plan.take_ready_planners());++ if let Some(pending_planner) = plan.take_pending_planner() {+ planners.push_front(pending_planner.await?);+ continue;+ }++ if morsels.is_empty() && planners.is_empty() {+ return internal_err!("planner returned an empty morsel plan");+ }+ }}}@@ -1989,12 +1934,7 @@ mod test {}async fn count_batches_and_rows(- mut stream: std::pin::Pin<- Box<- dyn Stream<Item = Result<arrow::array::RecordBatch, DataFusionError>>- + Send,- >,- >,+ mut stream: BoxStream<'static, Result<RecordBatch>>,) -> (usize, usize) {let mut num_batches = 0;let mut num_rows = 0;@@ -2007,12 +1947,7 @@ mod test {/// Helper to collect all int32 values from the first column of batchesasync fn collect_int32_values(- mut stream: std::pin::Pin<- Box<- dyn Stream<Item = Result<arrow::array::RecordBatch, DataFusionError>>- + Send,- >,- >,+ mut stream: BoxStream<'static, Result<RecordBatch>>,) -> Vec<i32> {use arrow::array::Array;let mut values = vec![];@@ -2098,7 +2033,7 @@ mod test {));let make_opener = |predicate| {- ParquetOpenerBuilder::new()+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_schema(Arc::clone(&schema)).with_projection_indices(&[0, 1])@@ -2111,7 +2046,7 @@ mod test {let expr = col("a").eq(lit(1));let predicate = logical2physical(&expr, &schema);let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 1);assert_eq!(num_rows, 3);@@ -2120,7 +2055,7 @@ mod test {let expr = col("b").eq(lit(ScalarValue::Float32(Some(5.0))));let predicate = logical2physical(&expr, &schema);let opener = make_opener(predicate);- let stream = opener.open(file).unwrap().await.unwrap();+ let stream = open_file(&opener, file).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2151,7 +2086,7 @@ mod test {vec![Arc::new(Field::new("part", DataType::Int32, false))],);let make_opener = |predicate| {- ParquetOpenerBuilder::new()+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_table_schema(table_schema_for_opener.clone()).with_projection_indices(&[0])@@ -2166,7 +2101,7 @@ mod test {// Otherwise we assume it already happened at the planning stage and won't re-do the work herelet predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 1);assert_eq!(num_rows, 3);@@ -2177,7 +2112,7 @@ mod test {// Otherwise we assume it already happened at the planning stage and won't re-do the work herelet predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));let opener = make_opener(predicate);- let stream = opener.open(file).unwrap().await.unwrap();+ let stream = open_file(&opener, file).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2220,7 +2155,7 @@ mod test {vec![Arc::new(Field::new("part", DataType::Int32, false))],);let make_opener = |predicate| {- ParquetOpenerBuilder::new()+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_table_schema(table_schema_for_opener.clone()).with_projection_indices(&[0])@@ -2233,7 +2168,7 @@ mod test {let expr = col("part").eq(lit(1)).and(col("b").eq(lit(1.0)));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 1);assert_eq!(num_rows, 3);@@ -2242,7 +2177,7 @@ mod test {let expr = col("part").eq(lit(2)).and(col("b").eq(lit(1.0)));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2251,7 +2186,7 @@ mod test {let expr = col("part").eq(lit(1)).and(col("b").eq(lit(7.0)));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2260,7 +2195,7 @@ mod test {let expr = col("part").eq(lit(2)).and(col("b").eq(lit(7.0)));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);- let stream = opener.open(file).unwrap().await.unwrap();+ let stream = open_file(&opener, file).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2292,7 +2227,7 @@ mod test {vec![Arc::new(Field::new("part", DataType::Int32, false))],);let make_opener = |predicate| {- ParquetOpenerBuilder::new()+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_table_schema(table_schema_for_opener.clone()).with_projection_indices(&[0])@@ -2306,7 +2241,7 @@ mod test {let expr = col("part").eq(lit(1)).or(col("a").eq(lit(1)));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 1);assert_eq!(num_rows, 3);@@ -2315,7 +2250,7 @@ mod test {let expr = col("part").eq(lit(1)).or(col("a").eq(lit(3)));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 1);assert_eq!(num_rows, 3);@@ -2324,7 +2259,7 @@ mod test {let expr = col("part").eq(lit(2)).or(col("a").eq(lit(1)));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 1);assert_eq!(num_rows, 1);@@ -2333,7 +2268,7 @@ mod test {let expr = col("part").eq(lit(2)).or(col("a").eq(lit(3)));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);- let stream = opener.open(file).unwrap().await.unwrap();+ let stream = open_file(&opener, file).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2373,7 +2308,7 @@ mod test {vec![Arc::new(Field::new("part", DataType::Int32, false))],);let make_opener = |predicate| {- ParquetOpenerBuilder::new()+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_table_schema(table_schema_for_opener.clone()).with_projection_indices(&[0])@@ -2386,7 +2321,7 @@ mod test {let expr = col("a").eq(lit(42));let predicate = logical2physical(&expr, &table_schema);let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2395,7 +2330,7 @@ mod test {// This allows dynamic filters to prune partitions/files even if they are populated late into execution.let predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2405,7 +2340,7 @@ mod test {let expr = col("part").eq(lit(2));let predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2414,7 +2349,7 @@ mod test {let expr = col("part").eq(lit(2)).and(col("a").eq(lit(42)));let predicate = make_dynamic_expr(logical2physical(&expr, &table_schema));let opener = make_opener(predicate);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let (num_batches, num_rows) = count_batches_and_rows(stream).await;assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);@@ -2455,7 +2390,7 @@ mod test {);let make_opener = |reverse_scan: bool| {- ParquetOpenerBuilder::new()+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_schema(Arc::clone(&schema)).with_projection_indices(&[0])@@ -2465,12 +2400,12 @@ mod test {// Test normal scan (forward)let opener = make_opener(false);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let forward_values = collect_int32_values(stream).await;// Test reverse scanlet opener = make_opener(true);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let reverse_values = collect_int32_values(stream).await;// The forward scan should return data in the order written@@ -2497,7 +2432,7 @@ mod test {);let make_opener = |reverse_scan: bool| {- ParquetOpenerBuilder::new()+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_schema(Arc::clone(&schema)).with_projection_indices(&[0])@@ -2508,11 +2443,11 @@ mod test {// With a single row group, forward and reverse should be the same// (only the row group order is reversed, not the rows within)let opener_forward = make_opener(false);- let stream_forward = opener_forward.open(file.clone()).unwrap().await.unwrap();+ let stream_forward = open_file(&opener_forward, file.clone()).await.unwrap();let (batches_forward, _) = count_batches_and_rows(stream_forward).await;let opener_reverse = make_opener(true);- let stream_reverse = opener_reverse.open(file).unwrap().await.unwrap();+ let stream_reverse = open_file(&opener_reverse, file).await.unwrap();let (batches_reverse, _) = count_batches_and_rows(stream_reverse).await;// Both should have the same number of batches since there's only one row group@@ -2573,7 +2508,7 @@ mod test {.with_extensions(Arc::new(access_plan));let make_opener = |reverse_scan: bool| {- ParquetOpenerBuilder::new()+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_schema(Arc::clone(&schema)).with_projection_indices(&[0])@@ -2583,7 +2518,7 @@ mod test {// Forward scan: RG0(3,4), RG1(5,6,7,8), RG2(9,10)let opener = make_opener(false);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let forward_values = collect_int32_values(stream).await;// Forward scan should produce: RG0(3,4), RG1(5,6,7,8), RG2(9,10)@@ -2599,7 +2534,7 @@ mod test {// - RG1 is read second, WITH RG1's selection (select all) -> 5, 6, 7, 8// - RG0 is read third, WITH RG0's selection (skip 2, select 2) -> 3, 4let opener = make_opener(true);- let stream = opener.open(file).unwrap().await.unwrap();+ let stream = open_file(&opener, file).await.unwrap();let reverse_values = collect_int32_values(stream).await;// Correct expected result: row groups reversed but each keeps its own selection@@ -2674,7 +2609,7 @@ mod test {.with_extensions(Arc::new(access_plan));let make_opener = |reverse_scan: bool| {- ParquetOpenerBuilder::new()+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_schema(Arc::clone(&schema)).with_projection_indices(&[0])@@ -2685,7 +2620,7 @@ mod test {// Forward scan: RG0(1), RG2(5), RG3(7)// Note: RG1 is completely skippedlet opener = make_opener(false);- let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let stream = open_file(&opener, file.clone()).await.unwrap();let forward_values = collect_int32_values(stream).await;assert_eq!(@@ -2698,7 +2633,7 @@ mod test {// WITHOUT the bug fix, this would return WRONG values// because the RowSelection would be incorrectly mappedlet opener = make_opener(true);- let stream = opener.open(file).unwrap().await.unwrap();+ let stream = open_file(&opener, file).await.unwrap();let reverse_values = collect_int32_values(stream).await;assert_eq!(@@ -2746,8 +2681,8 @@ mod test {// predicate: a > 90 — should allow page index to prune first 9 pageslet predicate = logical2physical(&col("a").gt(lit(90i32)), &schema);- let make_opener = |enable_page_index| {- ParquetOpenerBuilder::new()+ let make_morselizer = |enable_page_index| {+ ParquetMorselizerBuilder::new().with_store(Arc::clone(&store)).with_schema(Arc::clone(&schema)).with_predicate(Arc::clone(&predicate))@@ -2758,12 +2693,15 @@ mod test {.build()};let (_, rows_with_page_index) = count_batches_and_rows(- make_opener(true).open(file.clone()).unwrap().await.unwrap(),+ open_file(&make_morselizer(true), file.clone())+ .await+ .unwrap(),+ )+ .await;+ let (_, rows_without_page_index) = count_batches_and_rows(+ open_file(&make_morselizer(false), file).await.unwrap(),).await;- let (_, rows_without_page_index) =- count_batches_and_rows(make_opener(false).open(file).unwrap().await.unwrap())- .await;assert_eq!(rows_with_page_index, 10,diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 7fad6862fa..3037cb1afe 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -23,14 +23,15 @@ use std::sync::Arc;use crate::DefaultParquetFileReaderFactory;use crate::ParquetFileReaderFactory;+use crate::opener::ParquetMorselizer;use crate::opener::build_pruning_predicates;-use crate::opener::{ParquetMorselizer, ParquetOpener};use crate::row_filter::can_expr_be_pushed_down_with_schemas;use datafusion_common::config::ConfigOptions;#[cfg(feature = "parquet_encryption")]use datafusion_common::config::EncryptionFactoryOptions;use datafusion_datasource::as_file_source;use datafusion_datasource::file_stream::FileOpener;+use datafusion_datasource::morsel::Morselizer;use arrow::datatypes::TimeUnit;use datafusion_common::DataFusionError;@@ -246,12 +247,12 @@ use parquet::encryption::decrypt::FileDecryptionProperties;/// # Execution Overview////// * Step 1: `DataSourceExec::execute` is called, returning a `FileStream`-/// configured to open parquet files with a `ParquetOpener`.+/// configured to morselize parquet files with a `ParquetMorselizer`.///-/// * Step 2: When the stream is polled, the `ParquetOpener` is called to open-/// the file.+/// * Step 2: When the stream is polled, the `ParquetMorselizer` is called to+/// plan the file.///-/// * Step 3: The `ParquetOpener` gets the [`ParquetMetaData`] (file metadata)+/// * Step 3: The `ParquetMorselizer` gets the [`ParquetMetaData`] (file metadata)/// via [`ParquetFileReaderFactory`], creating a `ParquetAccessPlan` by/// applying predicates to metadata. The plan and projections are used to/// determine what pages must be read.@@ -511,11 +512,22 @@ impl From<ParquetSource> for Arc<dyn FileSource> {impl FileSource for ParquetSource {fn create_file_opener(+ &self,+ _object_store: Arc<dyn ObjectStore>,+ _base_config: &FileScanConfig,+ _partition: usize,+ ) -> datafusion_common::Result<Arc<dyn FileOpener>> {+ datafusion_common::internal_err!(+ "ParquetSource::create_file_opener called but it supports the Morsel API, please use that instead"+ )+ }++ fn create_morselizer(&self,object_store: Arc<dyn ObjectStore>,base_config: &FileScanConfig,partition: usize,- ) -> datafusion_common::Result<Arc<dyn FileOpener>> {+ ) -> datafusion_common::Result<Box<dyn Morselizer>> {let expr_adapter_factory = base_config.expr_adapter_factory.clone()@@ -542,37 +554,34 @@ impl FileSource for ParquetSource {.as_ref().map(|time_unit| parse_coerce_int96_string(time_unit.as_str()).unwrap());- let opener = Arc::new(ParquetOpener {- morselizer: ParquetMorselizer {- partition_index: partition,- projection: self.projection.clone(),- batch_size: self- .batch_size- .expect("Batch size must set before creating ParquetOpener"),- limit: base_config.limit,- preserve_order: base_config.preserve_order,- predicate: self.predicate.clone(),- table_schema: self.table_schema.clone(),- metadata_size_hint: self.metadata_size_hint,- metrics: self.metrics().clone(),- parquet_file_reader_factory,- pushdown_filters: self.pushdown_filters(),- reorder_filters: self.reorder_filters(),- force_filter_selections: self.force_filter_selections(),- enable_page_index: self.enable_page_index(),- enable_bloom_filter: self.bloom_filter_on_read(),- enable_row_group_stats_pruning: self.table_parquet_options.global.pruning,- coerce_int96,- #[cfg(feature = "parquet_encryption")]- file_decryption_properties,- expr_adapter_factory,- #[cfg(feature = "parquet_encryption")]- encryption_factory: self.get_encryption_factory_with_config(),- max_predicate_cache_size: self.max_predicate_cache_size(),- reverse_row_groups: self.reverse_row_groups,- },- });- Ok(opener)+ Ok(Box::new(ParquetMorselizer {+ partition_index: partition,+ projection: self.projection.clone(),+ batch_size: self+ .batch_size+ .expect("Batch size must set before creating ParquetMorselizer"),+ limit: base_config.limit,+ preserve_order: base_config.preserve_order,+ predicate: self.predicate.clone(),+ table_schema: self.table_schema.clone(),+ metadata_size_hint: self.metadata_size_hint,+ metrics: self.metrics().clone(),+ parquet_file_reader_factory,+ pushdown_filters: self.pushdown_filters(),+ reorder_filters: self.reorder_filters(),+ force_filter_selections: self.force_filter_selections(),+ enable_page_index: self.enable_page_index(),+ enable_bloom_filter: self.bloom_filter_on_read(),+ enable_row_group_stats_pruning: self.table_parquet_options.global.pruning,+ coerce_int96,+ #[cfg(feature = "parquet_encryption")]+ file_decryption_properties,+ expr_adapter_factory,+ #[cfg(feature = "parquet_encryption")]+ encryption_factory: self.get_encryption_factory_with_config(),+ max_predicate_cache_size: self.max_predicate_cache_size(),+ reverse_row_groups: self.reverse_row_groups,+ }))}fn as_any(&self) -> &dyn Any {diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.tomlindex 1315f871a6..4027521658 100644--- a/datafusion/datasource/Cargo.toml+++ b/datafusion/datasource/Cargo.toml@@ -31,6 +31,7 @@ version.workspace = trueall-features = true[features]+backtrace = ["datafusion-common/backtrace"]compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"]default = ["compression"]@@ -72,6 +73,7 @@ zstd = { workspace = true, optional = true }[dev-dependencies]criterion = { workspace = true }+insta = { workspace = true }tempfile = { workspace = true }# Note: add additional linter rules in lib.rs.diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex b5a6760cae..9c43eeca2b 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -25,6 +25,7 @@ use std::sync::Arc;use crate::file_groups::FileGroupPartitioner;use crate::file_scan_config::FileScanConfig;use crate::file_stream::FileOpener;+use crate::morsel::{FileOpenerMorselizer, Morselizer};#[expect(deprecated)]use crate::schema_adapter::SchemaAdapterFactory;use datafusion_common::config::ConfigOptions;@@ -63,13 +64,33 @@ pub fn as_file_source<T: FileSource + 'static>(source: T) -> Arc<dyn FileSource>////// [`DataSource`]: crate::source::DataSourcepub trait FileSource: Send + Sync {- /// Creates a `dyn FileOpener` based on given parameters+ /// Creates a `dyn FileOpener` based on given parameters.+ ///+ /// Note: File sources with a native morsel implementation should return an+ /// error from this method and implementing [`Self::create_morselizer`] instead.fn create_file_opener(&self,object_store: Arc<dyn ObjectStore>,base_config: &FileScanConfig,partition: usize,) -> Result<Arc<dyn FileOpener>>;++ /// Creates a `dyn Morselizer` based on given parameters.+ ///+ /// The default implementation preserves existing behavior by adapting the+ /// legacy [`FileOpener`] API into a [`Morselizer`].+ ///+ /// It is preferred to implement the [`Morselizer`] API directly by+ /// implementing this method.+ fn create_morselizer(+ &self,+ object_store: Arc<dyn ObjectStore>,+ base_config: &FileScanConfig,+ partition: usize,+ ) -> Result<Box<dyn Morselizer>> {+ let opener = self.create_file_opener(object_store, base_config, partition)?;+ Ok(Box::new(FileOpenerMorselizer::new(opener)))+ }/// Anyfn as_any(&self) -> &dyn Any;diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex cb39e44e7b..f4e4e0a4de 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -587,11 +587,11 @@ impl DataSource for FileScanConfig {let source = self.file_source.with_batch_size(batch_size);- let opener = source.create_file_opener(object_store, self, partition)?;+ let morselizer = source.create_morselizer(object_store, self, partition)?;let stream = FileStreamBuilder::new(self).with_partition(partition)- .with_file_opener(opener)+ .with_morselizer(morselizer).with_metrics(source.metrics()).build()?;Ok(Box::pin(cooperative(stream)))diff --git a/datafusion/datasource/src/file_stream/builder.rs b/datafusion/datasource/src/file_stream/builder.rsindex 6d99f4b56a..efe9c39ce3 100644--- a/datafusion/datasource/src/file_stream/builder.rs+++ b/datafusion/datasource/src/file_stream/builder.rs@@ -18,6 +18,8 @@use std::sync::Arc;use crate::file_scan_config::FileScanConfig;+use crate::file_stream::scan_state::ScanState;+use crate::morsel::{FileOpenerMorselizer, Morselizer};use datafusion_common::{Result, internal_err};use datafusion_physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet};@@ -28,7 +30,7 @@ use super::{FileOpener, FileStream, FileStreamState, OnError};pub struct FileStreamBuilder<'a> {config: &'a FileScanConfig,partition: Option<usize>,- file_opener: Option<Arc<dyn FileOpener>>,+ morselizer: Option<Box<dyn Morselizer>>,metrics: Option<&'a ExecutionPlanMetricsSet>,on_error: OnError,}@@ -39,7 +41,7 @@ impl<'a> FileStreamBuilder<'a> {Self {config,partition: None,- file_opener: None,+ morselizer: None,metrics: None,on_error: OnError::Fail,}@@ -52,8 +54,18 @@ impl<'a> FileStreamBuilder<'a> {}/// Configure the [`FileOpener`] used to open files.+ ///+ /// This will overwrite any setting from [`Self::with_morselizer`]pub fn with_file_opener(mut self, file_opener: Arc<dyn FileOpener>) -> Self {- self.file_opener = Some(file_opener);+ self.morselizer = Some(Box::new(FileOpenerMorselizer::new(file_opener)));+ self+ }++ /// Configure the [`Morselizer`] used to open files.+ ///+ /// This will overwrite any setting from [`Self::with_file_opener`]+ pub fn with_morselizer(mut self, morselizer: Box<dyn Morselizer>) -> Self {+ self.morselizer = Some(morselizer);self}@@ -74,7 +86,7 @@ impl<'a> FileStreamBuilder<'a> {let Self {config,partition,- file_opener,+ morselizer,metrics,on_error,} = self;@@ -82,8 +94,8 @@ impl<'a> FileStreamBuilder<'a> {let Some(partition) = partition else {return internal_err!("FileStreamBuilder missing required partition");};- let Some(file_opener) = file_opener else {- return internal_err!("FileStreamBuilder missing required file_opener");+ let Some(morselizer) = morselizer else {+ return internal_err!("FileStreamBuilder missing required morselizer");};let Some(metrics) = metrics else {return internal_err!("FileStreamBuilder missing required metrics");@@ -95,15 +107,19 @@ impl<'a> FileStreamBuilder<'a> {);};+ let file_stream_metrics = FileStreamMetrics::new(metrics, partition);+ let scan_state = Box::new(ScanState::new(+ file_group.into_inner(),+ config.limit,+ morselizer,+ on_error,+ file_stream_metrics,+ ));+Ok(FileStream {- file_iter: file_group.into_inner().into_iter().collect(),projected_schema,- remain: config.limit,- file_opener,- state: FileStreamState::Idle,- file_stream_metrics: FileStreamMetrics::new(metrics, partition),+ state: FileStreamState::Scan { scan_state },baseline_metrics: BaselineMetrics::new(metrics, partition),- on_error,})}}diff --git a/datafusion/datasource/src/file_stream/metrics.rs b/datafusion/datasource/src/file_stream/metrics.rsindex f4dddeaee8..5f3894404f 100644--- a/datafusion/datasource/src/file_stream/metrics.rs+++ b/datafusion/datasource/src/file_stream/metrics.rs@@ -77,7 +77,7 @@ pub struct FileStreamMetrics {/// Wall clock time elapsed for data decompression + decoding////// Time spent waiting for the FileStream's input.- pub time_processing: StartableTime,+ pub time_processing: Time,/// Count of errors opening file.////// If using `OnError::Skip` this will provide a count of the number of files@@ -126,11 +126,8 @@ impl FileStreamMetrics {start: None,};- let time_processing = StartableTime {- metrics: MetricBuilder::new(metrics)- .subset_time("time_elapsed_processing", partition),- start: None,- };+ let time_processing =+ MetricBuilder::new(metrics).subset_time("time_elapsed_processing", partition);let file_open_errors = MetricBuilder::new(metrics).with_category(MetricCategory::Rows)diff --git a/datafusion/datasource/src/file_stream/mod.rs b/datafusion/datasource/src/file_stream/mod.rsindex 33e5065cb5..ff71f16023 100644--- a/datafusion/datasource/src/file_stream/mod.rs+++ b/datafusion/datasource/src/file_stream/mod.rs@@ -23,8 +23,8 @@mod builder;mod metrics;+mod scan_state;-use std::collections::VecDeque;use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};@@ -38,33 +38,24 @@ use datafusion_physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSetuse arrow::record_batch::RecordBatch;+use futures::Stream;use futures::future::BoxFuture;use futures::stream::BoxStream;-use futures::{FutureExt as _, Stream, StreamExt as _, ready};++use self::scan_state::{ScanAndReturn, ScanState};pub use builder::FileStreamBuilder;pub use metrics::{FileStreamMetrics, StartableTime};/// A stream that iterates record batch by record batch, file over file.pub struct FileStream {- /// An iterator over input files.- file_iter: VecDeque<PartitionedFile>,/// The stream schema (file schema including partition columns and after/// projection).projected_schema: SchemaRef,- /// The remaining number of records to parse, None if no limit- remain: Option<usize>,- /// A dynamic [`FileOpener`]. Calling `open()` returns a [`FileOpenFuture`],- /// which can be resolved to a stream of `RecordBatch`.- file_opener: Arc<dyn FileOpener>,/// The stream statestate: FileStreamState,- /// File stream specific metrics- file_stream_metrics: FileStreamMetrics,/// runtime baseline metricsbaseline_metrics: BaselineMetrics,- /// Describes the behavior of the `FileStream` if file opening or scanning fails- on_error: OnError,}impl FileStream {@@ -88,105 +79,34 @@ impl FileStream {/// If `OnError::Skip` the stream will skip files which encounter an error and continue/// If `OnError:Fail` (default) the stream will fail and stop processing when an error occurspub fn with_on_error(mut self, on_error: OnError) -> Self {- self.on_error = on_error;+ match &mut self.state {+ FileStreamState::Scan { scan_state } => scan_state.set_on_error(on_error),+ FileStreamState::Error | FileStreamState::Done => {+ // no effect as there are no more files to process+ }+ };self}- fn start_next_file(&mut self) -> Option<Result<FileOpenFuture>> {- let part_file = self.file_iter.pop_front()?;- Some(self.file_opener.open(part_file))- }-fn poll_inner(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<RecordBatch>>> {loop {match &mut self.state {- FileStreamState::Idle => match self.start_next_file().transpose() {- Ok(Some(future)) => {- self.file_stream_metrics.time_opening.start();- self.state = FileStreamState::Open { future };- }- Ok(None) => return Poll::Ready(None),- Err(e) => {- self.state = FileStreamState::Error;- return Poll::Ready(Some(Err(e)));- }- },- FileStreamState::Open { future } => match ready!(future.poll_unpin(cx)) {- Ok(reader) => {- self.file_stream_metrics.files_opened.add(1);- self.file_stream_metrics.time_opening.stop();- self.file_stream_metrics.time_scanning_until_data.start();- self.file_stream_metrics.time_scanning_total.start();- self.state = FileStreamState::Scan { reader };- }- Err(e) => {- self.file_stream_metrics.file_open_errors.add(1);- match self.on_error {- OnError::Skip => {- self.file_stream_metrics.files_processed.add(1);- self.file_stream_metrics.time_opening.stop();- self.state = FileStreamState::Idle- }- OnError::Fail => {- self.state = FileStreamState::Error;- return Poll::Ready(Some(Err(e)));- }- }- }- },- FileStreamState::Scan { reader } => {- match ready!(reader.poll_next_unpin(cx)) {- Some(Ok(batch)) => {- self.file_stream_metrics.time_scanning_until_data.stop();- self.file_stream_metrics.time_scanning_total.stop();- let batch = match &mut self.remain {- Some(remain) => {- if *remain > batch.num_rows() {- *remain -= batch.num_rows();- batch- } else {- let batch = batch.slice(0, *remain);- // Count this file and all remaining files- // we will never open.- let done = 1 + self.file_iter.len();- self.file_stream_metrics- .files_processed- .add(done);- self.state = FileStreamState::Limit;- *remain = 0;- batch- }- }- None => batch,- };- self.file_stream_metrics.time_scanning_total.start();- return Poll::Ready(Some(Ok(batch)));- }- Some(Err(err)) => {- self.file_stream_metrics.file_scan_errors.add(1);- self.file_stream_metrics.time_scanning_until_data.stop();- self.file_stream_metrics.time_scanning_total.stop();-- match self.on_error {- OnError::Skip => {- self.file_stream_metrics.files_processed.add(1);- self.state = FileStreamState::Idle;- }- OnError::Fail => {- self.state = FileStreamState::Error;- return Poll::Ready(Some(Err(err)));- }- }+ FileStreamState::Scan { scan_state: queue } => {+ let action = queue.poll_scan(cx);+ match action {+ ScanAndReturn::Continue => continue,+ ScanAndReturn::Done(result) => {+ self.state = FileStreamState::Done;+ return Poll::Ready(result);}- None => {- self.file_stream_metrics.files_processed.add(1);- self.file_stream_metrics.time_scanning_until_data.stop();- self.file_stream_metrics.time_scanning_total.stop();- self.state = FileStreamState::Idle;+ ScanAndReturn::Error(err) => {+ self.state = FileStreamState::Error;+ return Poll::Ready(Some(Err(err)));}+ ScanAndReturn::Return(result) => return result,}}- FileStreamState::Error | FileStreamState::Limit => {+ FileStreamState::Error | FileStreamState::Done => {return Poll::Ready(None);}}@@ -201,9 +121,7 @@ impl Stream for FileStream {mut self: Pin<&mut Self>,cx: &mut Context<'_>,) -> Poll<Option<Self::Item>> {- self.file_stream_metrics.time_processing.start();let result = self.poll_inner(cx);- self.file_stream_metrics.time_processing.stop();self.baseline_metrics.record_poll(result)}}@@ -238,33 +156,30 @@ pub trait FileOpener: Unpin + Send + Sync {fn open(&self, partitioned_file: PartitionedFile) -> Result<FileOpenFuture>;}-pub enum FileStreamState {- /// The idle state, no file is currently being read- Idle,- /// Currently performing asynchronous IO to obtain a stream of RecordBatch- /// for a given file- Open {- /// A [`FileOpenFuture`] returned by [`FileOpener::open`]- future: FileOpenFuture,- },- /// Scanning the [`BoxStream`] returned by the completion of a [`FileOpenFuture`]- /// returned by [`FileOpener::open`]+enum FileStreamState {+ /// Actively processing readers, ready morsels, and planner work.Scan {- /// The reader instance- reader: BoxStream<'static, Result<RecordBatch>>,+ /// The ready queues and active reader for the current file.+ scan_state: Box<ScanState>,},/// Encountered an errorError,- /// Reached the row limit- Limit,+ /// Finished scanning all requested data, possibly because a limit was reached+ Done,}-/// A timer that can be started and stopped.#[cfg(test)]mod tests {use crate::file_scan_config::{FileScanConfig, FileScanConfigBuilder};+ use crate::morsel::mocks::{+ IoFutureId, MockMorselizer, MockPlanBuilder, MockPlanner, MorselId,+ PendingPlannerBuilder, PollsToResolve,+ };use crate::tests::make_partition;use crate::{PartitionedFile, TableSchema};+ use arrow::array::{AsArray, RecordBatch};+ use arrow::datatypes::{DataType, Field, Int32Type, Schema};+ use datafusion_common::DataFusionError;use datafusion_common::error::Result;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;@@ -274,8 +189,6 @@ mod tests {use crate::file_stream::{FileOpenFuture, FileOpener, FileStreamBuilder, OnError};use crate::test_util::MockSource;- use arrow::array::RecordBatch;- use arrow::datatypes::Schema;use datafusion_common::{assert_batches_eq, exec_err, internal_err};@@ -747,7 +660,7 @@ mod tests {assert!(err.contains("FileStreamBuilder missing required partition"));let err = builder_error(FileStreamBuilder::new(&config).with_partition(0));- assert!(err.contains("FileStreamBuilder missing required file_opener"));+ assert!(err.contains("FileStreamBuilder missing required morselizer"));let err = builder_error(FileStreamBuilder::new(&config)@@ -770,4 +683,417 @@ mod tests {);assert!(err.contains("FileStreamBuilder invalid partition index: 1"));}++ /// Verifies the simplest morsel-driven flow: one planner produces one+ /// morsel immediately, and that morsel is then scanned to completion.+ #[tokio::test]+ async fn morsel_no_io() -> Result<()> {+ let test = FileStreamMorselTest::new().with_file(+ MockPlanner::builder("file1.parquet")+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 42))+ .return_none(),+ );++ insta::assert_snapshot!(test.run().await.unwrap(), @r"+ ----- Output Stream -----+ Batch: 42+ Done+ ----- File Stream Events -----+ morselize_file: file1.parquet+ planner_created: file1.parquet+ planner_called: file1.parquet+ morsel_produced: file1.parquet, MorselId(10)+ morsel_stream_started: MorselId(10)+ morsel_stream_batch_produced: MorselId(10), BatchId(42)+ morsel_stream_finished: MorselId(10)+ ");++ Ok(())+ }++ /// Verifies that a planner can block on one I/O phase and then produce a+ /// morsel containing two batches.+ #[tokio::test]+ async fn morsel_single_io_two_batches() -> Result<()> {+ let test = FileStreamMorselTest::new().with_file(+ MockPlanner::builder("file1.parquet")+ .add_plan(+ PendingPlannerBuilder::new(IoFutureId(1))+ .with_polls_to_resolve(PollsToResolve(1)),+ )+ .add_plan(+ MockPlanBuilder::new()+ .with_morsel_batches(MorselId(10), vec![42, 43]),+ )+ .return_none(),+ );++ insta::assert_snapshot!(test.run().await.unwrap(), @r"+ ----- Output Stream -----+ Batch: 42+ Batch: 43+ Done+ ----- File Stream Events -----+ morselize_file: file1.parquet+ planner_created: file1.parquet+ planner_called: file1.parquet+ io_future_created: file1.parquet, IoFutureId(1)+ io_future_polled: file1.parquet, IoFutureId(1)+ io_future_polled: file1.parquet, IoFutureId(1)+ io_future_resolved: file1.parquet, IoFutureId(1)+ planner_called: file1.parquet+ morsel_produced: file1.parquet, MorselId(10)+ morsel_stream_started: MorselId(10)+ morsel_stream_batch_produced: MorselId(10), BatchId(42)+ morsel_stream_batch_produced: MorselId(10), BatchId(43)+ morsel_stream_finished: MorselId(10)+ ");++ Ok(())+ }++ /// Verifies that a planner can traverse two sequential I/O phases before+ /// producing one batch, similar to Parquet.+ #[tokio::test]+ async fn morsel_two_ios_one_batch() -> Result<()> {+ let test = FileStreamMorselTest::new().with_file(+ MockPlanner::builder("file1.parquet")+ .add_plan(PendingPlannerBuilder::new(IoFutureId(1)).build())+ .add_plan(PendingPlannerBuilder::new(IoFutureId(2)).build())+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 42))+ .return_none(),+ );++ insta::assert_snapshot!(test.run().await.unwrap(), @r"+ ----- Output Stream -----+ Batch: 42+ Done+ ----- File Stream Events -----+ morselize_file: file1.parquet+ planner_created: file1.parquet+ planner_called: file1.parquet+ io_future_created: file1.parquet, IoFutureId(1)+ io_future_polled: file1.parquet, IoFutureId(1)+ io_future_resolved: file1.parquet, IoFutureId(1)+ planner_called: file1.parquet+ io_future_created: file1.parquet, IoFutureId(2)+ io_future_polled: file1.parquet, IoFutureId(2)+ io_future_resolved: file1.parquet, IoFutureId(2)+ planner_called: file1.parquet+ morsel_produced: file1.parquet, MorselId(10)+ morsel_stream_started: MorselId(10)+ morsel_stream_batch_produced: MorselId(10), BatchId(42)+ morsel_stream_finished: MorselId(10)+ ");++ Ok(())+ }++ /// Verifies that a planner I/O future can fail and terminate the stream.+ #[tokio::test]+ async fn morsel_io_error() -> Result<()> {+ let test = FileStreamMorselTest::new().with_file(+ MockPlanner::builder("file1.parquet").add_plan(+ PendingPlannerBuilder::new(IoFutureId(1))+ .with_error("io failed while opening file"),+ ),+ );++ insta::assert_snapshot!(test.run().await.unwrap(), @r"+ ----- Output Stream -----+ Error: io failed while opening file+ Done+ ----- File Stream Events -----+ morselize_file: file1.parquet+ planner_created: file1.parquet+ planner_called: file1.parquet+ io_future_created: file1.parquet, IoFutureId(1)+ io_future_polled: file1.parquet, IoFutureId(1)+ io_future_errored: file1.parquet, IoFutureId(1), io failed while opening file+ ");++ Ok(())+ }++ /// Verifies that pending planner I/O does not block draining the current+ /// morsel stream.+ #[tokio::test]+ async fn morsel_pending_planner_does_not_block_active_reader() -> Result<()> {+ let test = FileStreamMorselTest::new().with_file(+ MockPlanner::builder("file1.parquet")+ .add_plan(+ MockPlanBuilder::new()+ .with_morsel_batches(MorselId(10), vec![41, 42])+ .with_pending_planner(IoFutureId(1), PollsToResolve(3), Ok(())),+ )+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(11), 43))+ .return_none(),+ );++ // The key events are:+ // 1. the first `planner_called` produces `MorselId(10)` and creates `IoFutureId(1)`+ // 2. `MorselId(10)` continues yielding both batches while that I/O is pending+ // 3. after the I/O resolves, planning resumes and yields `MorselId(11)`+ insta::assert_snapshot!(test.run().await.unwrap(), @r"+ ----- Output Stream -----+ Batch: 41+ Batch: 42+ Batch: 43+ Done+ ----- File Stream Events -----+ morselize_file: file1.parquet+ planner_created: file1.parquet+ planner_called: file1.parquet+ morsel_produced: file1.parquet, MorselId(10)+ io_future_created: file1.parquet, IoFutureId(1)+ io_future_polled: file1.parquet, IoFutureId(1)+ morsel_stream_started: MorselId(10)+ io_future_polled: file1.parquet, IoFutureId(1)+ morsel_stream_batch_produced: MorselId(10), BatchId(41)+ io_future_polled: file1.parquet, IoFutureId(1)+ morsel_stream_batch_produced: MorselId(10), BatchId(42)+ io_future_polled: file1.parquet, IoFutureId(1)+ io_future_resolved: file1.parquet, IoFutureId(1)+ morsel_stream_finished: MorselId(10)+ planner_called: file1.parquet+ morsel_produced: file1.parquet, MorselId(11)+ morsel_stream_started: MorselId(11)+ morsel_stream_batch_produced: MorselId(11), BatchId(43)+ morsel_stream_finished: MorselId(11)+ ");++ Ok(())+ }++ /// Verifies that one `plan()` call can return a ready child planner, which+ /// is then called to produce the morsel.+ #[tokio::test]+ async fn morsel_ready_child_planner() -> Result<()> {+ let child_planner = MockPlanner::builder("child planner")+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 42))+ .return_none()+ .build();++ let test = FileStreamMorselTest::new().with_file(+ MockPlanner::builder("file1.parquet")+ .add_plan(MockPlanBuilder::new().with_ready_planner(child_planner))+ .return_none(),+ );++ insta::assert_snapshot!(test.run().await.unwrap(), @r"+ ----- Output Stream -----+ Batch: 42+ Done+ ----- File Stream Events -----+ morselize_file: file1.parquet+ planner_created: file1.parquet+ planner_called: file1.parquet+ planner_created: child planner+ planner_called: child planner+ morsel_produced: child planner, MorselId(10)+ morsel_stream_started: MorselId(10)+ morsel_stream_batch_produced: MorselId(10), BatchId(42)+ morsel_stream_finished: MorselId(10)+ ");++ Ok(())+ }++ /// Verifies that planning can fail after a successful I/O phase.+ #[tokio::test]+ async fn morsel_plan_error_after_io() -> Result<()> {+ let test = FileStreamMorselTest::new().with_file(+ MockPlanner::builder("file1.parquet")+ .add_plan(PendingPlannerBuilder::new(IoFutureId(1)))+ .return_error("planner failed after io"),+ );++ insta::assert_snapshot!(test.run().await.unwrap(), @r"+ ----- Output Stream -----+ Error: planner failed after io+ Done+ ----- File Stream Events -----+ morselize_file: file1.parquet+ planner_created: file1.parquet+ planner_called: file1.parquet+ io_future_created: file1.parquet, IoFutureId(1)+ io_future_polled: file1.parquet, IoFutureId(1)+ io_future_resolved: file1.parquet, IoFutureId(1)+ planner_called: file1.parquet+ ");++ Ok(())+ }++ /// Verifies that `FileStream` scans multiple files in order.+ #[tokio::test]+ async fn morsel_multiple_files() -> Result<()> {+ let test = FileStreamMorselTest::new()+ .with_file(+ MockPlanner::builder("file1.parquet")+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(10), 41))+ .return_none(),+ )+ .with_file(+ MockPlanner::builder("file2.parquet")+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(11), 42))+ .return_none(),+ );++ insta::assert_snapshot!(test.run().await.unwrap(), @r"+ ----- Output Stream -----+ Batch: 41+ Batch: 42+ Done+ ----- File Stream Events -----+ morselize_file: file1.parquet+ planner_created: file1.parquet+ planner_called: file1.parquet+ morsel_produced: file1.parquet, MorselId(10)+ morsel_stream_started: MorselId(10)+ morsel_stream_batch_produced: MorselId(10), BatchId(41)+ morsel_stream_finished: MorselId(10)+ morselize_file: file2.parquet+ planner_created: file2.parquet+ planner_called: file2.parquet+ morsel_produced: file2.parquet, MorselId(11)+ morsel_stream_started: MorselId(11)+ morsel_stream_batch_produced: MorselId(11), BatchId(42)+ morsel_stream_finished: MorselId(11)+ ");++ Ok(())+ }++ /// Verifies that a global limit can stop the stream before a second file is opened.+ #[tokio::test]+ async fn morsel_limit_prevents_second_file() -> Result<()> {+ let test = FileStreamMorselTest::new()+ .with_file(+ MockPlanner::builder("file1.parquet")+ .add_plan(+ MockPlanBuilder::new()+ .with_morsel_batches(MorselId(10), vec![41, 42]),+ )+ .return_none(),+ )+ .with_file(+ MockPlanner::builder("file2.parquet")+ .add_plan(MockPlanBuilder::new().with_morsel(MorselId(11), 43))+ .return_none(),+ )+ .with_limit(1);++ // Note the snapshot should not ever see planner id2+ insta::assert_snapshot!(test.run().await.unwrap(), @r"+ ----- Output Stream -----+ Batch: 41+ Done+ ----- File Stream Events -----+ morselize_file: file1.parquet+ planner_created: file1.parquet+ planner_called: file1.parquet+ morsel_produced: file1.parquet, MorselId(10)+ morsel_stream_started: MorselId(10)+ morsel_stream_batch_produced: MorselId(10), BatchId(41)+ ");++ Ok(())+ }++ /// Tests how FileStream opens and processes files.+ #[derive(Clone)]+ struct FileStreamMorselTest {+ morselizer: MockMorselizer,+ file_names: Vec<String>,+ limit: Option<usize>,+ }++ impl FileStreamMorselTest {+ /// Creates an empty test harness.+ fn new() -> Self {+ Self {+ morselizer: MockMorselizer::new(),+ file_names: vec![],+ limit: None,+ }+ }++ /// Adds one file and its root planner to the test input.+ fn with_file(mut self, planner: impl Into<MockPlanner>) -> Self {+ let planner = planner.into();+ self.file_names.push(planner.file_path().to_string());+ self.morselizer = self.morselizer.with_file(planner);+ self+ }++ /// Sets a global output limit for the stream.+ fn with_limit(mut self, limit: usize) -> Self {+ self.limit = Some(limit);+ self+ }++ /// Runs the test returns combined output and scheduler trace text as a String.+ async fn run(self) -> Result<String> {+ let observer = self.morselizer.observer().clone();+ observer.clear();++ let config = self.test_config();+ let metrics_set = ExecutionPlanMetricsSet::new();+ let mut stream = FileStreamBuilder::new(&config)+ .with_partition(0)+ .with_morselizer(Box::new(self.morselizer))+ .with_metrics(&metrics_set)+ .build()?;++ let mut stream_contents = Vec::new();+ while let Some(result) = stream.next().await {+ match result {+ Ok(batch) => {+ let col = batch.column(0).as_primitive::<Int32Type>();+ let batch_id = col.value(0);+ stream_contents.push(format!("Batch: {batch_id}"));+ }+ Err(e) => {+ // Pull the actual message for external errors rather than+ // relying on DataFusionError formatting, which changes+ // if backtraces are enabled, etc+ let message = if let DataFusionError::External(generic) = e {+ generic.to_string()+ } else {+ e.to_string()+ };+ stream_contents.push(format!("Error: {message}"));+ }+ }+ }+ stream_contents.push("Done".to_string());++ Ok(format!(+ "----- Output Stream -----\n{}\n----- File Stream Events -----\n{}",+ stream_contents.join("\n"),+ observer.format_events()+ ))+ }++ /// Builds the `FileScanConfig` for the configured mock file set.+ fn test_config(&self) -> FileScanConfig {+ let file_group = self+ .file_names+ .iter()+ .map(|name| PartitionedFile::new(name, 10))+ .collect();+ let table_schema = TableSchema::new(+ Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)])),+ vec![],+ );+ FileScanConfigBuilder::new(+ ObjectStoreUrl::parse("test:///").unwrap(),+ Arc::new(MockSource::new(table_schema)),+ )+ .with_file_group(file_group)+ .with_limit(self.limit)+ .build()+ }+ }}diff --git a/datafusion/datasource/src/file_stream/scan_state.rs b/datafusion/datasource/src/file_stream/scan_state.rsnew file mode 100644index 0000000000..025164c29c--- /dev/null+++ b/datafusion/datasource/src/file_stream/scan_state.rs@@ -0,0 +1,302 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use datafusion_common::internal_datafusion_err;+use std::collections::VecDeque;+use std::task::{Context, Poll};++use crate::PartitionedFile;+use crate::morsel::{Morsel, MorselPlanner, Morselizer, PendingMorselPlanner};+use arrow::record_batch::RecordBatch;+use datafusion_common::{DataFusionError, Result};+use datafusion_physical_plan::metrics::ScopedTimerGuard;+use futures::stream::BoxStream;+use futures::{FutureExt as _, StreamExt as _};++use super::{FileStreamMetrics, OnError};++/// State [`FileStreamState::Scan`].+///+/// There is one `ScanState` per `FileStream`, and thus per output partition.+///+/// It groups together the lifecycle of scanning that partition's files:+/// unopened files, CPU-ready planners, pending planner I/O, ready morsels,+/// the active reader, and the metrics associated with processing that work.+///+/// # I/O+///+/// To avoid challenges controlling buffering, the ScanState only ever has a+/// single I/O outstanding at any time.+///+/// # State Transitions+///+/// ```text+/// file_iter+/// |+/// v+/// morselizer.plan_file(file)+/// |+/// v+/// ready_planners ---> plan() ---> ready_morsels ---> into_stream() ---> reader ---> RecordBatches+/// ^ |+/// | v+/// | pending_planner+/// | |+/// | v+/// +-------- poll until ready+/// ```+///+/// [`FileStreamState::Scan`]: super::FileStreamState::Scan+pub(super) struct ScanState {+ /// Files that still need to be planned.+ file_iter: VecDeque<PartitionedFile>,+ /// Remaining row limit, if any.+ remain: Option<usize>,+ /// The morselizer used to plan files.+ morselizer: Box<dyn Morselizer>,+ /// Behavior if opening or scanning a file fails.+ on_error: OnError,+ /// CPU-ready planners for the current file.+ ready_planners: VecDeque<Box<dyn MorselPlanner>>,+ /// Ready morsels for the current file.+ ready_morsels: VecDeque<Box<dyn Morsel>>,+ /// The active reader, if any.+ reader: Option<BoxStream<'static, Result<RecordBatch>>>,+ /// The currently outstanding I/O, if any.+ pending_planner: Option<PendingMorselPlanner>,+ /// Metrics for the active scan queues.+ metrics: FileStreamMetrics,+}++impl ScanState {+ pub(super) fn new(+ file_iter: impl Into<VecDeque<PartitionedFile>>,+ remain: Option<usize>,+ morselizer: Box<dyn Morselizer>,+ on_error: OnError,+ metrics: FileStreamMetrics,+ ) -> Self {+ let file_iter = file_iter.into();+ Self {+ file_iter,+ remain,+ morselizer,+ on_error,+ ready_planners: Default::default(),+ ready_morsels: Default::default(),+ reader: None,+ pending_planner: None,+ metrics,+ }+ }++ /// Updates how scan errors are handled while the stream is still active.+ pub(super) fn set_on_error(&mut self, on_error: OnError) {+ self.on_error = on_error;+ }++ /// Drives one iteration of the active scan state.+ ///+ /// Work is attempted in this order:+ /// 1. resolve any pending planner I/O+ /// 2. poll the active reader+ /// 3. turn a ready morsel into the active reader+ /// 4. run CPU planning on a ready planner+ /// 5. morselize the next unopened file+ ///+ /// The return [`ScanAndReturn`] tells `poll_inner` how to update the+ /// outer `FileStreamState`.+ pub(super) fn poll_scan(&mut self, cx: &mut Context<'_>) -> ScanAndReturn {+ let _processing_timer: ScopedTimerGuard<'_> =+ self.metrics.time_processing.timer();++ // Try and resolve outstanding IO first. If it is still pending, check+ // the current reader or ready morsels before yielding. New planning+ // work must still wait for this I/O to resolve.+ if let Some(mut pending_planner) = self.pending_planner.take() {+ match pending_planner.poll_unpin(cx) {+ // IO is still pending+ Poll::Pending => {+ self.pending_planner = Some(pending_planner);+ }+ // IO resolved, and the planner is ready for CPU work+ Poll::Ready(Ok(planner)) => {+ self.ready_planners.push_back(planner);+ }+ // IO Error+ Poll::Ready(Err(err)) => {+ self.metrics.file_open_errors.add(1);+ self.metrics.time_opening.stop();+ return match self.on_error {+ OnError::Skip => {+ self.metrics.files_processed.add(1);+ ScanAndReturn::Continue+ }+ OnError::Fail => ScanAndReturn::Error(err),+ };+ }+ }+ }++ // Next try and get the next batch from the active reader, if any.+ if let Some(reader) = self.reader.as_mut() {+ match reader.poll_next_unpin(cx) {+ // Morsels should ideally only expose ready-to-decode streams,+ // but tolerate pending readers here.+ Poll::Pending => return ScanAndReturn::Return(Poll::Pending),+ Poll::Ready(Some(Ok(batch))) => {+ self.metrics.time_scanning_until_data.stop();+ self.metrics.time_scanning_total.stop();+ // Apply any remaining row limit.+ let (batch, finished) = match &mut self.remain {+ Some(remain) => {+ if *remain > batch.num_rows() {+ *remain -= batch.num_rows();+ self.metrics.time_scanning_total.start();+ (batch, false)+ } else {+ let batch = batch.slice(0, *remain);+ let done = 1 + self.file_iter.len();+ self.metrics.files_processed.add(done);+ *remain = 0;+ (batch, true)+ }+ }+ None => {+ self.metrics.time_scanning_total.start();+ (batch, false)+ }+ };+ return if finished {+ ScanAndReturn::Done(Some(Ok(batch)))+ } else {+ ScanAndReturn::Return(Poll::Ready(Some(Ok(batch))))+ };+ }+ Poll::Ready(Some(Err(err))) => {+ self.reader = None;+ self.metrics.file_scan_errors.add(1);+ self.metrics.time_scanning_until_data.stop();+ self.metrics.time_scanning_total.stop();+ return match self.on_error {+ OnError::Skip => {+ self.metrics.files_processed.add(1);+ ScanAndReturn::Continue+ }+ OnError::Fail => ScanAndReturn::Error(err),+ };+ }+ Poll::Ready(None) => {+ self.reader = None;+ self.metrics.files_processed.add(1);+ self.metrics.time_scanning_until_data.stop();+ self.metrics.time_scanning_total.stop();+ return ScanAndReturn::Continue;+ }+ }+ }++ // No active reader, but a morsel is ready to become the reader.+ if let Some(morsel) = self.ready_morsels.pop_front() {+ self.metrics.time_opening.stop();+ self.metrics.time_scanning_until_data.start();+ self.metrics.time_scanning_total.start();+ self.reader = Some(morsel.into_stream());+ return ScanAndReturn::Continue;+ }++ // Do not start CPU planning or open another file while planner I/O is+ // still outstanding because they may need additional IO and ScanState+ // currently only permits a single outstanding IO+ if self.pending_planner.is_some() {+ return ScanAndReturn::Return(Poll::Pending);+ }++ // No reader or morsel, so try to produce more work via CPU planning.+ if let Some(planner) = self.ready_planners.pop_front() {+ return match planner.plan() {+ Ok(Some(mut plan)) => {+ // Queue any newly-ready morsels, planners, or planner I/O.+ self.ready_morsels.extend(plan.take_morsels());+ self.ready_planners.extend(plan.take_ready_planners());+ if let Some(pending_planner) = plan.take_pending_planner() {+ // should not have planned if we have outstanding I/O+ if self.pending_planner.is_some() {+ return ScanAndReturn::Error(internal_datafusion_err!(+ "Conflicting pending planner state in FileStream ScanState"+ ));+ }+ self.pending_planner = Some(pending_planner);+ }+ ScanAndReturn::Continue+ }+ Ok(None) => {+ self.metrics.files_processed.add(1);+ self.metrics.time_opening.stop();+ ScanAndReturn::Continue+ }+ Err(err) => {+ self.metrics.file_open_errors.add(1);+ self.metrics.time_opening.stop();+ match self.on_error {+ OnError::Skip => {+ self.metrics.files_processed.add(1);+ ScanAndReturn::Continue+ }+ OnError::Fail => ScanAndReturn::Error(err),+ }+ }+ };+ }++ // No outstanding work remains, so morselize the next unopened file.+ let part_file = match self.file_iter.pop_front() {+ Some(part_file) => part_file,+ None => return ScanAndReturn::Done(None),+ };++ self.metrics.time_opening.start();+ match self.morselizer.plan_file(part_file) {+ Ok(planner) => {+ self.metrics.files_opened.add(1);+ self.ready_planners.push_back(planner);+ ScanAndReturn::Continue+ }+ Err(err) => match self.on_error {+ OnError::Skip => {+ self.metrics.file_open_errors.add(1);+ self.metrics.time_opening.stop();+ self.metrics.files_processed.add(1);+ ScanAndReturn::Continue+ }+ OnError::Fail => ScanAndReturn::Error(err),+ },+ }+ }+}++/// What should be done on the next iteration of [`ScanState::poll_scan`]?+pub(super) enum ScanAndReturn {+ /// Poll again.+ Continue,+ /// Return the provided result without changing the outer state.+ Return(Poll<Option<Result<RecordBatch>>>),+ /// Update the outer `FileStreamState` to `Done` and return the provided result.+ Done(Option<Result<RecordBatch>>),+ /// Update the outer `FileStreamState` to `Error` and return the provided error.+ Error(DataFusionError),+}diff --git a/datafusion/datasource/src/morsel/adapters.rs b/datafusion/datasource/src/morsel/adapters.rsnew file mode 100644index 0000000000..6fa6d49167--- /dev/null+++ b/datafusion/datasource/src/morsel/adapters.rs@@ -0,0 +1,122 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use crate::PartitionedFile;+use crate::file_stream::FileOpener;+use crate::morsel::{Morsel, MorselPlan, MorselPlanner, Morselizer};+use arrow::array::RecordBatch;+use datafusion_common::Result;+use futures::FutureExt;+use futures::stream::BoxStream;+use std::fmt::Debug;+use std::sync::Arc;++/// Adapt a legacy [`FileOpener`] to the morsel API.+///+/// This preserves backwards compatibility for file formats that have not yet+/// implemented a native [`Morselizer`].+pub struct FileOpenerMorselizer {+ file_opener: Arc<dyn FileOpener>,+}++impl Debug for FileOpenerMorselizer {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("FileOpenerMorselizer")+ .field("file_opener", &"...")+ .finish()+ }+}++impl FileOpenerMorselizer {+ pub fn new(file_opener: Arc<dyn FileOpener>) -> Self {+ Self { file_opener }+ }+}++impl Morselizer for FileOpenerMorselizer {+ fn plan_file(&self, file: PartitionedFile) -> Result<Box<dyn MorselPlanner>> {+ Ok(Box::new(FileOpenFutureMorselPlanner::new(+ Arc::clone(&self.file_opener),+ file,+ )))+ }+}++enum FileOpenFutureMorselPlanner {+ Unopened {+ file_opener: Arc<dyn FileOpener>,+ file: Box<PartitionedFile>,+ },+ ReadyStream(BoxStream<'static, Result<RecordBatch>>),+}++impl Debug for FileOpenFutureMorselPlanner {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::Unopened { .. } => f+ .debug_tuple("FileOpenFutureMorselPlanner::Unopened")+ .finish(),+ Self::ReadyStream(_) => f+ .debug_tuple("FileOpenFutureMorselPlanner::ReadyStream")+ .finish(),+ }+ }+}++impl FileOpenFutureMorselPlanner {+ fn new(file_opener: Arc<dyn FileOpener>, file: PartitionedFile) -> Self {+ Self::Unopened {+ file_opener,+ file: Box::new(file),+ }+ }+}++impl MorselPlanner for FileOpenFutureMorselPlanner {+ fn plan(self: Box<Self>) -> Result<Option<MorselPlan>> {+ match *self {+ Self::Unopened { file_opener, file } => {+ let io_future = async move {+ let stream = file_opener.open(*file)?.await?;+ Ok(Box::new(Self::ReadyStream(stream)) as Box<dyn MorselPlanner>)+ }+ .boxed();+ Ok(Some(MorselPlan::new().with_pending_planner(io_future)))+ }+ Self::ReadyStream(stream) => Ok(Some(+ MorselPlan::new()+ .with_morsels(vec![Box::new(FileStreamMorsel { stream })]),+ )),+ }+ }+}++struct FileStreamMorsel {+ stream: BoxStream<'static, Result<RecordBatch>>,+}++impl Debug for FileStreamMorsel {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("FileStreamMorsel").finish_non_exhaustive()+ }+}++impl Morsel for FileStreamMorsel {+ fn into_stream(self: Box<Self>) -> BoxStream<'static, Result<RecordBatch>> {+ self.stream+ }+}diff --git a/datafusion/datasource/src/morsel/mocks.rs b/datafusion/datasource/src/morsel/mocks.rsnew file mode 100644index 0000000000..cd1fa3732f--- /dev/null+++ b/datafusion/datasource/src/morsel/mocks.rs@@ -0,0 +1,742 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Test-only mocks for exercising the morsel-driven `FileStream` scheduler.++use std::collections::{HashMap, VecDeque};+use std::fmt::{Display, Formatter};+use std::pin::Pin;+use std::sync::{Arc, Mutex};+use std::task::{Context, Poll};++use crate::PartitionedFile;+use crate::morsel::{Morsel, MorselPlan, MorselPlanner, Morselizer};+use arrow::array::{Int32Array, RecordBatch};+use arrow::datatypes::{DataType, Field, Schema};+use datafusion_common::{DataFusionError, Result, internal_datafusion_err};+use futures::stream::BoxStream;+use futures::{Future, FutureExt};++// Use thin wrappers around usize so the test setups are more explicit++/// Identifier for a mock morsel in scheduler snapshots.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub(crate) struct MorselId(pub usize);++/// Identifier for a produced batch in scheduler snapshots.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub(crate) struct BatchId(pub usize);++/// Identifier for a mock I/O future in scheduler snapshots.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub(crate) struct IoFutureId(pub usize);++/// Number of pending polls before a mock I/O future resolves.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub(crate) struct PollsToResolve(pub usize);++/// Error message returned by a mock planner or I/O future.+#[derive(Debug, Clone, PartialEq, Eq)]+pub(crate) struct MockError(pub String);++impl Display for MockError {+ fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {+ write!(f, "{}", self.0)+ }+}++impl std::error::Error for MockError {+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {+ None+ }+}++/// Scheduler-visible event captured by the mock morsel test harness.+#[derive(Debug, Clone, PartialEq, Eq)]+pub(crate) enum MorselEvent {+ MorselizeFile {+ path: String,+ },+ PlannerCreated {+ planner_name: String,+ },+ PlannerCalled {+ planner_name: String,+ },+ IoFutureCreated {+ planner_name: String,+ io_future_id: IoFutureId,+ },+ IoFuturePolled {+ planner_name: String,+ io_future_id: IoFutureId,+ },+ IoFutureResolved {+ planner_name: String,+ io_future_id: IoFutureId,+ },+ IoFutureErrored {+ planner_name: String,+ io_future_id: IoFutureId,+ message: String,+ },+ MorselProduced {+ planner_name: String,+ morsel_id: MorselId,+ },+ MorselStreamStarted {+ morsel_id: MorselId,+ },+ MorselStreamBatchProduced {+ morsel_id: MorselId,+ batch_id: BatchId,+ },+ MorselStreamFinished {+ morsel_id: MorselId,+ },+}++impl Display for MorselEvent {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ match self {+ MorselEvent::MorselizeFile { path } => {+ write!(f, "morselize_file: {path}")+ }+ MorselEvent::PlannerCreated { planner_name } => {+ write!(f, "planner_created: {planner_name}")+ }+ MorselEvent::PlannerCalled { planner_name } => {+ write!(f, "planner_called: {planner_name}")+ }+ MorselEvent::IoFutureCreated {+ planner_name,+ io_future_id,+ } => write!(f, "io_future_created: {planner_name}, {io_future_id:?}"),+ MorselEvent::IoFuturePolled {+ planner_name,+ io_future_id,+ } => write!(f, "io_future_polled: {planner_name}, {io_future_id:?}"),+ MorselEvent::IoFutureResolved {+ planner_name,+ io_future_id,+ } => write!(f, "io_future_resolved: {planner_name}, {io_future_id:?}"),+ MorselEvent::IoFutureErrored {+ planner_name,+ io_future_id,+ message,+ } => write!(+ f,+ "io_future_errored: {planner_name}, {io_future_id:?}, {message}"+ ),+ MorselEvent::MorselProduced {+ planner_name,+ morsel_id,+ } => write!(f, "morsel_produced: {planner_name}, {morsel_id:?}"),+ MorselEvent::MorselStreamStarted { morsel_id } => {+ write!(f, "morsel_stream_started: {morsel_id:?}")+ }+ MorselEvent::MorselStreamBatchProduced {+ morsel_id,+ batch_id,+ } => write!(+ f,+ "morsel_stream_batch_produced: {morsel_id:?}, {batch_id:?}"+ ),+ MorselEvent::MorselStreamFinished { morsel_id } => {+ write!(f, "morsel_stream_finished: {morsel_id:?}")+ }+ }+ }+}++/// Shared observer that records scheduler events for snapshot tests.+#[derive(Debug, Default, Clone)]+pub(crate) struct MorselObserver {+ events: Arc<Mutex<Vec<MorselEvent>>>,+}++impl MorselObserver {+ /// Clears any previously recorded events.+ pub(crate) fn clear(&self) {+ self.events.lock().unwrap().clear();+ }++ /// Records one new scheduler event.+ pub(crate) fn push(&self, event: MorselEvent) {+ self.events.lock().unwrap().push(event);+ }++ /// Formats all recorded events into a stable, snapshot-friendly trace.+ pub(crate) fn format_events(&self) -> String {+ self.events+ .lock()+ .unwrap()+ .iter()+ .map(ToString::to_string)+ .collect::<Vec<_>>()+ .join("\n")+ }+}++/// Declarative planner spec used by the mock morselizer.+#[derive(Debug, Clone)]+pub(crate) struct MockPlanner {+ file_path: String,+ steps: VecDeque<PlannerStep>,+}++impl MockPlanner {+ /// Creates a fluent builder for one mock planner.+ pub(crate) fn builder(file_path: impl Into<String>) -> MockPlannerBuilder {+ MockPlannerBuilder {+ file_path: file_path.into(),+ ..Default::default()+ }+ }++ /// Returns the file path associated with this planner.+ pub(crate) fn file_path(&self) -> &str {+ &self.file_path+ }+}++/// One scheduler-visible result from calling `MorselPlanner::plan`.+#[derive(Debug, Clone)]+enum PlannerStep {+ Plan {+ morsels: Vec<MockMorselSpec>,+ ready_planners: Vec<MockPlanner>,+ pending_planner: Option<MockPendingPlanner>,+ },+ Error {+ error: MockError,+ },+ None,+}++/// One mock morsel returned from a planning step.+#[derive(Debug, Clone)]+struct MockMorselSpec {+ morsel_id: MorselId,+ batch_ids: Vec<i32>,+}++/// One pending planner I/O future returned from a planning step.+#[derive(Debug, Clone)]+struct MockPendingPlanner {+ io_future_id: IoFutureId,+ polls_to_resolve: PollsToResolve,+ result: std::result::Result<(), MockError>,+}++/// Builder for one mock `PlannerStep::Plan`.+#[derive(Debug, Default)]+pub(crate) struct MockPlanBuilder {+ morsels: Vec<MockMorselSpec>,+ ready_planners: Vec<MockPlanner>,+ pending_planner: Option<MockPendingPlanner>,+}++impl MockPlanBuilder {+ /// Create an empty mock plan.+ pub(crate) fn new() -> Self {+ Self::default()+ }++ /// Add one ready morsel with a single batch.+ pub(crate) fn with_morsel(mut self, morsel_id: MorselId, batch_id: i32) -> Self {+ self.morsels.push(MockMorselSpec {+ morsel_id,+ batch_ids: vec![batch_id],+ });+ self+ }++ /// Add one ready morsel with multiple batches.+ pub(crate) fn with_morsel_batches(+ mut self,+ morsel_id: MorselId,+ batch_ids: Vec<i32>,+ ) -> Self {+ self.morsels.push(MockMorselSpec {+ morsel_id,+ batch_ids,+ });+ self+ }++ /// Add a pending planner I/O future produced by this planning step.+ pub(crate) fn with_pending_planner(+ mut self,+ io_future_id: IoFutureId,+ polls_to_resolve: PollsToResolve,+ result: std::result::Result<(), MockError>,+ ) -> Self {+ self.pending_planner = Some(MockPendingPlanner {+ io_future_id,+ polls_to_resolve,+ result,+ });+ self+ }++ /// Add a ready child planner+ pub(crate) fn with_ready_planner(self, ready_planners: MockPlanner) -> Self {+ self.with_ready_planners(vec![ready_planners])+ }++ /// Add ready child planners produced by this planning step.+ pub(crate) fn with_ready_planners(+ mut self,+ ready_planners: Vec<MockPlanner>,+ ) -> Self {+ self.ready_planners.extend(ready_planners);+ self+ }++ /// Build the planner step.+ fn build(self) -> PlannerStep {+ PlannerStep::Plan {+ morsels: self.morsels,+ ready_planners: self.ready_planners,+ pending_planner: self.pending_planner,+ }+ }+}++/// Builder for a planning step that only returns a pending planner.+#[derive(Debug, Clone)]+pub(crate) struct PendingPlannerBuilder {+ io_future_id: IoFutureId,+ polls_to_resolve: PollsToResolve,+ result: std::result::Result<(), MockError>,+}++impl From<PendingPlannerBuilder> for MockPlanBuilder {+ fn from(builder: PendingPlannerBuilder) -> Self {+ builder.build()+ }+}++impl PendingPlannerBuilder {+ /// Create a pending-planner step with a successful I/O future.+ pub(crate) fn new(io_future_id: IoFutureId) -> Self {+ Self {+ io_future_id,+ polls_to_resolve: PollsToResolve(0),+ result: Ok(()),+ }+ }++ /// Configure how many pending polls occur before the I/O future resolves.+ pub(crate) fn with_polls_to_resolve(+ mut self,+ polls_to_resolve: PollsToResolve,+ ) -> Self {+ self.polls_to_resolve = polls_to_resolve;+ self+ }++ /// Configure a failing I/O future for this pending planner.+ pub(crate) fn with_error(mut self, message: impl Into<String>) -> Self {+ self.result = Err(MockError(message.into()));+ self+ }++ /// Build a `MockPlanBuilder` containing only this pending planner.+ pub(crate) fn build(self) -> MockPlanBuilder {+ MockPlanBuilder::new().with_pending_planner(+ self.io_future_id,+ self.polls_to_resolve,+ self.result,+ )+ }+}++/// Fluent builder for [`MockPlanner`] test specs.+#[derive(Debug, Default)]+pub(crate) struct MockPlannerBuilder {+ file_path: String,+ steps: Vec<PlannerStep>,+}++impl From<MockPlannerBuilder> for MockPlanner {+ fn from(value: MockPlannerBuilder) -> Self {+ value.build()+ }+}++impl MockPlannerBuilder {+ pub(crate) fn add_plan(mut self, builder: impl Into<MockPlanBuilder>) -> Self {+ let builder = builder.into();+ self.steps.push(builder.build());+ self+ }++ /// Adds one planning step that reports the planner is exhausted.+ pub(crate) fn return_none(mut self) -> Self {+ self.steps.push(PlannerStep::None);+ self+ }++ /// Adds one planning step that fails during CPU planning.+ pub(crate) fn return_error(mut self, message: impl Into<String>) -> Self {+ self.steps.push(PlannerStep::Error {+ error: MockError(message.into()),+ });+ self+ }++ /// Finalizes the configured mock planner.+ pub(crate) fn build(self) -> MockPlanner {+ let Self { file_path, steps } = self;++ MockPlanner {+ file_path,+ steps: VecDeque::from(steps),+ }+ }+}++/// Mock [`Morselizer`] that maps file paths to fixed planner specs.+#[derive(Debug, Clone, Default)]+pub(crate) struct MockMorselizer {+ observer: MorselObserver,+ files: HashMap<String, MockPlanner>,+}++impl MockMorselizer {+ /// Creates an empty mock morselizer.+ pub(crate) fn new() -> Self {+ Self::default()+ }++ /// Returns the shared event observer for this test harness.+ pub(crate) fn observer(&self) -> &MorselObserver {+ &self.observer+ }++ /// Associates a file path with the planner spec used to open it.+ pub(crate) fn with_file(mut self, planner: MockPlanner) -> Self {+ self.files.insert(planner.file_path.clone(), planner);+ self+ }+}++impl Morselizer for MockMorselizer {+ fn plan_file(&self, file: PartitionedFile) -> Result<Box<dyn MorselPlanner>> {+ let path = file.object_meta.location.to_string();+ self.observer+ .push(MorselEvent::MorselizeFile { path: path.clone() });++ let planner = self.files.get(&path).cloned().ok_or_else(|| {+ internal_datafusion_err!("No mock planner configured for file: {path}")+ })?;++ self.observer.push(MorselEvent::PlannerCreated {+ planner_name: planner.file_path.clone(),+ });++ Ok(Box::new(MockMorselPlanner::new(+ self.observer.clone(),+ planner,+ )))+ }+}++/// Concrete mock planner that executes one predefined step per `plan()` call.+#[derive(Debug)]+struct MockMorselPlanner {+ observer: MorselObserver,+ planner_name: String,+ steps: VecDeque<PlannerStep>,+}++impl MockMorselPlanner {+ /// Creates a concrete planner from its declarative test spec.+ fn new(observer: MorselObserver, planner: MockPlanner) -> Self {+ Self {+ observer,+ planner_name: planner.file_path,+ steps: planner.steps,+ }+ }+}++/// Rebuilds the mock planner continuation after one step completes.+fn current_planner_continuation(+ observer: MorselObserver,+ planner_name: String,+ steps: VecDeque<PlannerStep>,+) -> Vec<Box<dyn MorselPlanner>> {+ let only_none_remaining =+ matches!(steps.front(), Some(PlannerStep::None)) && steps.len() == 1;++ if steps.is_empty() || only_none_remaining {+ Vec::new()+ } else {+ vec![Box::new(MockMorselPlanner {+ observer,+ planner_name,+ steps,+ }) as Box<dyn MorselPlanner>]+ }+}++/// Create any child planners produced by this planning step.+fn child_planners(+ observer: MorselObserver,+ ready_planners: Vec<MockPlanner>,+) -> Vec<Box<dyn MorselPlanner>> {+ ready_planners+ .into_iter()+ .map(|planner| {+ observer.push(MorselEvent::PlannerCreated {+ planner_name: planner.file_path.clone(),+ });+ Box::new(MockMorselPlanner::new(observer.clone(), planner))+ as Box<dyn MorselPlanner>+ })+ .collect()+}++impl MorselPlanner for MockMorselPlanner {+ fn plan(self: Box<Self>) -> Result<Option<MorselPlan>> {+ let Self {+ observer,+ planner_name,+ mut steps,+ } = *self;++ observer.push(MorselEvent::PlannerCalled {+ planner_name: planner_name.clone(),+ });++ let Some(step) = steps.pop_front() else {+ return Ok(None);+ };++ match step {+ PlannerStep::Plan {+ morsels,+ ready_planners,+ pending_planner,+ } => {+ let mut ready_morsels = Vec::new();+ for MockMorselSpec {+ morsel_id,+ batch_ids,+ } in morsels+ {+ observer.push(MorselEvent::MorselProduced {+ planner_name: planner_name.clone(),+ morsel_id,+ });+ ready_morsels.push(Box::new(MockMorsel::new(+ observer.clone(),+ morsel_id,+ batch_ids,+ )) as Box<dyn Morsel>);+ }++ let mut planners = child_planners(observer.clone(), ready_planners);+ if pending_planner.is_none() {+ planners.extend(current_planner_continuation(+ observer.clone(),+ planner_name.clone(),+ steps.clone(),+ ));+ }++ let mut plan = MorselPlan::new()+ .with_morsels(ready_morsels)+ .with_planners(planners);++ if let Some(MockPendingPlanner {+ io_future_id,+ polls_to_resolve,+ result,+ }) = pending_planner+ {+ observer.push(MorselEvent::IoFutureCreated {+ planner_name: planner_name.clone(),+ io_future_id,+ });+ let io_future = MockIoFuture::new(+ observer.clone(),+ planner_name.clone(),+ io_future_id,+ polls_to_resolve,+ result,+ )+ .map(move |result| {+ result?;+ Ok(Box::new(MockMorselPlanner {+ observer,+ planner_name,+ steps,+ }) as Box<dyn MorselPlanner>)+ })+ .boxed();+ plan = plan.with_pending_planner(io_future);+ }++ Ok(Some(plan))+ }+ PlannerStep::Error { error } => {+ Err(DataFusionError::External(Box::new(error)))+ }+ PlannerStep::None => Ok(None),+ }+ }+}++/// Concrete morsel used by the mock scheduler tests.+#[derive(Debug)]+pub(crate) struct MockMorsel {+ observer: MorselObserver,+ morsel_id: MorselId,+ batch_ids: Vec<i32>,+}++impl MockMorsel {+ /// Creates a mock morsel with a deterministic sequence of batches.+ fn new(observer: MorselObserver, morsel_id: MorselId, batch_ids: Vec<i32>) -> Self {+ Self {+ observer,+ morsel_id,+ batch_ids,+ }+ }+}++impl Morsel for MockMorsel {+ fn into_stream(self: Box<Self>) -> BoxStream<'static, Result<RecordBatch>> {+ self.observer.push(MorselEvent::MorselStreamStarted {+ morsel_id: self.morsel_id,+ });+ Box::pin(MockMorselStream {+ observer: self.observer.clone(),+ morsel_id: self.morsel_id,+ batch_ids: self.batch_ids.into(),+ finished: false,+ })+ }+}++/// Stream returned by [`MockMorsel::into_stream`].+struct MockMorselStream {+ observer: MorselObserver,+ morsel_id: MorselId,+ batch_ids: VecDeque<i32>,+ finished: bool,+}++impl futures::Stream for MockMorselStream {+ type Item = Result<RecordBatch>;++ fn poll_next(+ mut self: Pin<&mut Self>,+ _cx: &mut Context<'_>,+ ) -> Poll<Option<Self::Item>> {+ if let Some(batch_id) = self.batch_ids.pop_front() {+ self.observer.push(MorselEvent::MorselStreamBatchProduced {+ morsel_id: self.morsel_id,+ batch_id: BatchId(batch_id as usize),+ });+ return Poll::Ready(Some(Ok(single_value_batch(batch_id))));+ }++ if !self.finished {+ self.finished = true;+ self.observer.push(MorselEvent::MorselStreamFinished {+ morsel_id: self.morsel_id,+ });+ }++ Poll::Ready(None)+ }+}++/// Deterministic future used to simulate planner I/O in tests.+struct MockIoFuture {+ observer: MorselObserver,+ planner_name: String,+ io_future_id: IoFutureId,+ pending_polls_remaining: usize,+ result: std::result::Result<(), MockError>,+}++impl MockIoFuture {+ /// Creates a future that resolves after `io_polls` pending polls.+ fn new(+ observer: MorselObserver,+ planner_name: String,+ io_future_id: IoFutureId,+ polls_to_resolve: PollsToResolve,+ result: std::result::Result<(), MockError>,+ ) -> Self {+ Self {+ observer,+ planner_name,+ io_future_id,+ pending_polls_remaining: polls_to_resolve.0,+ result,+ }+ }+}++impl Future for MockIoFuture {+ type Output = Result<()>;++ fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {+ self.observer.push(MorselEvent::IoFuturePolled {+ planner_name: self.planner_name.clone(),+ io_future_id: self.io_future_id,+ });++ if self.pending_polls_remaining > 0 {+ self.pending_polls_remaining -= 1;+ cx.waker().wake_by_ref();+ return Poll::Pending;+ }++ match &self.result {+ Ok(()) => {+ self.observer.push(MorselEvent::IoFutureResolved {+ planner_name: self.planner_name.clone(),+ io_future_id: self.io_future_id,+ });+ Poll::Ready(Ok(()))+ }+ Err(e) => {+ self.observer.push(MorselEvent::IoFutureErrored {+ planner_name: self.planner_name.clone(),+ io_future_id: self.io_future_id,+ message: e.0.clone(),+ });+ Poll::Ready(Err(DataFusionError::External(Box::new(e.clone()))))+ }+ }+ }+}++/// Creates a one-row batch so snapshot output stays compact and readable.+fn single_value_batch(value: i32) -> RecordBatch {+ let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)]));+ RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![value]))]).unwrap()+}diff --git a/datafusion/datasource/src/morsel/mod.rs b/datafusion/datasource/src/morsel/mod.rsindex 5f200d7794..7b5066ca07 100644--- a/datafusion/datasource/src/morsel/mod.rs+++ b/datafusion/datasource/src/morsel/mod.rs@@ -26,7 +26,12 @@//! It is inspired by the paper [Morsel-Driven Parallelism: A NUMA-Aware Query//! Evaluation Framework for the Many-Core Age](https://db.in.tum.de/~leis/papers/morsels.pdf).+mod adapters;+#[cfg(test)]+pub(crate) mod mocks;+use crate::PartitionedFile;+pub(crate) use adapters::FileOpenerMorselizer;use arrow::array::RecordBatch;use datafusion_common::Result;use futures::FutureExt;