Skip to content

Apache DataFusion SQL Query Engine

Consolidate ArrowFileSource and ArrowStreamFileSource (#18720)

Adrian Garcia Badaracco97af468f3da6
diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs
index dd90c0f107..5e4d7c4606 100644
--- a/datafusion/datasource-arrow/src/source.rs
+++ b/datafusion/datasource-arrow/src/source.rs
@@ -20,12 +20,14 @@
//! # Naming Note
//!
//! The naming in this module can be confusing:
-//! - `ArrowFileSource` / `ArrowFileOpener` handle the Arrow IPC **file format**
+//! - `ArrowFileOpener` handles the Arrow IPC **file format**
//! (with footer, supports parallel reading)
-//! - `ArrowStreamFileSource` / `ArrowStreamFileOpener` handle the Arrow IPC **stream format**
+//! - `ArrowStreamFileOpener` handles the Arrow IPC **stream format**
//! (without footer, sequential only)
+//! - `ArrowSource` is the unified `FileSource` implementation that uses either opener
+//! depending on the format specified at construction
//!
-//! Despite the name "ArrowStreamFileSource", it still reads from files - the "Stream"
+//! Despite the name "ArrowStreamFileOpener", it still reads from files - the "Stream"
//! refers to the Arrow IPC stream format, not streaming I/O. Both formats can be stored
//! in files on disk or object storage.
@@ -51,207 +53,13 @@ use futures::StreamExt;
use itertools::Itertools;
use object_store::{GetOptions, GetRange, GetResultPayload, ObjectStore};
-/// `FileSource` for Arrow IPC file format. Supports range-based parallel reading.
-#[derive(Clone)]
-pub(crate) struct ArrowFileSource {
- table_schema: TableSchema,
- metrics: ExecutionPlanMetricsSet,
- projected_statistics: Option<Statistics>,
- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,
-}
-
-impl ArrowFileSource {
- /// Initialize an ArrowFileSource with the provided schema
- pub fn new(table_schema: impl Into<TableSchema>) -> Self {
- Self {
- table_schema: table_schema.into(),
- metrics: ExecutionPlanMetricsSet::new(),
- projected_statistics: None,
- schema_adapter_factory: None,
- }
- }
-}
-
-impl From<ArrowFileSource> for Arc<dyn FileSource> {
- fn from(source: ArrowFileSource) -> Self {
- as_file_source(source)
- }
-}
-
-impl FileSource for ArrowFileSource {
- fn create_file_opener(
- &self,
- object_store: Arc<dyn ObjectStore>,
- base_config: &FileScanConfig,
- _partition: usize,
- ) -> Arc<dyn FileOpener> {
- Arc::new(ArrowFileOpener {
- object_store,
- projection: base_config.file_column_projection_indices(),
- })
- }
-
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn table_schema(&self) -> &TableSchema {
- &self.table_schema
- }
-
- fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {
- Arc::new(Self { ..self.clone() })
- }
-
- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {
- let mut conf = self.clone();
- conf.projected_statistics = Some(statistics);
- Arc::new(conf)
- }
-
- fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {
- Arc::new(Self { ..self.clone() })
- }
-
- fn metrics(&self) -> &ExecutionPlanMetricsSet {
- &self.metrics
- }
-
- fn statistics(&self) -> Result<Statistics> {
- let statistics = &self.projected_statistics;
- Ok(statistics
- .clone()
- .expect("projected_statistics must be set"))
- }
-
- fn file_type(&self) -> &str {
- "arrow"
- }
-
- fn with_schema_adapter_factory(
- &self,
- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,
- ) -> Result<Arc<dyn FileSource>> {
- Ok(Arc::new(Self {
- schema_adapter_factory: Some(schema_adapter_factory),
- ..self.clone()
- }))
- }
-
- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {
- self.schema_adapter_factory.clone()
- }
-}
-
-/// `FileSource` for Arrow IPC stream format. Supports only sequential reading.
-#[derive(Clone)]
-pub(crate) struct ArrowStreamFileSource {
- table_schema: TableSchema,
- metrics: ExecutionPlanMetricsSet,
- projected_statistics: Option<Statistics>,
- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,
-}
-
-impl ArrowStreamFileSource {
- /// Initialize an ArrowStreamFileSource with the provided schema
- pub fn new(table_schema: impl Into<TableSchema>) -> Self {
- Self {
- table_schema: table_schema.into(),
- metrics: ExecutionPlanMetricsSet::new(),
- projected_statistics: None,
- schema_adapter_factory: None,
- }
- }
-}
-
-impl From<ArrowStreamFileSource> for Arc<dyn FileSource> {
- fn from(source: ArrowStreamFileSource) -> Self {
- as_file_source(source)
- }
-}
-
-impl FileSource for ArrowStreamFileSource {
- fn create_file_opener(
- &self,
- object_store: Arc<dyn ObjectStore>,
- base_config: &FileScanConfig,
- _partition: usize,
- ) -> Arc<dyn FileOpener> {
- Arc::new(ArrowStreamFileOpener {
- object_store,
- projection: base_config.file_column_projection_indices(),
- })
- }
-
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {
- Arc::new(Self { ..self.clone() })
- }
-
- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {
- let mut conf = self.clone();
- conf.projected_statistics = Some(statistics);
- Arc::new(conf)
- }
-
- fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {
- Arc::new(Self { ..self.clone() })
- }
-
- fn repartitioned(
- &self,
- _target_partitions: usize,
- _repartition_file_min_size: usize,
- _output_ordering: Option<LexOrdering>,
- _config: &FileScanConfig,
- ) -> Result<Option<FileScanConfig>> {
- // The Arrow IPC stream format doesn't support range-based parallel reading
- // because it lacks a footer with the information that would be needed to
- // make range-based parallel reading practical. Without the data in the
- // footer you would either need to read the the entire file and record the
- // offsets of the record batches and dictionaries, essentially recreating
- // the footer's contents, or else each partition would need to read the
- // entire file up to the correct offset which is a lot of duplicate I/O.
- // We're opting to avoid that entirely by only acting on a single partition
- // and reading sequentially.
- Ok(None)
- }
-
- fn metrics(&self) -> &ExecutionPlanMetricsSet {
- &self.metrics
- }
-
- fn statistics(&self) -> Result<Statistics> {
- let statistics = &self.projected_statistics;
- Ok(statistics
- .clone()
- .expect("projected_statistics must be set"))
- }
-
- fn file_type(&self) -> &str {
- "arrow_stream"
- }
-
- fn with_schema_adapter_factory(
- &self,
- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,
- ) -> Result<Arc<dyn FileSource>> {
- Ok(Arc::new(Self {
- schema_adapter_factory: Some(schema_adapter_factory),
- ..self.clone()
- }))
- }
-
- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {
- self.schema_adapter_factory.clone()
- }
-
- fn table_schema(&self) -> &TableSchema {
- &self.table_schema
- }
+/// Enum indicating which Arrow IPC format to use
+#[derive(Clone, Copy, Debug)]
+enum ArrowFormat {
+ /// Arrow IPC file format (with footer, supports parallel reading)
+ File,
+ /// Arrow IPC stream format (without footer, sequential only)
+ Stream,
}
/// `FileOpener` for Arrow IPC stream format. Supports only sequential reading.
@@ -436,29 +244,36 @@ impl FileOpener for ArrowFileOpener {
}
}
-/// `FileSource` wrapper for both Arrow IPC file and stream formats
+/// `FileSource` for both Arrow IPC file and stream formats
#[derive(Clone)]
pub struct ArrowSource {
- pub(crate) inner: Arc<dyn FileSource>,
+ format: ArrowFormat,
+ table_schema: TableSchema,
+ metrics: ExecutionPlanMetricsSet,
+ projected_statistics: Option<Statistics>,
+ schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,
}
impl ArrowSource {
- /// Creates a new [`ArrowSource`]
- pub fn new(inner: Arc<dyn FileSource>) -> Self {
- Self { inner }
- }
-
/// Creates an [`ArrowSource`] for file format
pub fn new_file_source(table_schema: impl Into<TableSchema>) -> Self {
Self {
- inner: Arc::new(ArrowFileSource::new(table_schema)),
+ format: ArrowFormat::File,
+ table_schema: table_schema.into(),
+ metrics: ExecutionPlanMetricsSet::new(),
+ projected_statistics: None,
+ schema_adapter_factory: None,
}
}
/// Creates an [`ArrowSource`] for stream format
pub fn new_stream_file_source(table_schema: impl Into<TableSchema>) -> Self {
Self {
- inner: Arc::new(ArrowStreamFileSource::new(table_schema)),
+ format: ArrowFormat::Stream,
+ table_schema: table_schema.into(),
+ metrics: ExecutionPlanMetricsSet::new(),
+ projected_statistics: None,
+ schema_adapter_factory: None,
}
}
}
@@ -468,44 +283,54 @@ impl FileSource for ArrowSource {
&self,
object_store: Arc<dyn ObjectStore>,
base_config: &FileScanConfig,
- partition: usize,
+ _partition: usize,
) -> Arc<dyn FileOpener> {
- self.inner
- .create_file_opener(object_store, base_config, partition)
+ match self.format {
+ ArrowFormat::File => Arc::new(ArrowFileOpener {
+ object_store,
+ projection: base_config.file_column_projection_indices(),
+ }),
+ ArrowFormat::Stream => Arc::new(ArrowStreamFileOpener {
+ object_store,
+ projection: base_config.file_column_projection_indices(),
+ }),
+ }
}
fn as_any(&self) -> &dyn Any {
- self.inner.as_any()
+ self
}
- fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource> {
- Arc::new(Self {
- inner: self.inner.with_batch_size(batch_size),
- })
+ fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {
+ Arc::new(Self { ..self.clone() })
}
- fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource> {
- Arc::new(Self {
- inner: self.inner.with_projection(config),
- })
+ fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {
+ let mut conf = self.clone();
+ conf.projected_statistics = Some(statistics);
+ Arc::new(conf)
}
- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {
- Arc::new(Self {
- inner: self.inner.with_statistics(statistics),
- })
+ fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {
+ Arc::new(Self { ..self.clone() })
}
fn metrics(&self) -> &ExecutionPlanMetricsSet {
- self.inner.metrics()
+ &self.metrics
}
fn statistics(&self) -> Result<Statistics> {
- self.inner.statistics()
+ let statistics = &self.projected_statistics;
+ Ok(statistics
+ .clone()
+ .expect("projected_statistics must be set"))
}
fn file_type(&self) -> &str {
- self.inner.file_type()
+ match self.format {
+ ArrowFormat::File => "arrow",
+ ArrowFormat::Stream => "arrow_stream",
+ }
}
fn with_schema_adapter_factory(
@@ -513,14 +338,13 @@ impl FileSource for ArrowSource {
schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,
) -> Result<Arc<dyn FileSource>> {
Ok(Arc::new(Self {
- inner: self
- .inner
- .with_schema_adapter_factory(schema_adapter_factory)?,
+ schema_adapter_factory: Some(schema_adapter_factory),
+ ..self.clone()
}))
}
fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {
- self.inner.schema_adapter_factory()
+ self.schema_adapter_factory.clone()
}
fn repartitioned(
@@ -530,16 +354,48 @@ impl FileSource for ArrowSource {
output_ordering: Option<LexOrdering>,
config: &FileScanConfig,
) -> Result<Option<FileScanConfig>> {
- self.inner.repartitioned(
- target_partitions,
- repartition_file_min_size,
- output_ordering,
- config,
- )
+ match self.format {
+ ArrowFormat::Stream => {
+ // The Arrow IPC stream format doesn't support range-based parallel reading
+ // because it lacks a footer with the information that would be needed to
+ // make range-based parallel reading practical. Without the data in the
+ // footer you would either need to read the the entire file and record the
+ // offsets of the record batches and dictionaries, essentially recreating
+ // the footer's contents, or else each partition would need to read the
+ // entire file up to the correct offset which is a lot of duplicate I/O.
+ // We're opting to avoid that entirely by only acting on a single partition
+ // and reading sequentially.
+ Ok(None)
+ }
+ ArrowFormat::File => {
+ // Use the default trait implementation logic for file format
+ use datafusion_datasource::file_groups::FileGroupPartitioner;
+
+ if config.file_compression_type.is_compressed()
+ || config.new_lines_in_values
+ {
+ return Ok(None);
+ }
+
+ let repartitioned_file_groups_option = FileGroupPartitioner::new()
+ .with_target_partitions(target_partitions)
+ .with_repartition_file_min_size(repartition_file_min_size)
+ .with_preserve_order_within_groups(output_ordering.is_some())
+ .repartition_file_groups(&config.file_groups);
+
+ if let Some(repartitioned_file_groups) = repartitioned_file_groups_option
+ {
+ let mut source = config.clone();
+ source.file_groups = repartitioned_file_groups;
+ return Ok(Some(source));
+ }
+ Ok(None)
+ }
+ }
}
fn table_schema(&self) -> &TableSchema {
- self.inner.table_schema()
+ &self.table_schema
}
}
@@ -628,9 +484,9 @@ mod tests {
};
let source: Arc<dyn FileSource> = if filename.contains("stream") {
- Arc::new(ArrowStreamFileSource::new(schema))
+ Arc::new(ArrowSource::new_stream_file_source(schema))
} else {
- Arc::new(ArrowFileSource::new(schema))
+ Arc::new(ArrowSource::new_file_source(schema))
};
let scan_config = FileScanConfigBuilder::new(
@@ -673,7 +529,7 @@ mod tests {
let schema = FileReader::try_new(File::open(path_str)?, None)?.schema();
- let source = Arc::new(ArrowFileSource::new(schema));
+ let source = Arc::new(ArrowSource::new_file_source(schema));
let scan_config = FileScanConfigBuilder::new(
ObjectStoreUrl::local_filesystem(),
@@ -714,7 +570,7 @@ mod tests {
let schema = StreamReader::try_new(File::open(path_str)?, None)?.schema();
- let source = Arc::new(ArrowStreamFileSource::new(schema));
+ let source = Arc::new(ArrowSource::new_stream_file_source(schema));
let scan_config = FileScanConfigBuilder::new(
ObjectStoreUrl::local_filesystem(),
@@ -733,7 +589,7 @@ mod tests {
async fn test_arrow_stream_repartitioning_not_supported() -> Result<()> {
let schema =
Arc::new(Schema::new(vec![Field::new("f0", DataType::Int64, false)]));
- let source = ArrowStreamFileSource::new(schema);
+ let source = ArrowSource::new_stream_file_source(schema);
let config = FileScanConfigBuilder::new(
ObjectStoreUrl::local_filesystem(),