Apache DataFusion SQL Query Engine
Consolidate ArrowFileSource and ArrowStreamFileSource (#18720)
Adrian Garcia Badaracco97af468f3da6
diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rsindex dd90c0f107..5e4d7c4606 100644--- a/datafusion/datasource-arrow/src/source.rs+++ b/datafusion/datasource-arrow/src/source.rs@@ -20,12 +20,14 @@//! # Naming Note//!//! The naming in this module can be confusing:-//! - `ArrowFileSource` / `ArrowFileOpener` handle the Arrow IPC **file format**+//! - `ArrowFileOpener` handles the Arrow IPC **file format**//! (with footer, supports parallel reading)-//! - `ArrowStreamFileSource` / `ArrowStreamFileOpener` handle the Arrow IPC **stream format**+//! - `ArrowStreamFileOpener` handles the Arrow IPC **stream format**//! (without footer, sequential only)+//! - `ArrowSource` is the unified `FileSource` implementation that uses either opener+//! depending on the format specified at construction//!-//! Despite the name "ArrowStreamFileSource", it still reads from files - the "Stream"+//! Despite the name "ArrowStreamFileOpener", it still reads from files - the "Stream"//! refers to the Arrow IPC stream format, not streaming I/O. Both formats can be stored//! in files on disk or object storage.@@ -51,207 +53,13 @@ use futures::StreamExt;use itertools::Itertools;use object_store::{GetOptions, GetRange, GetResultPayload, ObjectStore};-/// `FileSource` for Arrow IPC file format. Supports range-based parallel reading.-#[derive(Clone)]-pub(crate) struct ArrowFileSource {- table_schema: TableSchema,- metrics: ExecutionPlanMetricsSet,- projected_statistics: Option<Statistics>,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,-}--impl ArrowFileSource {- /// Initialize an ArrowFileSource with the provided schema- pub fn new(table_schema: impl Into<TableSchema>) -> Self {- Self {- table_schema: table_schema.into(),- metrics: ExecutionPlanMetricsSet::new(),- projected_statistics: None,- schema_adapter_factory: None,- }- }-}--impl From<ArrowFileSource> for Arc<dyn FileSource> {- fn from(source: ArrowFileSource) -> Self {- as_file_source(source)- }-}--impl FileSource for ArrowFileSource {- fn create_file_opener(- &self,- object_store: Arc<dyn ObjectStore>,- base_config: &FileScanConfig,- _partition: usize,- ) -> Arc<dyn FileOpener> {- Arc::new(ArrowFileOpener {- object_store,- projection: base_config.file_column_projection_indices(),- })- }-- fn as_any(&self) -> &dyn Any {- self- }-- fn table_schema(&self) -> &TableSchema {- &self.table_schema- }-- fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {- Arc::new(Self { ..self.clone() })- }-- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- let mut conf = self.clone();- conf.projected_statistics = Some(statistics);- Arc::new(conf)- }-- fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {- Arc::new(Self { ..self.clone() })- }-- fn metrics(&self) -> &ExecutionPlanMetricsSet {- &self.metrics- }-- fn statistics(&self) -> Result<Statistics> {- let statistics = &self.projected_statistics;- Ok(statistics- .clone()- .expect("projected_statistics must be set"))- }-- fn file_type(&self) -> &str {- "arrow"- }-- fn with_schema_adapter_factory(- &self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Result<Arc<dyn FileSource>> {- Ok(Arc::new(Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self.clone()- }))- }-- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.clone()- }-}--/// `FileSource` for Arrow IPC stream format. Supports only sequential reading.-#[derive(Clone)]-pub(crate) struct ArrowStreamFileSource {- table_schema: TableSchema,- metrics: ExecutionPlanMetricsSet,- projected_statistics: Option<Statistics>,- schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,-}--impl ArrowStreamFileSource {- /// Initialize an ArrowStreamFileSource with the provided schema- pub fn new(table_schema: impl Into<TableSchema>) -> Self {- Self {- table_schema: table_schema.into(),- metrics: ExecutionPlanMetricsSet::new(),- projected_statistics: None,- schema_adapter_factory: None,- }- }-}--impl From<ArrowStreamFileSource> for Arc<dyn FileSource> {- fn from(source: ArrowStreamFileSource) -> Self {- as_file_source(source)- }-}--impl FileSource for ArrowStreamFileSource {- fn create_file_opener(- &self,- object_store: Arc<dyn ObjectStore>,- base_config: &FileScanConfig,- _partition: usize,- ) -> Arc<dyn FileOpener> {- Arc::new(ArrowStreamFileOpener {- object_store,- projection: base_config.file_column_projection_indices(),- })- }-- fn as_any(&self) -> &dyn Any {- self- }-- fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {- Arc::new(Self { ..self.clone() })- }-- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- let mut conf = self.clone();- conf.projected_statistics = Some(statistics);- Arc::new(conf)- }-- fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {- Arc::new(Self { ..self.clone() })- }-- fn repartitioned(- &self,- _target_partitions: usize,- _repartition_file_min_size: usize,- _output_ordering: Option<LexOrdering>,- _config: &FileScanConfig,- ) -> Result<Option<FileScanConfig>> {- // The Arrow IPC stream format doesn't support range-based parallel reading- // because it lacks a footer with the information that would be needed to- // make range-based parallel reading practical. Without the data in the- // footer you would either need to read the the entire file and record the- // offsets of the record batches and dictionaries, essentially recreating- // the footer's contents, or else each partition would need to read the- // entire file up to the correct offset which is a lot of duplicate I/O.- // We're opting to avoid that entirely by only acting on a single partition- // and reading sequentially.- Ok(None)- }-- fn metrics(&self) -> &ExecutionPlanMetricsSet {- &self.metrics- }-- fn statistics(&self) -> Result<Statistics> {- let statistics = &self.projected_statistics;- Ok(statistics- .clone()- .expect("projected_statistics must be set"))- }-- fn file_type(&self) -> &str {- "arrow_stream"- }-- fn with_schema_adapter_factory(- &self,- schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,- ) -> Result<Arc<dyn FileSource>> {- Ok(Arc::new(Self {- schema_adapter_factory: Some(schema_adapter_factory),- ..self.clone()- }))- }-- fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- self.schema_adapter_factory.clone()- }-- fn table_schema(&self) -> &TableSchema {- &self.table_schema- }+/// Enum indicating which Arrow IPC format to use+#[derive(Clone, Copy, Debug)]+enum ArrowFormat {+ /// Arrow IPC file format (with footer, supports parallel reading)+ File,+ /// Arrow IPC stream format (without footer, sequential only)+ Stream,}/// `FileOpener` for Arrow IPC stream format. Supports only sequential reading.@@ -436,29 +244,36 @@ impl FileOpener for ArrowFileOpener {}}-/// `FileSource` wrapper for both Arrow IPC file and stream formats+/// `FileSource` for both Arrow IPC file and stream formats#[derive(Clone)]pub struct ArrowSource {- pub(crate) inner: Arc<dyn FileSource>,+ format: ArrowFormat,+ table_schema: TableSchema,+ metrics: ExecutionPlanMetricsSet,+ projected_statistics: Option<Statistics>,+ schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}impl ArrowSource {- /// Creates a new [`ArrowSource`]- pub fn new(inner: Arc<dyn FileSource>) -> Self {- Self { inner }- }-/// Creates an [`ArrowSource`] for file formatpub fn new_file_source(table_schema: impl Into<TableSchema>) -> Self {Self {- inner: Arc::new(ArrowFileSource::new(table_schema)),+ format: ArrowFormat::File,+ table_schema: table_schema.into(),+ metrics: ExecutionPlanMetricsSet::new(),+ projected_statistics: None,+ schema_adapter_factory: None,}}/// Creates an [`ArrowSource`] for stream formatpub fn new_stream_file_source(table_schema: impl Into<TableSchema>) -> Self {Self {- inner: Arc::new(ArrowStreamFileSource::new(table_schema)),+ format: ArrowFormat::Stream,+ table_schema: table_schema.into(),+ metrics: ExecutionPlanMetricsSet::new(),+ projected_statistics: None,+ schema_adapter_factory: None,}}}@@ -468,44 +283,54 @@ impl FileSource for ArrowSource {&self,object_store: Arc<dyn ObjectStore>,base_config: &FileScanConfig,- partition: usize,+ _partition: usize,) -> Arc<dyn FileOpener> {- self.inner- .create_file_opener(object_store, base_config, partition)+ match self.format {+ ArrowFormat::File => Arc::new(ArrowFileOpener {+ object_store,+ projection: base_config.file_column_projection_indices(),+ }),+ ArrowFormat::Stream => Arc::new(ArrowStreamFileOpener {+ object_store,+ projection: base_config.file_column_projection_indices(),+ }),+ }}fn as_any(&self) -> &dyn Any {- self.inner.as_any()+ self}- fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource> {- Arc::new(Self {- inner: self.inner.with_batch_size(batch_size),- })+ fn with_batch_size(&self, _batch_size: usize) -> Arc<dyn FileSource> {+ Arc::new(Self { ..self.clone() })}- fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource> {- Arc::new(Self {- inner: self.inner.with_projection(config),- })+ fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {+ let mut conf = self.clone();+ conf.projected_statistics = Some(statistics);+ Arc::new(conf)}- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- Arc::new(Self {- inner: self.inner.with_statistics(statistics),- })+ fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {+ Arc::new(Self { ..self.clone() })}fn metrics(&self) -> &ExecutionPlanMetricsSet {- self.inner.metrics()+ &self.metrics}fn statistics(&self) -> Result<Statistics> {- self.inner.statistics()+ let statistics = &self.projected_statistics;+ Ok(statistics+ .clone()+ .expect("projected_statistics must be set"))}fn file_type(&self) -> &str {- self.inner.file_type()+ match self.format {+ ArrowFormat::File => "arrow",+ ArrowFormat::Stream => "arrow_stream",+ }}fn with_schema_adapter_factory(@@ -513,14 +338,13 @@ impl FileSource for ArrowSource {schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,) -> Result<Arc<dyn FileSource>> {Ok(Arc::new(Self {- inner: self- .inner- .with_schema_adapter_factory(schema_adapter_factory)?,+ schema_adapter_factory: Some(schema_adapter_factory),+ ..self.clone()}))}fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {- self.inner.schema_adapter_factory()+ self.schema_adapter_factory.clone()}fn repartitioned(@@ -530,16 +354,48 @@ impl FileSource for ArrowSource {output_ordering: Option<LexOrdering>,config: &FileScanConfig,) -> Result<Option<FileScanConfig>> {- self.inner.repartitioned(- target_partitions,- repartition_file_min_size,- output_ordering,- config,- )+ match self.format {+ ArrowFormat::Stream => {+ // The Arrow IPC stream format doesn't support range-based parallel reading+ // because it lacks a footer with the information that would be needed to+ // make range-based parallel reading practical. Without the data in the+ // footer you would either need to read the the entire file and record the+ // offsets of the record batches and dictionaries, essentially recreating+ // the footer's contents, or else each partition would need to read the+ // entire file up to the correct offset which is a lot of duplicate I/O.+ // We're opting to avoid that entirely by only acting on a single partition+ // and reading sequentially.+ Ok(None)+ }+ ArrowFormat::File => {+ // Use the default trait implementation logic for file format+ use datafusion_datasource::file_groups::FileGroupPartitioner;++ if config.file_compression_type.is_compressed()+ || config.new_lines_in_values+ {+ return Ok(None);+ }++ let repartitioned_file_groups_option = FileGroupPartitioner::new()+ .with_target_partitions(target_partitions)+ .with_repartition_file_min_size(repartition_file_min_size)+ .with_preserve_order_within_groups(output_ordering.is_some())+ .repartition_file_groups(&config.file_groups);++ if let Some(repartitioned_file_groups) = repartitioned_file_groups_option+ {+ let mut source = config.clone();+ source.file_groups = repartitioned_file_groups;+ return Ok(Some(source));+ }+ Ok(None)+ }+ }}fn table_schema(&self) -> &TableSchema {- self.inner.table_schema()+ &self.table_schema}}@@ -628,9 +484,9 @@ mod tests {};let source: Arc<dyn FileSource> = if filename.contains("stream") {- Arc::new(ArrowStreamFileSource::new(schema))+ Arc::new(ArrowSource::new_stream_file_source(schema))} else {- Arc::new(ArrowFileSource::new(schema))+ Arc::new(ArrowSource::new_file_source(schema))};let scan_config = FileScanConfigBuilder::new(@@ -673,7 +529,7 @@ mod tests {let schema = FileReader::try_new(File::open(path_str)?, None)?.schema();- let source = Arc::new(ArrowFileSource::new(schema));+ let source = Arc::new(ArrowSource::new_file_source(schema));let scan_config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),@@ -714,7 +570,7 @@ mod tests {let schema = StreamReader::try_new(File::open(path_str)?, None)?.schema();- let source = Arc::new(ArrowStreamFileSource::new(schema));+ let source = Arc::new(ArrowSource::new_stream_file_source(schema));let scan_config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),@@ -733,7 +589,7 @@ mod tests {async fn test_arrow_stream_repartitioning_not_supported() -> Result<()> {let schema =Arc::new(Schema::new(vec![Field::new("f0", DataType::Int64, false)]));- let source = ArrowStreamFileSource::new(schema);+ let source = ArrowSource::new_stream_file_source(schema);let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(),