Apache DataFusion SQL Query Engine
refactor: move arrow datasource to new `datafusion-datasource-arrow` crate (#18082)
Tim Saucer8e1d13a9c6e9
diff --git a/.github/workflows/labeler/labeler-config.yml b/.github/workflows/labeler/labeler-config.ymlindex e408130725..38d88059da 100644--- a/.github/workflows/labeler/labeler-config.yml+++ b/.github/workflows/labeler/labeler-config.yml@@ -58,7 +58,7 @@ execution:datasource:- changed-files:- - any-glob-to-any-file: ['datafusion/datasource/**/*', 'datafusion/datasource-avro/**/*', 'datafusion/datasource-csv/**/*', 'datafusion/datasource-json/**/*', 'datafusion/datasource-parquet/**/*']+ - any-glob-to-any-file: ['datafusion/datasource/**/*', 'datafusion/datasource-avro/**/*', 'datafusion/datasource-arrow/**/*', 'datafusion/datasource-csv/**/*', 'datafusion/datasource-json/**/*', 'datafusion/datasource-parquet/**/*']functions:- changed-files:diff --git a/Cargo.lock b/Cargo.lockindex 0392c8147a..d69ece6d3f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1789,7 +1789,6 @@ name = "datafusion"version = "50.2.0"dependencies = ["arrow",- "arrow-ipc","arrow-schema","async-trait","bytes",@@ -1803,6 +1802,7 @@ dependencies = ["datafusion-common","datafusion-common-runtime","datafusion-datasource",+ "datafusion-datasource-arrow","datafusion-datasource-avro","datafusion-datasource-csv","datafusion-datasource-json",@@ -2030,6 +2030,29 @@ dependencies = ["zstd",]+[[package]]+name = "datafusion-datasource-arrow"+version = "50.2.0"+dependencies = [+ "arrow",+ "arrow-ipc",+ "async-trait",+ "bytes",+ "chrono",+ "datafusion-common",+ "datafusion-common-runtime",+ "datafusion-datasource",+ "datafusion-execution",+ "datafusion-expr",+ "datafusion-physical-expr-common",+ "datafusion-physical-plan",+ "datafusion-session",+ "futures",+ "itertools 0.14.0",+ "object_store",+ "tokio",+]+[[package]]name = "datafusion-datasource-avro"version = "50.2.0"diff --git a/Cargo.toml b/Cargo.tomlindex dd0b20de52..79c14d6cca 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -22,6 +22,7 @@ members = ["datafusion/catalog","datafusion/catalog-listing","datafusion/datasource",+ "datafusion/datasource-arrow","datafusion/datasource-avro","datafusion/datasource-csv","datafusion/datasource-json",@@ -116,6 +117,7 @@ datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "5datafusion-common = { path = "datafusion/common", version = "50.2.0", default-features = false }datafusion-common-runtime = { path = "datafusion/common-runtime", version = "50.2.0" }datafusion-datasource = { path = "datafusion/datasource", version = "50.2.0", default-features = false }+datafusion-datasource-arrow = { path = "datafusion/datasource-arrow", version = "50.2.0", default-features = false }datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "50.2.0", default-features = false }datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "50.2.0", default-features = false }datafusion-datasource-json = { path = "datafusion/datasource-json", version = "50.2.0", default-features = false }diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.tomlindex d3bc454658..a5a715cea9 100644--- a/datafusion/core/Cargo.toml+++ b/datafusion/core/Cargo.toml@@ -47,7 +47,7 @@ compression = ["bzip2","flate2","zstd",- "arrow-ipc/zstd",+ "datafusion-datasource-arrow/compression","datafusion-datasource/compression",]crypto_expressions = ["datafusion-functions/crypto_expressions"]@@ -109,7 +109,6 @@ extended_tests = [][dependencies]arrow = { workspace = true }-arrow-ipc = { workspace = true }arrow-schema = { workspace = true, features = ["canonical_extension_types"] }async-trait = { workspace = true }bytes = { workspace = true }@@ -120,6 +119,7 @@ datafusion-catalog-listing = { workspace = true }datafusion-common = { workspace = true, features = ["object_store"] }datafusion-common-runtime = { workspace = true }datafusion-datasource = { workspace = true }+datafusion-datasource-arrow = { workspace = true }datafusion-datasource-avro = { workspace = true, optional = true }datafusion-datasource-csv = { workspace = true }datafusion-datasource-json = { workspace = true }diff --git a/datafusion/core/src/datasource/file_format/arrow.rs b/datafusion/core/src/datasource/file_format/arrow.rsindex 25bc166d65..8701f96eb3 100644--- a/datafusion/core/src/datasource/file_format/arrow.rs+++ b/datafusion/core/src/datasource/file_format/arrow.rs@@ -15,510 +15,5 @@// specific language governing permissions and limitations// under the License.-//! [`ArrowFormat`]: Apache Arrow [`FileFormat`] abstractions-//!-//! Works with files following the [Arrow IPC format](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format)--use std::any::Any;-use std::borrow::Cow;-use std::collections::HashMap;-use std::fmt::{self, Debug};-use std::sync::Arc;--use super::file_compression_type::FileCompressionType;-use super::write::demux::DemuxedStreamReceiver;-use super::write::SharedBuffer;-use super::FileFormatFactory;-use crate::datasource::file_format::write::get_writer_schema;-use crate::datasource::file_format::FileFormat;-use crate::datasource::physical_plan::{ArrowSource, FileSink, FileSinkConfig};-use crate::error::Result;-use crate::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan};--use arrow::datatypes::{Schema, SchemaRef};-use arrow::error::ArrowError;-use arrow::ipc::convert::fb_to_schema;-use arrow::ipc::reader::FileReader;-use arrow::ipc::writer::IpcWriteOptions;-use arrow::ipc::{root_as_message, CompressionType};-use datafusion_catalog::Session;-use datafusion_common::parsers::CompressionTypeVariant;-use datafusion_common::{- internal_datafusion_err, not_impl_err, DataFusionError, GetExt, Statistics,- DEFAULT_ARROW_EXTENSION,-};-use datafusion_common_runtime::{JoinSet, SpawnedTask};-use datafusion_datasource::display::FileGroupDisplay;-use datafusion_datasource::file::FileSource;-use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};-use datafusion_datasource::sink::{DataSink, DataSinkExec};-use datafusion_datasource::write::ObjectWriterBuilder;-use datafusion_execution::{SendableRecordBatchStream, TaskContext};-use datafusion_expr::dml::InsertOp;-use datafusion_physical_expr_common::sort_expr::LexRequirement;--use async_trait::async_trait;-use bytes::Bytes;-use datafusion_datasource::source::DataSourceExec;-use futures::stream::BoxStream;-use futures::StreamExt;-use object_store::{GetResultPayload, ObjectMeta, ObjectStore};-use tokio::io::AsyncWriteExt;--/// Initial writing buffer size. Note this is just a size hint for efficiency. It-/// will grow beyond the set value if needed.-const INITIAL_BUFFER_BYTES: usize = 1048576;--/// If the buffered Arrow data exceeds this size, it is flushed to object store-const BUFFER_FLUSH_BYTES: usize = 1024000;--#[derive(Default, Debug)]-/// Factory struct used to create [ArrowFormat]-pub struct ArrowFormatFactory;--impl ArrowFormatFactory {- /// Creates an instance of [ArrowFormatFactory]- pub fn new() -> Self {- Self {}- }-}--impl FileFormatFactory for ArrowFormatFactory {- fn create(- &self,- _state: &dyn Session,- _format_options: &HashMap<String, String>,- ) -> Result<Arc<dyn FileFormat>> {- Ok(Arc::new(ArrowFormat))- }-- fn default(&self) -> Arc<dyn FileFormat> {- Arc::new(ArrowFormat)- }-- fn as_any(&self) -> &dyn Any {- self- }-}--impl GetExt for ArrowFormatFactory {- fn get_ext(&self) -> String {- // Removes the dot, i.e. ".parquet" -> "parquet"- DEFAULT_ARROW_EXTENSION[1..].to_string()- }-}--/// Arrow `FileFormat` implementation.-#[derive(Default, Debug)]-pub struct ArrowFormat;--#[async_trait]-impl FileFormat for ArrowFormat {- fn as_any(&self) -> &dyn Any {- self- }-- fn get_ext(&self) -> String {- ArrowFormatFactory::new().get_ext()- }-- fn get_ext_with_compression(- &self,- file_compression_type: &FileCompressionType,- ) -> Result<String> {- let ext = self.get_ext();- match file_compression_type.get_variant() {- CompressionTypeVariant::UNCOMPRESSED => Ok(ext),- _ => Err(internal_datafusion_err!(- "Arrow FileFormat does not support compression."- )),- }- }-- fn compression_type(&self) -> Option<FileCompressionType> {- None- }-- async fn infer_schema(- &self,- _state: &dyn Session,- store: &Arc<dyn ObjectStore>,- objects: &[ObjectMeta],- ) -> Result<SchemaRef> {- let mut schemas = vec![];- for object in objects {- let r = store.as_ref().get(&object.location).await?;- let schema = match r.payload {- #[cfg(not(target_arch = "wasm32"))]- GetResultPayload::File(mut file, _) => {- let reader = FileReader::try_new(&mut file, None)?;- reader.schema()- }- GetResultPayload::Stream(stream) => {- infer_schema_from_file_stream(stream).await?- }- };- schemas.push(schema.as_ref().clone());- }- let merged_schema = Schema::try_merge(schemas)?;- Ok(Arc::new(merged_schema))- }-- async fn infer_stats(- &self,- _state: &dyn Session,- _store: &Arc<dyn ObjectStore>,- table_schema: SchemaRef,- _object: &ObjectMeta,- ) -> Result<Statistics> {- Ok(Statistics::new_unknown(&table_schema))- }-- async fn create_physical_plan(- &self,- _state: &dyn Session,- conf: FileScanConfig,- ) -> Result<Arc<dyn ExecutionPlan>> {- let source = Arc::new(ArrowSource::default());- let config = FileScanConfigBuilder::from(conf)- .with_source(source)- .build();-- Ok(DataSourceExec::from_data_source(config))- }-- async fn create_writer_physical_plan(- &self,- input: Arc<dyn ExecutionPlan>,- _state: &dyn Session,- conf: FileSinkConfig,- order_requirements: Option<LexRequirement>,- ) -> Result<Arc<dyn ExecutionPlan>> {- if conf.insert_op != InsertOp::Append {- return not_impl_err!("Overwrites are not implemented yet for Arrow format");- }-- let sink = Arc::new(ArrowFileSink::new(conf));-- Ok(Arc::new(DataSinkExec::new(input, sink, order_requirements)) as _)- }-- fn file_source(&self) -> Arc<dyn FileSource> {- Arc::new(ArrowSource::default())- }-}--/// Implements [`FileSink`] for writing to arrow_ipc files-struct ArrowFileSink {- config: FileSinkConfig,-}--impl ArrowFileSink {- fn new(config: FileSinkConfig) -> Self {- Self { config }- }-}--#[async_trait]-impl FileSink for ArrowFileSink {- fn config(&self) -> &FileSinkConfig {- &self.config- }-- async fn spawn_writer_tasks_and_join(- &self,- context: &Arc<TaskContext>,- demux_task: SpawnedTask<Result<()>>,- mut file_stream_rx: DemuxedStreamReceiver,- object_store: Arc<dyn ObjectStore>,- ) -> Result<u64> {- let mut file_write_tasks: JoinSet<std::result::Result<usize, DataFusionError>> =- JoinSet::new();-- let ipc_options =- IpcWriteOptions::try_new(64, false, arrow_ipc::MetadataVersion::V5)?- .try_with_compression(Some(CompressionType::LZ4_FRAME))?;- while let Some((path, mut rx)) = file_stream_rx.recv().await {- let shared_buffer = SharedBuffer::new(INITIAL_BUFFER_BYTES);- let mut arrow_writer = arrow_ipc::writer::FileWriter::try_new_with_options(- shared_buffer.clone(),- &get_writer_schema(&self.config),- ipc_options.clone(),- )?;- let mut object_store_writer = ObjectWriterBuilder::new(- FileCompressionType::UNCOMPRESSED,- &path,- Arc::clone(&object_store),- )- .with_buffer_size(Some(- context- .session_config()- .options()- .execution- .objectstore_writer_buffer_size,- ))- .build()?;- file_write_tasks.spawn(async move {- let mut row_count = 0;- while let Some(batch) = rx.recv().await {- row_count += batch.num_rows();- arrow_writer.write(&batch)?;- let mut buff_to_flush = shared_buffer.buffer.try_lock().unwrap();- if buff_to_flush.len() > BUFFER_FLUSH_BYTES {- object_store_writer- .write_all(buff_to_flush.as_slice())- .await?;- buff_to_flush.clear();- }- }- arrow_writer.finish()?;- let final_buff = shared_buffer.buffer.try_lock().unwrap();-- object_store_writer.write_all(final_buff.as_slice()).await?;- object_store_writer.shutdown().await?;- Ok(row_count)- });- }-- let mut row_count = 0;- while let Some(result) = file_write_tasks.join_next().await {- match result {- Ok(r) => {- row_count += r?;- }- Err(e) => {- if e.is_panic() {- std::panic::resume_unwind(e.into_panic());- } else {- unreachable!();- }- }- }- }-- demux_task- .join_unwind()- .await- .map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;- Ok(row_count as u64)- }-}--impl Debug for ArrowFileSink {- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- f.debug_struct("ArrowFileSink").finish()- }-}--impl DisplayAs for ArrowFileSink {- fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter<'_>) -> fmt::Result {- match t {- DisplayFormatType::Default | DisplayFormatType::Verbose => {- write!(f, "ArrowFileSink(file_groups=",)?;- FileGroupDisplay(&self.config.file_group).fmt_as(t, f)?;- write!(f, ")")- }- DisplayFormatType::TreeRender => {- writeln!(f, "format: arrow")?;- write!(f, "file={}", &self.config.original_url)- }- }- }-}--#[async_trait]-impl DataSink for ArrowFileSink {- fn as_any(&self) -> &dyn Any {- self- }-- fn schema(&self) -> &SchemaRef {- self.config.output_schema()- }-- async fn write_all(- &self,- data: SendableRecordBatchStream,- context: &Arc<TaskContext>,- ) -> Result<u64> {- FileSink::write_all(self, data, context).await- }-}--const ARROW_MAGIC: [u8; 6] = [b'A', b'R', b'R', b'O', b'W', b'1'];-const CONTINUATION_MARKER: [u8; 4] = [0xff; 4];--/// Custom implementation of inferring schema. Should eventually be moved upstream to arrow-rs.-/// See <https://github.com/apache/arrow-rs/issues/5021>-async fn infer_schema_from_file_stream(- mut stream: BoxStream<'static, object_store::Result<Bytes>>,-) -> Result<SchemaRef> {- // Expected format:- // <magic number "ARROW1"> - 6 bytes- // <empty padding bytes [to 8 byte boundary]> - 2 bytes- // <continuation: 0xFFFFFFFF> - 4 bytes, not present below v0.15.0- // <metadata_size: int32> - 4 bytes- // <metadata_flatbuffer: bytes>- // <rest of file bytes>-- // So in first read we need at least all known sized sections,- // which is 6 + 2 + 4 + 4 = 16 bytes.- let bytes = collect_at_least_n_bytes(&mut stream, 16, None).await?;-- // Files should start with these magic bytes- if bytes[0..6] != ARROW_MAGIC {- return Err(ArrowError::ParseError(- "Arrow file does not contain correct header".to_string(),- ))?;- }-- // Since continuation marker bytes added in later versions- let (meta_len, rest_of_bytes_start_index) = if bytes[8..12] == CONTINUATION_MARKER {- (&bytes[12..16], 16)- } else {- (&bytes[8..12], 12)- };-- let meta_len = [meta_len[0], meta_len[1], meta_len[2], meta_len[3]];- let meta_len = i32::from_le_bytes(meta_len);-- // Read bytes for Schema message- let block_data = if bytes[rest_of_bytes_start_index..].len() < meta_len as usize {- // Need to read more bytes to decode Message- let mut block_data = Vec::with_capacity(meta_len as usize);- // In case we had some spare bytes in our initial read chunk- block_data.extend_from_slice(&bytes[rest_of_bytes_start_index..]);- let size_to_read = meta_len as usize - block_data.len();- let block_data =- collect_at_least_n_bytes(&mut stream, size_to_read, Some(block_data)).await?;- Cow::Owned(block_data)- } else {- // Already have the bytes we need- let end_index = meta_len as usize + rest_of_bytes_start_index;- let block_data = &bytes[rest_of_bytes_start_index..end_index];- Cow::Borrowed(block_data)- };-- // Decode Schema message- let message = root_as_message(&block_data).map_err(|err| {- ArrowError::ParseError(format!("Unable to read IPC message as metadata: {err:?}"))- })?;- let ipc_schema = message.header_as_schema().ok_or_else(|| {- ArrowError::IpcError("Unable to read IPC message as schema".to_string())- })?;- let schema = fb_to_schema(ipc_schema);-- Ok(Arc::new(schema))-}--async fn collect_at_least_n_bytes(- stream: &mut BoxStream<'static, object_store::Result<Bytes>>,- n: usize,- extend_from: Option<Vec<u8>>,-) -> Result<Vec<u8>> {- let mut buf = extend_from.unwrap_or_else(|| Vec::with_capacity(n));- // If extending existing buffer then ensure we read n additional bytes- let n = n + buf.len();- while let Some(bytes) = stream.next().await.transpose()? {- buf.extend_from_slice(&bytes);- if buf.len() >= n {- break;- }- }- if buf.len() < n {- return Err(ArrowError::ParseError(- "Unexpected end of byte stream for Arrow IPC file".to_string(),- ))?;- }- Ok(buf)-}--#[cfg(test)]-mod tests {- use super::*;- use crate::execution::context::SessionContext;-- use chrono::DateTime;- use object_store::{chunked::ChunkedStore, memory::InMemory, path::Path};-- #[tokio::test]- async fn test_infer_schema_stream() -> Result<()> {- let mut bytes = std::fs::read("tests/data/example.arrow")?;- bytes.truncate(bytes.len() - 20); // mangle end to show we don't need to read whole file- let location = Path::parse("example.arrow")?;- let in_memory_store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());- in_memory_store.put(&location, bytes.into()).await?;-- let session_ctx = SessionContext::new();- let state = session_ctx.state();- let object_meta = ObjectMeta {- location,- last_modified: DateTime::default(),- size: u64::MAX,- e_tag: None,- version: None,- };-- let arrow_format = ArrowFormat {};- let expected = vec!["f0: Int64", "f1: Utf8", "f2: Boolean"];-- // Test chunk sizes where too small so we keep having to read more bytes- // And when large enough that first read contains all we need- for chunk_size in [7, 3000] {- let store = Arc::new(ChunkedStore::new(in_memory_store.clone(), chunk_size));- let inferred_schema = arrow_format- .infer_schema(- &state,- &(store.clone() as Arc<dyn ObjectStore>),- std::slice::from_ref(&object_meta),- )- .await?;- let actual_fields = inferred_schema- .fields()- .iter()- .map(|f| format!("{}: {:?}", f.name(), f.data_type()))- .collect::<Vec<_>>();- assert_eq!(expected, actual_fields);- }-- Ok(())- }-- #[tokio::test]- async fn test_infer_schema_short_stream() -> Result<()> {- let mut bytes = std::fs::read("tests/data/example.arrow")?;- bytes.truncate(20); // should cause error that file shorter than expected- let location = Path::parse("example.arrow")?;- let in_memory_store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());- in_memory_store.put(&location, bytes.into()).await?;-- let session_ctx = SessionContext::new();- let state = session_ctx.state();- let object_meta = ObjectMeta {- location,- last_modified: DateTime::default(),- size: u64::MAX,- e_tag: None,- version: None,- };-- let arrow_format = ArrowFormat {};-- let store = Arc::new(ChunkedStore::new(in_memory_store.clone(), 7));- let err = arrow_format- .infer_schema(- &state,- &(store.clone() as Arc<dyn ObjectStore>),- std::slice::from_ref(&object_meta),- )- .await;-- assert!(err.is_err());- assert_eq!(- "Arrow error: Parser error: Unexpected end of byte stream for Arrow IPC file",- err.unwrap_err().to_string().lines().next().unwrap()- );-- Ok(())- }-}+//! Re-exports the [`datafusion_datasource_arrow::file_format`] module, and contains tests for it.+pub use datafusion_datasource_arrow::file_format::*;diff --git a/datafusion/core/src/datasource/physical_plan/arrow.rs b/datafusion/core/src/datasource/physical_plan/arrow.rsnew file mode 100644index 0000000000..392eaa8c4b--- /dev/null+++ b/datafusion/core/src/datasource/physical_plan/arrow.rs@@ -0,0 +1,23 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Reexports the [`datafusion_datasource_arrow::source`] module, containing [Arrow] based [`FileSource`].+//!+//! [Arrow]: https://arrow.apache.org/docs/python/ipc.html+//! [`FileSource`]: datafusion_datasource::file::FileSource++pub use datafusion_datasource_arrow::source::*;diff --git a/datafusion/core/src/datasource/physical_plan/mod.rs b/datafusion/core/src/datasource/physical_plan/mod.rsindex 3a9dedaa02..1ac292e260 100644--- a/datafusion/core/src/datasource/physical_plan/mod.rs+++ b/datafusion/core/src/datasource/physical_plan/mod.rs@@ -17,7 +17,7 @@//! Execution plans that read file formats-mod arrow_file;+pub mod arrow;pub mod csv;pub mod json;@@ -35,10 +35,9 @@ pub use datafusion_datasource_parquet::source::ParquetSource;#[cfg(feature = "parquet")]pub use datafusion_datasource_parquet::{ParquetFileMetrics, ParquetFileReaderFactory};-pub use arrow_file::ArrowSource;-pub use json::{JsonOpener, JsonSource};+pub use arrow::{ArrowOpener, ArrowSource};pub use csv::{CsvOpener, CsvSource};pub use datafusion_datasource::file::FileSource;pub use datafusion_datasource::file_groups::FileGroup;diff --git a/datafusion/datasource-arrow/Cargo.toml b/datafusion/datasource-arrow/Cargo.tomlnew file mode 100644index 0000000000..b3d1e3f2ac--- /dev/null+++ b/datafusion/datasource-arrow/Cargo.toml@@ -0,0 +1,64 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++[package]+name = "datafusion-datasource-arrow"+description = "datafusion-datasource-arrow"+readme = "README.md"+authors.workspace = true+edition.workspace = true+homepage.workspace = true+license.workspace = true+repository.workspace = true+rust-version.workspace = true+version.workspace = true++[package.metadata.docs.rs]+all-features = true++[dependencies]+arrow = { workspace = true }+arrow-ipc = { workspace = true }+async-trait = { workspace = true }+bytes = { workspace = true }+datafusion-common = { workspace = true, features = ["object_store"] }+datafusion-common-runtime = { workspace = true }+datafusion-datasource = { workspace = true }+datafusion-execution = { workspace = true }+datafusion-expr = { workspace = true }+datafusion-physical-expr-common = { workspace = true }+datafusion-physical-plan = { workspace = true }+datafusion-session = { workspace = true }+futures = { workspace = true }+itertools = { workspace = true }+object_store = { workspace = true }+tokio = { workspace = true }++[dev-dependencies]+chrono = { workspace = true }++[lints]+workspace = true++[lib]+name = "datafusion_datasource_arrow"+path = "src/mod.rs"++[features]+compression = [+ "arrow-ipc/zstd",+]diff --git a/datafusion/datasource-arrow/LICENSE.txt b/datafusion/datasource-arrow/LICENSE.txtnew file mode 100644index 0000000000..d74c6b599d--- /dev/null+++ b/datafusion/datasource-arrow/LICENSE.txt@@ -0,0 +1,212 @@++ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+++This project includes code from Apache Aurora.++* dev/release/{release,changelog,release-candidate} are based on the scripts from+ Apache Aurora++Copyright: 2016 The Apache Software Foundation.+Home page: https://aurora.apache.org/+License: http://www.apache.org/licenses/LICENSE-2.0diff --git a/datafusion/datasource-arrow/NOTICE.txt b/datafusion/datasource-arrow/NOTICE.txtnew file mode 100644index 0000000000..7f3c80d606--- /dev/null+++ b/datafusion/datasource-arrow/NOTICE.txt@@ -0,0 +1,5 @@+Apache DataFusion+Copyright 2019-2025 The Apache Software Foundation++This product includes software developed at+The Apache Software Foundation (http://www.apache.org/).diff --git a/datafusion/datasource-arrow/README.md b/datafusion/datasource-arrow/README.mdnew file mode 100644index 0000000000..9901b52105--- /dev/null+++ b/datafusion/datasource-arrow/README.md@@ -0,0 +1,34 @@+<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# Apache DataFusion Arrow DataSource++[Apache DataFusion] is an extensible query execution framework, written in Rust, that uses [Apache Arrow] as its in-memory format.++This crate is a submodule of DataFusion that defines a Arrow based file source.+It works with files following the [Arrow IPC format].++Most projects should use the [`datafusion`] crate directly, which re-exports+this module. If you are already using the [`datafusion`] crate, there is no+reason to use this crate directly in your project as well.++[apache arrow]: https://arrow.apache.org/+[apache datafusion]: https://datafusion.apache.org/+[`datafusion`]: https://crates.io/crates/datafusion+[arrow ipc format]: https://arrow.apache.org/docs/format/Columnar.html#ipc-file-formatdiff --git a/datafusion/datasource-arrow/src/file_format.rs b/datafusion/datasource-arrow/src/file_format.rsnew file mode 100644index 0000000000..3b85640804--- /dev/null+++ b/datafusion/datasource-arrow/src/file_format.rs@@ -0,0 +1,603 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! [`ArrowFormat`]: Apache Arrow [`FileFormat`] abstractions+//!+//! Works with files following the [Arrow IPC format](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format)++use std::any::Any;+use std::borrow::Cow;+use std::collections::HashMap;+use std::fmt::{self, Debug};+use std::sync::Arc;++use arrow::datatypes::{Schema, SchemaRef};+use arrow::error::ArrowError;+use arrow::ipc::convert::fb_to_schema;+use arrow::ipc::reader::FileReader;+use arrow::ipc::writer::IpcWriteOptions;+use arrow::ipc::{root_as_message, CompressionType};+use datafusion_common::error::Result;+use datafusion_common::parsers::CompressionTypeVariant;+use datafusion_common::{+ internal_datafusion_err, not_impl_err, DataFusionError, GetExt, Statistics,+ DEFAULT_ARROW_EXTENSION,+};+use datafusion_common_runtime::{JoinSet, SpawnedTask};+use datafusion_datasource::display::FileGroupDisplay;+use datafusion_datasource::file::FileSource;+use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};+use datafusion_datasource::sink::{DataSink, DataSinkExec};+use datafusion_datasource::write::{+ get_writer_schema, ObjectWriterBuilder, SharedBuffer,+};+use datafusion_execution::{SendableRecordBatchStream, TaskContext};+use datafusion_expr::dml::InsertOp;+use datafusion_physical_expr_common::sort_expr::LexRequirement;++use crate::source::ArrowSource;+use async_trait::async_trait;+use bytes::Bytes;+use datafusion_datasource::file_compression_type::FileCompressionType;+use datafusion_datasource::file_format::{FileFormat, FileFormatFactory};+use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};+use datafusion_datasource::source::DataSourceExec;+use datafusion_datasource::write::demux::DemuxedStreamReceiver;+use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan};+use datafusion_session::Session;+use futures::stream::BoxStream;+use futures::StreamExt;+use object_store::{GetResultPayload, ObjectMeta, ObjectStore};+use tokio::io::AsyncWriteExt;++/// Initial writing buffer size. Note this is just a size hint for efficiency. It+/// will grow beyond the set value if needed.+const INITIAL_BUFFER_BYTES: usize = 1048576;++/// If the buffered Arrow data exceeds this size, it is flushed to object store+const BUFFER_FLUSH_BYTES: usize = 1024000;++#[derive(Default, Debug)]+/// Factory struct used to create [ArrowFormat]+pub struct ArrowFormatFactory;++impl ArrowFormatFactory {+ /// Creates an instance of [ArrowFormatFactory]+ pub fn new() -> Self {+ Self {}+ }+}++impl FileFormatFactory for ArrowFormatFactory {+ fn create(+ &self,+ _state: &dyn Session,+ _format_options: &HashMap<String, String>,+ ) -> Result<Arc<dyn FileFormat>> {+ Ok(Arc::new(ArrowFormat))+ }++ fn default(&self) -> Arc<dyn FileFormat> {+ Arc::new(ArrowFormat)+ }++ fn as_any(&self) -> &dyn Any {+ self+ }+}++impl GetExt for ArrowFormatFactory {+ fn get_ext(&self) -> String {+ // Removes the dot, i.e. ".parquet" -> "parquet"+ DEFAULT_ARROW_EXTENSION[1..].to_string()+ }+}++/// Arrow `FileFormat` implementation.+#[derive(Default, Debug)]+pub struct ArrowFormat;++#[async_trait]+impl FileFormat for ArrowFormat {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn get_ext(&self) -> String {+ ArrowFormatFactory::new().get_ext()+ }++ fn get_ext_with_compression(+ &self,+ file_compression_type: &FileCompressionType,+ ) -> Result<String> {+ let ext = self.get_ext();+ match file_compression_type.get_variant() {+ CompressionTypeVariant::UNCOMPRESSED => Ok(ext),+ _ => Err(internal_datafusion_err!(+ "Arrow FileFormat does not support compression."+ )),+ }+ }++ fn compression_type(&self) -> Option<FileCompressionType> {+ None+ }++ async fn infer_schema(+ &self,+ _state: &dyn Session,+ store: &Arc<dyn ObjectStore>,+ objects: &[ObjectMeta],+ ) -> Result<SchemaRef> {+ let mut schemas = vec![];+ for object in objects {+ let r = store.as_ref().get(&object.location).await?;+ let schema = match r.payload {+ #[cfg(not(target_arch = "wasm32"))]+ GetResultPayload::File(mut file, _) => {+ let reader = FileReader::try_new(&mut file, None)?;+ reader.schema()+ }+ GetResultPayload::Stream(stream) => {+ infer_schema_from_file_stream(stream).await?+ }+ };+ schemas.push(schema.as_ref().clone());+ }+ let merged_schema = Schema::try_merge(schemas)?;+ Ok(Arc::new(merged_schema))+ }++ async fn infer_stats(+ &self,+ _state: &dyn Session,+ _store: &Arc<dyn ObjectStore>,+ table_schema: SchemaRef,+ _object: &ObjectMeta,+ ) -> Result<Statistics> {+ Ok(Statistics::new_unknown(&table_schema))+ }++ async fn create_physical_plan(+ &self,+ _state: &dyn Session,+ conf: FileScanConfig,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let source = Arc::new(ArrowSource::default());+ let config = FileScanConfigBuilder::from(conf)+ .with_source(source)+ .build();++ Ok(DataSourceExec::from_data_source(config))+ }++ async fn create_writer_physical_plan(+ &self,+ input: Arc<dyn ExecutionPlan>,+ _state: &dyn Session,+ conf: FileSinkConfig,+ order_requirements: Option<LexRequirement>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ if conf.insert_op != InsertOp::Append {+ return not_impl_err!("Overwrites are not implemented yet for Arrow format");+ }++ let sink = Arc::new(ArrowFileSink::new(conf));++ Ok(Arc::new(DataSinkExec::new(input, sink, order_requirements)) as _)+ }++ fn file_source(&self) -> Arc<dyn FileSource> {+ Arc::new(ArrowSource::default())+ }+}++/// Implements [`FileSink`] for writing to arrow_ipc files+struct ArrowFileSink {+ config: FileSinkConfig,+}++impl ArrowFileSink {+ fn new(config: FileSinkConfig) -> Self {+ Self { config }+ }+}++#[async_trait]+impl FileSink for ArrowFileSink {+ fn config(&self) -> &FileSinkConfig {+ &self.config+ }++ async fn spawn_writer_tasks_and_join(+ &self,+ context: &Arc<TaskContext>,+ demux_task: SpawnedTask<Result<()>>,+ mut file_stream_rx: DemuxedStreamReceiver,+ object_store: Arc<dyn ObjectStore>,+ ) -> Result<u64> {+ let mut file_write_tasks: JoinSet<std::result::Result<usize, DataFusionError>> =+ JoinSet::new();++ let ipc_options =+ IpcWriteOptions::try_new(64, false, arrow_ipc::MetadataVersion::V5)?+ .try_with_compression(Some(CompressionType::LZ4_FRAME))?;+ while let Some((path, mut rx)) = file_stream_rx.recv().await {+ let shared_buffer = SharedBuffer::new(INITIAL_BUFFER_BYTES);+ let mut arrow_writer = arrow_ipc::writer::FileWriter::try_new_with_options(+ shared_buffer.clone(),+ &get_writer_schema(&self.config),+ ipc_options.clone(),+ )?;+ let mut object_store_writer = ObjectWriterBuilder::new(+ FileCompressionType::UNCOMPRESSED,+ &path,+ Arc::clone(&object_store),+ )+ .with_buffer_size(Some(+ context+ .session_config()+ .options()+ .execution+ .objectstore_writer_buffer_size,+ ))+ .build()?;+ file_write_tasks.spawn(async move {+ let mut row_count = 0;+ while let Some(batch) = rx.recv().await {+ row_count += batch.num_rows();+ arrow_writer.write(&batch)?;+ let mut buff_to_flush = shared_buffer.buffer.try_lock().unwrap();+ if buff_to_flush.len() > BUFFER_FLUSH_BYTES {+ object_store_writer+ .write_all(buff_to_flush.as_slice())+ .await?;+ buff_to_flush.clear();+ }+ }+ arrow_writer.finish()?;+ let final_buff = shared_buffer.buffer.try_lock().unwrap();++ object_store_writer.write_all(final_buff.as_slice()).await?;+ object_store_writer.shutdown().await?;+ Ok(row_count)+ });+ }++ let mut row_count = 0;+ while let Some(result) = file_write_tasks.join_next().await {+ match result {+ Ok(r) => {+ row_count += r?;+ }+ Err(e) => {+ if e.is_panic() {+ std::panic::resume_unwind(e.into_panic());+ } else {+ unreachable!();+ }+ }+ }+ }++ demux_task+ .join_unwind()+ .await+ .map_err(|e| DataFusionError::ExecutionJoin(Box::new(e)))??;+ Ok(row_count as u64)+ }+}++impl Debug for ArrowFileSink {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("ArrowFileSink").finish()+ }+}++impl DisplayAs for ArrowFileSink {+ fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ match t {+ DisplayFormatType::Default | DisplayFormatType::Verbose => {+ write!(f, "ArrowFileSink(file_groups=",)?;+ FileGroupDisplay(&self.config.file_group).fmt_as(t, f)?;+ write!(f, ")")+ }+ DisplayFormatType::TreeRender => {+ writeln!(f, "format: arrow")?;+ write!(f, "file={}", &self.config.original_url)+ }+ }+ }+}++#[async_trait]+impl DataSink for ArrowFileSink {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn schema(&self) -> &SchemaRef {+ self.config.output_schema()+ }++ async fn write_all(+ &self,+ data: SendableRecordBatchStream,+ context: &Arc<TaskContext>,+ ) -> Result<u64> {+ FileSink::write_all(self, data, context).await+ }+}++const ARROW_MAGIC: [u8; 6] = [b'A', b'R', b'R', b'O', b'W', b'1'];+const CONTINUATION_MARKER: [u8; 4] = [0xff; 4];++/// Custom implementation of inferring schema. Should eventually be moved upstream to arrow-rs.+/// See <https://github.com/apache/arrow-rs/issues/5021>+async fn infer_schema_from_file_stream(+ mut stream: BoxStream<'static, object_store::Result<Bytes>>,+) -> Result<SchemaRef> {+ // Expected format:+ // <magic number "ARROW1"> - 6 bytes+ // <empty padding bytes [to 8 byte boundary]> - 2 bytes+ // <continuation: 0xFFFFFFFF> - 4 bytes, not present below v0.15.0+ // <metadata_size: int32> - 4 bytes+ // <metadata_flatbuffer: bytes>+ // <rest of file bytes>++ // So in first read we need at least all known sized sections,+ // which is 6 + 2 + 4 + 4 = 16 bytes.+ let bytes = collect_at_least_n_bytes(&mut stream, 16, None).await?;++ // Files should start with these magic bytes+ if bytes[0..6] != ARROW_MAGIC {+ return Err(ArrowError::ParseError(+ "Arrow file does not contain correct header".to_string(),+ ))?;+ }++ // Since continuation marker bytes added in later versions+ let (meta_len, rest_of_bytes_start_index) = if bytes[8..12] == CONTINUATION_MARKER {+ (&bytes[12..16], 16)+ } else {+ (&bytes[8..12], 12)+ };++ let meta_len = [meta_len[0], meta_len[1], meta_len[2], meta_len[3]];+ let meta_len = i32::from_le_bytes(meta_len);++ // Read bytes for Schema message+ let block_data = if bytes[rest_of_bytes_start_index..].len() < meta_len as usize {+ // Need to read more bytes to decode Message+ let mut block_data = Vec::with_capacity(meta_len as usize);+ // In case we had some spare bytes in our initial read chunk+ block_data.extend_from_slice(&bytes[rest_of_bytes_start_index..]);+ let size_to_read = meta_len as usize - block_data.len();+ let block_data =+ collect_at_least_n_bytes(&mut stream, size_to_read, Some(block_data)).await?;+ Cow::Owned(block_data)+ } else {+ // Already have the bytes we need+ let end_index = meta_len as usize + rest_of_bytes_start_index;+ let block_data = &bytes[rest_of_bytes_start_index..end_index];+ Cow::Borrowed(block_data)+ };++ // Decode Schema message+ let message = root_as_message(&block_data).map_err(|err| {+ ArrowError::ParseError(format!("Unable to read IPC message as metadata: {err:?}"))+ })?;+ let ipc_schema = message.header_as_schema().ok_or_else(|| {+ ArrowError::IpcError("Unable to read IPC message as schema".to_string())+ })?;+ let schema = fb_to_schema(ipc_schema);++ Ok(Arc::new(schema))+}++async fn collect_at_least_n_bytes(+ stream: &mut BoxStream<'static, object_store::Result<Bytes>>,+ n: usize,+ extend_from: Option<Vec<u8>>,+) -> Result<Vec<u8>> {+ let mut buf = extend_from.unwrap_or_else(|| Vec::with_capacity(n));+ // If extending existing buffer then ensure we read n additional bytes+ let n = n + buf.len();+ while let Some(bytes) = stream.next().await.transpose()? {+ buf.extend_from_slice(&bytes);+ if buf.len() >= n {+ break;+ }+ }+ if buf.len() < n {+ return Err(ArrowError::ParseError(+ "Unexpected end of byte stream for Arrow IPC file".to_string(),+ ))?;+ }+ Ok(buf)+}++#[cfg(test)]+mod tests {+ use super::*;++ use chrono::DateTime;+ use datafusion_common::config::TableOptions;+ use datafusion_common::DFSchema;+ use datafusion_execution::config::SessionConfig;+ use datafusion_execution::runtime_env::RuntimeEnv;+ use datafusion_expr::execution_props::ExecutionProps;+ use datafusion_expr::{AggregateUDF, Expr, LogicalPlan, ScalarUDF, WindowUDF};+ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;+ use object_store::{chunked::ChunkedStore, memory::InMemory, path::Path};++ struct MockSession {+ config: SessionConfig,+ runtime_env: Arc<RuntimeEnv>,+ }++ impl MockSession {+ fn new() -> Self {+ Self {+ config: SessionConfig::new(),+ runtime_env: Arc::new(RuntimeEnv::default()),+ }+ }+ }++ #[async_trait::async_trait]+ impl Session for MockSession {+ fn session_id(&self) -> &str {+ unimplemented!()+ }++ fn config(&self) -> &SessionConfig {+ &self.config+ }++ async fn create_physical_plan(+ &self,+ _logical_plan: &LogicalPlan,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ unimplemented!()+ }++ fn create_physical_expr(+ &self,+ _expr: Expr,+ _df_schema: &DFSchema,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ unimplemented!()+ }++ fn scalar_functions(&self) -> &HashMap<String, Arc<ScalarUDF>> {+ unimplemented!()+ }++ fn aggregate_functions(&self) -> &HashMap<String, Arc<AggregateUDF>> {+ unimplemented!()+ }++ fn window_functions(&self) -> &HashMap<String, Arc<WindowUDF>> {+ unimplemented!()+ }++ fn runtime_env(&self) -> &Arc<RuntimeEnv> {+ &self.runtime_env+ }++ fn execution_props(&self) -> &ExecutionProps {+ unimplemented!()+ }++ fn as_any(&self) -> &dyn Any {+ unimplemented!()+ }++ fn table_options(&self) -> &TableOptions {+ unimplemented!()+ }++ fn table_options_mut(&mut self) -> &mut TableOptions {+ unimplemented!()+ }++ fn task_ctx(&self) -> Arc<TaskContext> {+ unimplemented!()+ }+ }++ #[tokio::test]+ async fn test_infer_schema_stream() -> Result<()> {+ let mut bytes = std::fs::read("tests/data/example.arrow")?;+ bytes.truncate(bytes.len() - 20); // mangle end to show we don't need to read whole file+ let location = Path::parse("example.arrow")?;+ let in_memory_store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());+ in_memory_store.put(&location, bytes.into()).await?;++ let state = MockSession::new();+ let object_meta = ObjectMeta {+ location,+ last_modified: DateTime::default(),+ size: u64::MAX,+ e_tag: None,+ version: None,+ };++ let arrow_format = ArrowFormat {};+ let expected = vec!["f0: Int64", "f1: Utf8", "f2: Boolean"];++ // Test chunk sizes where too small so we keep having to read more bytes+ // And when large enough that first read contains all we need+ for chunk_size in [7, 3000] {+ let store = Arc::new(ChunkedStore::new(in_memory_store.clone(), chunk_size));+ let inferred_schema = arrow_format+ .infer_schema(+ &state,+ &(store.clone() as Arc<dyn ObjectStore>),+ std::slice::from_ref(&object_meta),+ )+ .await?;+ let actual_fields = inferred_schema+ .fields()+ .iter()+ .map(|f| format!("{}: {:?}", f.name(), f.data_type()))+ .collect::<Vec<_>>();+ assert_eq!(expected, actual_fields);+ }++ Ok(())+ }++ #[tokio::test]+ async fn test_infer_schema_short_stream() -> Result<()> {+ let mut bytes = std::fs::read("tests/data/example.arrow")?;+ bytes.truncate(20); // should cause error that file shorter than expected+ let location = Path::parse("example.arrow")?;+ let in_memory_store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());+ in_memory_store.put(&location, bytes.into()).await?;++ let state = MockSession::new();+ let object_meta = ObjectMeta {+ location,+ last_modified: DateTime::default(),+ size: u64::MAX,+ e_tag: None,+ version: None,+ };++ let arrow_format = ArrowFormat {};++ let store = Arc::new(ChunkedStore::new(in_memory_store.clone(), 7));+ let err = arrow_format+ .infer_schema(+ &state,+ &(store.clone() as Arc<dyn ObjectStore>),+ std::slice::from_ref(&object_meta),+ )+ .await;++ assert!(err.is_err());+ assert_eq!(+ "Arrow error: Parser error: Unexpected end of byte stream for Arrow IPC file",+ err.unwrap_err().to_string().lines().next().unwrap()+ );++ Ok(())+ }+}diff --git a/datafusion/datasource-arrow/src/mod.rs b/datafusion/datasource-arrow/src/mod.rsnew file mode 100644index 0000000000..18bb8792c3--- /dev/null+++ b/datafusion/datasource-arrow/src/mod.rs@@ -0,0 +1,25 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++// Make sure fast / cheap clones on Arc are explicit:+// https://github.com/apache/datafusion/issues/11143+#![cfg_attr(not(test), deny(clippy::clone_on_ref_ptr))]++pub mod file_format;+pub mod source;++pub use file_format::*;diff --git a/datafusion/core/src/datasource/physical_plan/arrow_file.rs b/datafusion/datasource-arrow/src/source.rssimilarity index 98%rename from datafusion/core/src/datasource/physical_plan/arrow_file.rsrename to datafusion/datasource-arrow/src/source.rsindex b37dc499d4..f43f118801 100644--- a/datafusion/core/src/datasource/physical_plan/arrow_file.rs+++ b/datafusion/datasource-arrow/src/source.rs@@ -18,20 +18,21 @@use std::any::Any;use std::sync::Arc;-use crate::datasource::physical_plan::{FileOpenFuture, FileOpener};-use crate::error::Result;use datafusion_datasource::as_file_source;use datafusion_datasource::schema_adapter::SchemaAdapterFactory;use arrow::buffer::Buffer;use arrow::datatypes::SchemaRef;use arrow_ipc::reader::FileDecoder;+use datafusion_common::error::Result;use datafusion_common::{exec_datafusion_err, Statistics};use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_datasource::PartitionedFile;use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;+use datafusion_datasource::file_stream::FileOpenFuture;+use datafusion_datasource::file_stream::FileOpener;use futures::StreamExt;use itertools::Itertools;use object_store::{GetOptions, GetRange, GetResultPayload, ObjectStore};diff --git a/datafusion/core/tests/data/example.arrow b/datafusion/datasource-arrow/tests/data/example.arrowsimilarity index 100%rename from datafusion/core/tests/data/example.arrowrename to datafusion/datasource-arrow/tests/data/example.arrowdiff --git a/datafusion/sqllogictest/test_files/arrow_files.slt b/datafusion/sqllogictest/test_files/arrow_files.sltindex 62453ec4bf..b3975e0c3f 100644--- a/datafusion/sqllogictest/test_files/arrow_files.slt+++ b/datafusion/sqllogictest/test_files/arrow_files.slt@@ -29,7 +29,7 @@ statement okCREATE EXTERNAL TABLE arrow_simpleSTORED AS ARROW-LOCATION '../core/tests/data/example.arrow';+LOCATION '../datasource-arrow/tests/data/example.arrow';# physical plan@@ -37,7 +37,7 @@ query TTEXPLAIN SELECT * FROM arrow_simple----logical_plan TableScan: arrow_simple projection=[f0, f1, f2]-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/example.arrow]]}, projection=[f0, f1, f2], file_type=arrow+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/datasource-arrow/tests/data/example.arrow]]}, projection=[f0, f1, f2], file_type=arrow# correct contentquery ITB@@ -50,8 +50,8 @@ SELECT * FROM arrow_simple# Ensure that local files can not be read by default (a potential security issue)# (url table is only supported when DynamicFileCatalog is enabled)-statement error DataFusion error: Error during planning: table 'datafusion.public.../core/tests/data/example.arrow' not found-SELECT * FROM '../core/tests/data/example.arrow';+statement error DataFusion error: Error during planning: table 'datafusion.public.../datasource-arrow/tests/data/example.arrow' not found+SELECT * FROM '../datasource-arrow/tests/data/example.arrow';# ARROW partitioned tablestatement okdiff --git a/datafusion/sqllogictest/test_files/ddl.slt b/datafusion/sqllogictest/test_files/ddl.sltindex 03ef08e1a5..bc6cbfab0c 100644--- a/datafusion/sqllogictest/test_files/ddl.slt+++ b/datafusion/sqllogictest/test_files/ddl.slt@@ -312,7 +312,7 @@ DROP TABLE aggregate_simple# Arrow formatstatement ok-CREATE external table arrow_simple STORED as ARROW LOCATION '../core/tests/data/example.arrow';+CREATE external table arrow_simple STORED as ARROW LOCATION '../datasource-arrow/tests/data/example.arrow';query ITB rowsortSELECT * FROM arrow_simple order by f1 LIMIT 1@@ -796,7 +796,7 @@ logical_plan02)--Values: (Int64(1), Int64(2), Int64(3))query TT-explain CREATE EXTERNAL TEMPORARY TABLE tty STORED as ARROW LOCATION '../core/tests/data/example.arrow';+explain CREATE EXTERNAL TEMPORARY TABLE tty STORED as ARROW LOCATION '../datasource-arrow/tests/data/example.arrow';----logical_plan CreateExternalTable: Bare { table: "tty" }@@ -804,7 +804,7 @@ statement okset datafusion.explain.logical_plan_only=false;statement error DataFusion error: This feature is not implemented: Temporary tables not supported-CREATE EXTERNAL TEMPORARY TABLE tty STORED as ARROW LOCATION '../core/tests/data/example.arrow';+CREATE EXTERNAL TEMPORARY TABLE tty STORED as ARROW LOCATION '../datasource-arrow/tests/data/example.arrow';statement error DataFusion error: This feature is not implemented: Temporary views not supportedCREATE TEMPORARY VIEW y AS VALUES (1,2,3);diff --git a/datafusion/sqllogictest/test_files/repartition_scan.slt b/datafusion/sqllogictest/test_files/repartition_scan.sltindex c536c8165c..41718b3aeb 100644--- a/datafusion/sqllogictest/test_files/repartition_scan.slt+++ b/datafusion/sqllogictest/test_files/repartition_scan.slt@@ -244,7 +244,7 @@ DROP TABLE json_table;statement okCREATE EXTERNAL TABLE arrow_tableSTORED AS ARROW-LOCATION '../core/tests/data/example.arrow';+LOCATION '../datasource-arrow/tests/data/example.arrow';# It would be great to see the file read as "4" groups with even sizes (offsets) eventually@@ -253,7 +253,7 @@ query TTEXPLAIN SELECT * FROM arrow_table----logical_plan TableScan: arrow_table projection=[f0, f1, f2]-physical_plan DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/example.arrow:0..461], [WORKSPACE_ROOT/datafusion/core/tests/data/example.arrow:461..922], [WORKSPACE_ROOT/datafusion/core/tests/data/example.arrow:922..1383], [WORKSPACE_ROOT/datafusion/core/tests/data/example.arrow:1383..1842]]}, projection=[f0, f1, f2], file_type=arrow+physical_plan DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/datasource-arrow/tests/data/example.arrow:0..461], [WORKSPACE_ROOT/datafusion/datasource-arrow/tests/data/example.arrow:461..922], [WORKSPACE_ROOT/datafusion/datasource-arrow/tests/data/example.arrow:922..1383], [WORKSPACE_ROOT/datafusion/datasource-arrow/tests/data/example.arrow:1383..1842]]}, projection=[f0, f1, f2], file_type=arrow# correct contentquery ITBdiff --git a/dev/release/README.md b/dev/release/README.mdindex d70e256f73..1b78f8d13b 100644--- a/dev/release/README.md+++ b/dev/release/README.md@@ -295,6 +295,7 @@ Verify that the Cargo.toml in the tarball contains the correct version(cd datafusion/catalog && cargo publish)(cd datafusion/catalog-listing && cargo publish)(cd datafusion/functions-table && cargo publish)+(cd datafusion/datasource-arrow && cargo publish)(cd datafusion/datasource-csv && cargo publish)(cd datafusion/datasource-json && cargo publish)(cd datafusion/datasource-parquet && cargo publish)