Apache DataFusion SQL Query Engine
feat: create a datafusion-example for in-memory file format (#20394)
Kumar Ujjawal2b7d4f9a5b00
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex 2cf0ec5240..ebb1bc5fa6 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -88,18 +88,19 @@ cargo run --example dataframe -- dataframe#### Category: Single Process-| Subcommand | File Path | Description |-| -------------------- | ----------------------------------------------------------------------------------------- | ------------------------------------------------------ |-| catalog | [`data_io/catalog.rs`](examples/data_io/catalog.rs) | Register tables into a custom catalog |-| json_shredding | [`data_io/json_shredding.rs`](examples/data_io/json_shredding.rs) | Implement filter rewriting for JSON shredding |-| parquet_adv_idx | [`data_io/parquet_advanced_index.rs`](examples/data_io/parquet_advanced_index.rs) | Create a secondary index across multiple parquet files |-| parquet_emb_idx | [`data_io/parquet_embedded_index.rs`](examples/data_io/parquet_embedded_index.rs) | Store a custom index inside Parquet files |-| parquet_enc | [`data_io/parquet_encrypted.rs`](examples/data_io/parquet_encrypted.rs) | Read & write encrypted Parquet files |-| parquet_enc_with_kms | [`data_io/parquet_encrypted_with_kms.rs`](examples/data_io/parquet_encrypted_with_kms.rs) | Encrypted Parquet I/O using a KMS-backed factory |-| parquet_exec_visitor | [`data_io/parquet_exec_visitor.rs`](examples/data_io/parquet_exec_visitor.rs) | Extract statistics by visiting an ExecutionPlan |-| parquet_idx | [`data_io/parquet_index.rs`](examples/data_io/parquet_index.rs) | Create a secondary index |-| query_http_csv | [`data_io/query_http_csv.rs`](examples/data_io/query_http_csv.rs) | Query CSV files via HTTP |-| remote_catalog | [`data_io/remote_catalog.rs`](examples/data_io/remote_catalog.rs) | Interact with a remote catalog |+| Subcommand | File Path | Description |+| ---------------------- | ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- |+| catalog | [`data_io/catalog.rs`](examples/data_io/catalog.rs) | Register tables into a custom catalog |+| in_memory_object_store | [`data_io/in_memory_object_store.rs`](examples/data_io/in_memory_object_store.rs) | Read CSV from an in-memory object store (pattern applies to JSON/Parquet) |+| json_shredding | [`data_io/json_shredding.rs`](examples/data_io/json_shredding.rs) | Implement filter rewriting for JSON shredding |+| parquet_adv_idx | [`data_io/parquet_advanced_index.rs`](examples/data_io/parquet_advanced_index.rs) | Create a secondary index across multiple parquet files |+| parquet_emb_idx | [`data_io/parquet_embedded_index.rs`](examples/data_io/parquet_embedded_index.rs) | Store a custom index inside Parquet files |+| parquet_enc | [`data_io/parquet_encrypted.rs`](examples/data_io/parquet_encrypted.rs) | Read & write encrypted Parquet files |+| parquet_enc_with_kms | [`data_io/parquet_encrypted_with_kms.rs`](examples/data_io/parquet_encrypted_with_kms.rs) | Encrypted Parquet I/O using a KMS-backed factory |+| parquet_exec_visitor | [`data_io/parquet_exec_visitor.rs`](examples/data_io/parquet_exec_visitor.rs) | Extract statistics by visiting an ExecutionPlan |+| parquet_idx | [`data_io/parquet_index.rs`](examples/data_io/parquet_index.rs) | Create a secondary index |+| query_http_csv | [`data_io/query_http_csv.rs`](examples/data_io/query_http_csv.rs) | Query CSV files via HTTP |+| remote_catalog | [`data_io/remote_catalog.rs`](examples/data_io/remote_catalog.rs) | Interact with a remote catalog |## DataFrame Examplesdiff --git a/datafusion-examples/examples/data_io/in_memory_object_store.rs b/datafusion-examples/examples/data_io/in_memory_object_store.rsnew file mode 100644index 0000000000..9a308f06c5--- /dev/null+++ b/datafusion-examples/examples/data_io/in_memory_object_store.rs@@ -0,0 +1,81 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! See `main.rs` for how to run it.+//!+//! This follows the recommended approach: implement the `ObjectStore` trait+//! (or use an existing implementation), register it with DataFusion, and then+//! read a URL "path" from that store.+//! See the in-memory reference implementation:+//! https://docs.rs/object_store/latest/object_store/memory/struct.InMemory.html++use std::sync::Arc;++use arrow::datatypes::{DataType, Field, Schema};+use datafusion::assert_batches_eq;+use datafusion::common::Result;+use datafusion::execution::object_store::ObjectStoreUrl;+use datafusion::prelude::{CsvReadOptions, SessionContext};+use object_store::memory::InMemory;+use object_store::path::Path;+use object_store::{ObjectStore, ObjectStoreExt, PutPayload};++/// Demonstrates reading CSV data from an in-memory object store.+///+/// The same pattern applies to JSON/Parquet: register a store for a URL+/// prefix, write bytes into the store, then read via that URL.+pub async fn in_memory_object_store() -> Result<()> {+ let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());+ let ctx = SessionContext::new();+ let object_store_url = ObjectStoreUrl::parse("memory://")?;+ // Register a URL prefix to route reads through this object store.+ ctx.register_object_store(object_store_url.as_ref(), Arc::clone(&store));++ let schema = Schema::new(vec![+ Field::new("id", DataType::Int64, false),+ Field::new("name", DataType::Utf8, false),+ ]);++ println!("=== CSV from memory ===");+ let csv_path = Path::from("/people.csv");+ let csv_data = b"id,name\n1,Alice\n2,Bob\n";+ // Write bytes into the in-memory object store.+ store+ .put(&csv_path, PutPayload::from_static(csv_data))+ .await?;+ // Read using the URL that matches the registered prefix.+ let csv = ctx+ .read_csv(+ "memory:///people.csv",+ CsvReadOptions::new().schema(&schema),+ )+ .await?+ .collect()+ .await?;+ #[rustfmt::skip]+ let expected = [+ "+----+-------+",+ "| id | name |",+ "+----+-------+",+ "| 1 | Alice |",+ "| 2 | Bob |",+ "+----+-------+",+ ];+ assert_batches_eq!(expected, &csv);++ Ok(())+}diff --git a/datafusion-examples/examples/data_io/main.rs b/datafusion-examples/examples/data_io/main.rsindex 0039585d15..4656a83670 100644--- a/datafusion-examples/examples/data_io/main.rs+++ b/datafusion-examples/examples/data_io/main.rs@@ -21,7 +21,7 @@//!//! ## Usage//! ```bash-//! cargo run --example data_io -- [all|catalog|json_shredding|parquet_adv_idx|parquet_emb_idx|parquet_enc_with_kms|parquet_enc|parquet_exec_visitor|parquet_idx|query_http_csv|remote_catalog]+//! cargo run --example data_io -- [all|catalog|in_memory_object_store|json_shredding|parquet_adv_idx|parquet_emb_idx|parquet_enc_with_kms|parquet_enc|parquet_exec_visitor|parquet_idx|query_http_csv|remote_catalog]//! ```//!//! Each subcommand runs a corresponding example:@@ -30,6 +30,9 @@//! - `catalog`//! (file: catalog.rs, desc: Register tables into a custom catalog)//!+//! - `in_memory_object_store`+//! (file: in_memory_object_store.rs, desc: Read CSV from an in-memory object store (pattern applies to JSON/Parquet))+//!//! - `json_shredding`//! (file: json_shredding.rs, desc: Implement filter rewriting for JSON shredding)//!@@ -58,6 +61,7 @@//! (file: remote_catalog.rs, desc: Interact with a remote catalog)mod catalog;+mod in_memory_object_store;mod json_shredding;mod parquet_advanced_index;mod parquet_embedded_index;@@ -77,6 +81,7 @@ use strum_macros::{Display, EnumIter, EnumString, VariantNames};enum ExampleKind {All,Catalog,+ InMemoryObjectStore,JsonShredding,ParquetAdvIdx,ParquetEmbIdx,@@ -104,6 +109,9 @@ impl ExampleKind {}}ExampleKind::Catalog => catalog::catalog().await?,+ ExampleKind::InMemoryObjectStore => {+ in_memory_object_store::in_memory_object_store().await?+ }ExampleKind::JsonShredding => json_shredding::json_shredding().await?,ExampleKind::ParquetAdvIdx => {parquet_advanced_index::parquet_advanced_index().await?