Apache DataFusion SQL Query Engine
Consolidate data io examples (#18142) (#18591)
Sergey Zhukov8226ebf2e51f
diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex 62e51a7900..f2d185c928 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -49,10 +49,11 @@ cargo run --example dataframe- [`examples/udf/advanced_udaf.rs`](examples/udf/advanced_udaf.rs): Define and invoke a more complicated User Defined Aggregate Function (UDAF)- [`examples/udf/advanced_udf.rs`](examples/udf/advanced_udf.rs): Define and invoke a more complicated User Defined Scalar Function (UDF)- [`examples/udf/advanced_udwf.rs`](examples/udf/advanced_udwf.rs): Define and invoke a more complicated User Defined Window Function (UDWF)-- [`advanced_parquet_index.rs`](examples/advanced_parquet_index.rs): Creates a detailed secondary index that covers the contents of several parquet files+- [`examples/data_io/parquet_advanced_index.rs`](examples/data_io/parquet_advanced_index.rs): Creates a detailed secondary index that covers the contents of several parquet files- [`examples/udf/async_udf.rs`](examples/udf/async_udf.rs): Define and invoke an asynchronous User Defined Scalar Function (UDF)- [`analyzer_rule.rs`](examples/analyzer_rule.rs): Use a custom AnalyzerRule to change a query's semantics (row level access control)-- [`catalog.rs`](examples/catalog.rs): Register the table into a custom catalog+- [`examples/data_io/catalog.rs`](examples/data_io/catalog.rs): Register the table into a custom catalog+- [`examples/data_io/json_shredding.rs`](examples/data_io/json_shredding.rs): Shows how to implement custom filter rewriting for JSON shredding- [`composed_extension_codec`](examples/composed_extension_codec.rs): Example of using multiple extension codecs for serialization / deserialization- [`examples/custom_data_source/csv_sql_streaming.rs`](examples/custom_data_source/csv_sql_streaming.rs): Build and run a streaming query plan from a SQL statement against a local CSV file- [`examples/custom_data_source/csv_json_opener.rs`](examples/custom_data_source/csv_json_opener.rs): Use low level `FileOpener` APIs to read CSV/JSON into Arrow `RecordBatch`es@@ -71,19 +72,19 @@ cargo run --example dataframe- [`memory_pool_tracking.rs`](examples/memory_pool_tracking.rs): Demonstrates TrackConsumersPool for memory tracking and debugging with enhanced error messages- [`memory_pool_execution_plan.rs`](examples/memory_pool_execution_plan.rs): Shows how to implement memory-aware ExecutionPlan with memory reservation and spilling- [`optimizer_rule.rs`](examples/optimizer_rule.rs): Use a custom OptimizerRule to replace certain predicates-- [`parquet_embedded_index.rs`](examples/parquet_embedded_index.rs): Store a custom index inside a Parquet file and use it to speed up queries-- [`parquet_encrypted.rs`](examples/parquet_encrypted.rs): Read and write encrypted Parquet files using DataFusion-- [`parquet_encrypted_with_kms.rs`](examples/parquet_encrypted_with_kms.rs): Read and write encrypted Parquet files using an encryption factory-- [`parquet_index.rs`](examples/parquet_index.rs): Create an secondary index over several parquet files and use it to speed up queries-- [`parquet_exec_visitor.rs`](examples/parquet_exec_visitor.rs): Extract statistics by visiting an ExecutionPlan after execution+- [`examples/data_io/parquet_embedded_index.rs`](examples/data_io/parquet_embedded_index.rs): Store a custom index inside a Parquet file and use it to speed up queries+- [`examples/data_io/parquet_encrypted.rs`](examples/data_io/parquet_encrypted.rs): Read and write encrypted Parquet files using DataFusion+- [`examples/data_io/parquet_encrypted_with_kms.rs`](examples/data_io/parquet_encrypted_with_kms.rs): Read and write encrypted Parquet files using an encryption factory+- [`examples/data_io/parquet_index.rs`](examples/data_io/parquet_index.rs): Create an secondary index over several parquet files and use it to speed up queries+- [`examples/data_io/parquet_exec_visitor.rs`](examples/data_io/parquet_exec_visitor.rs): Extract statistics by visiting an ExecutionPlan after execution- [`parse_sql_expr.rs`](examples/parse_sql_expr.rs): Parse SQL text into DataFusion `Expr`.- [`plan_to_sql.rs`](examples/plan_to_sql.rs): Generate SQL from DataFusion `Expr` and `LogicalPlan`- [`planner_api.rs`](examples/planner_api.rs) APIs to manipulate logical and physical plans- [`pruning.rs`](examples/pruning.rs): Use pruning to rule out files based on statistics- [`query-aws-s3.rs`](examples/external_dependency/query-aws-s3.rs): Configure `object_store` and run a query against files stored in AWS S3-- [`query-http-csv.rs`](examples/query-http-csv.rs): Configure `object_store` and run a query against files vi HTTP+- [`examples/data_io/query_http_csv.rs`](examples/data_io/query_http_csv.rs): Configure `object_store` and run a query against files via HTTP- [`examples/builtin_functions/regexp.rs`](examples/builtin_functions/regexp.rs): Examples of using regular expression functions-- [`remote_catalog.rs`](examples/regexp.rs): Examples of interfacing with a remote catalog (e.g. over a network)+- [`examples/data_io/remote_catalog.rs`](examples/data_io/remote_catalog.rs): Examples of interfacing with a remote catalog (e.g. over a network)- [`examples/udf/simple_udaf.rs`](examples/udf/simple_udaf.rs): Define and invoke a User Defined Aggregate Function (UDAF)- [`examples/udf/simple_udf.rs`](examples/udf/simple_udf.rs): Define and invoke a User Defined Scalar Function (UDF)- [`examples/udf/simple_udtf.rs`](examples/udf/simple_udtf.rs): Define and invoke a User Defined Table Function (UDTF)diff --git a/datafusion-examples/examples/catalog.rs b/datafusion-examples/examples/data_io/catalog.rssimilarity index 99%rename from datafusion-examples/examples/catalog.rsrename to datafusion-examples/examples/data_io/catalog.rsindex 229867cdfc..11614d0747 100644--- a/datafusion-examples/examples/catalog.rs+++ b/datafusion-examples/examples/data_io/catalog.rs@@ -34,8 +34,8 @@ use std::{any::Any, collections::HashMap, path::Path, sync::Arc};use std::{fs::File, io::Write};use tempfile::TempDir;-#[tokio::main]-async fn main() -> Result<()> {+/// Register the table into a custom catalog+pub async fn catalog() -> Result<()> {env_logger::builder().filter_level(log::LevelFilter::Info).init();@@ -134,12 +134,14 @@ struct DirSchemaOpts<'a> {dir: &'a Path,format: Arc<dyn FileFormat>,}+/// Schema where every file with extension `ext` in a given `dir` is a table.#[derive(Debug)]struct DirSchema {ext: String,tables: RwLock<HashMap<String, Arc<dyn TableProvider>>>,}+impl DirSchema {async fn create(state: &SessionState, opts: DirSchemaOpts<'_>) -> Result<Arc<Self>> {let DirSchemaOpts { ext, dir, format } = opts;@@ -172,6 +174,7 @@ impl DirSchema {ext: ext.to_string(),}))}+#[allow(unused)]fn name(&self) -> &str {&self.ext@@ -198,6 +201,7 @@ impl SchemaProvider for DirSchema {let tables = self.tables.read().unwrap();tables.contains_key(name)}+fn register_table(&self,name: String,@@ -223,6 +227,7 @@ impl SchemaProvider for DirSchema {struct DirCatalog {schemas: RwLock<HashMap<String, Arc<dyn SchemaProvider>>>,}+impl DirCatalog {fn new() -> Self {Self {@@ -230,10 +235,12 @@ impl DirCatalog {}}}+impl CatalogProvider for DirCatalog {fn as_any(&self) -> &dyn Any {self}+fn register_schema(&self,name: &str,@@ -260,11 +267,13 @@ impl CatalogProvider for DirCatalog {}}}+/// Catalog lists holds multiple catalog providers. Each context has a single catalog list.#[derive(Debug)]struct CustomCatalogProviderList {catalogs: RwLock<HashMap<String, Arc<dyn CatalogProvider>>>,}+impl CustomCatalogProviderList {fn new() -> Self {Self {@@ -272,10 +281,12 @@ impl CustomCatalogProviderList {}}}+impl CatalogProviderList for CustomCatalogProviderList {fn as_any(&self) -> &dyn Any {self}+fn register_catalog(&self,name: String,diff --git a/datafusion-examples/examples/json_shredding.rs b/datafusion-examples/examples/data_io/json_shredding.rssimilarity index 99%rename from datafusion-examples/examples/json_shredding.rsrename to datafusion-examples/examples/data_io/json_shredding.rsindex 5ef8b59b64..d93b2167c1 100644--- a/datafusion-examples/examples/json_shredding.rs+++ b/datafusion-examples/examples/data_io/json_shredding.rs@@ -63,8 +63,7 @@ use object_store::{ObjectStore, PutPayload};// 1. Push down predicates for better filtering// 2. Avoid expensive JSON parsing at query time// 3. Leverage columnar storage benefits for the materialized fields-#[tokio::main]-async fn main() -> Result<()> {+pub async fn json_shredding() -> Result<()> {println!("=== Creating example data with flat columns and underscore prefixes ===");// Create sample data with flat columns using underscore prefixesdiff --git a/datafusion-examples/examples/data_io/main.rs b/datafusion-examples/examples/data_io/main.rsnew file mode 100644index 0000000000..c9ad793410--- /dev/null+++ b/datafusion-examples/examples/data_io/main.rs@@ -0,0 +1,160 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! # These examples of data formats and I/O+//!+//! These examples demonstrate data formats and I/O.+//!+//! ## Usage+//! ```bash+//! cargo run --example data_io -- [catalog|json_shredding|parquet_adv_idx|parquet_emb_idx|parquet_enc_with_kms|parquet_enc|parquet_exec_visitor|parquet_idx|query_http_csv|remote_catalog]+//! ```+//!+//! Each subcommand runs a corresponding example:+//! - `catalog` — register the table into a custom catalog+//! - `json_shredding` — shows how to implement custom filter rewriting for JSON shredding+//! - `parquet_adv_idx` — create a detailed secondary index that covers the contents of several parquet files+//! - `parquet_emb_idx` — store a custom index inside a Parquet file and use it to speed up queries+//! - `parquet_enc_with_kms` — read and write encrypted Parquet files using an encryption factory+//! - `parquet_enc` — read and write encrypted Parquet files using DataFusion+//! - `parquet_exec_visitor` — extract statistics by visiting an ExecutionPlan after execution+//! - `parquet_idx` — create an secondary index over several parquet files and use it to speed up queries+//! - `query_http_csv` — configure `object_store` and run a query against files via HTTP+//! - `remote_catalog` — interfacing with a remote catalog (e.g. over a network)++mod catalog;+mod json_shredding;+mod parquet_advanced_index;+mod parquet_embedded_index;+mod parquet_encrypted;+mod parquet_encrypted_with_kms;+mod parquet_exec_visitor;+mod parquet_index;+mod query_http_csv;+mod remote_catalog;++use std::str::FromStr;++use datafusion::error::{DataFusionError, Result};++enum ExampleKind {+ Catalog,+ JsonShredding,+ ParquetAdvIdx,+ ParquetEmbIdx,+ ParquetEnc,+ ParquetEncWithKms,+ ParquetExecVisitor,+ ParquetIdx,+ QueryHttpCsv,+ RemoteCatalog,+}++impl AsRef<str> for ExampleKind {+ fn as_ref(&self) -> &str {+ match self {+ Self::Catalog => "catalog",+ Self::JsonShredding => "json_shredding",+ Self::ParquetAdvIdx => "parquet_adv_idx",+ Self::ParquetEmbIdx => "parquet_emb_idx",+ Self::ParquetEnc => "parquet_enc",+ Self::ParquetEncWithKms => "parquet_enc_with_kms",+ Self::ParquetExecVisitor => "parquet_exec_visitor",+ Self::ParquetIdx => "parquet_idx",+ Self::QueryHttpCsv => "query_http_csv",+ Self::RemoteCatalog => "remote_catalog",+ }+ }+}++impl FromStr for ExampleKind {+ type Err = DataFusionError;++ fn from_str(s: &str) -> Result<Self> {+ match s {+ "catalog" => Ok(Self::Catalog),+ "json_shredding" => Ok(Self::JsonShredding),+ "parquet_adv_idx" => Ok(Self::ParquetAdvIdx),+ "parquet_emb_idx" => Ok(Self::ParquetEmbIdx),+ "parquet_enc" => Ok(Self::ParquetEnc),+ "parquet_enc_with_kms" => Ok(Self::ParquetEncWithKms),+ "parquet_exec_visitor" => Ok(Self::ParquetExecVisitor),+ "parquet_idx" => Ok(Self::ParquetIdx),+ "query_http_csv" => Ok(Self::QueryHttpCsv),+ "remote_catalog" => Ok(Self::RemoteCatalog),+ _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))),+ }+ }+}++impl ExampleKind {+ const ALL: [Self; 10] = [+ Self::Catalog,+ Self::JsonShredding,+ Self::ParquetAdvIdx,+ Self::ParquetEmbIdx,+ Self::ParquetEnc,+ Self::ParquetEncWithKms,+ Self::ParquetExecVisitor,+ Self::ParquetIdx,+ Self::QueryHttpCsv,+ Self::RemoteCatalog,+ ];++ const EXAMPLE_NAME: &str = "data_io";++ fn variants() -> Vec<&'static str> {+ Self::ALL.iter().map(|x| x.as_ref()).collect()+ }+}++#[tokio::main]+async fn main() -> Result<()> {+ let usage = format!(+ "Usage: cargo run --example {} -- [{}]",+ ExampleKind::EXAMPLE_NAME,+ ExampleKind::variants().join("|")+ );++ let arg = std::env::args().nth(1).ok_or_else(|| {+ eprintln!("{usage}");+ DataFusionError::Execution("Missing argument".to_string())+ })?;++ match arg.parse::<ExampleKind>()? {+ ExampleKind::Catalog => catalog::catalog().await?,+ ExampleKind::JsonShredding => json_shredding::json_shredding().await?,+ ExampleKind::ParquetAdvIdx => {+ parquet_advanced_index::parquet_advanced_index().await?+ }+ ExampleKind::ParquetEmbIdx => {+ parquet_embedded_index::parquet_embedded_index().await?+ }+ ExampleKind::ParquetEncWithKms => {+ parquet_encrypted_with_kms::parquet_encrypted_with_kms().await?+ }+ ExampleKind::ParquetEnc => parquet_encrypted::parquet_encrypted().await?,+ ExampleKind::ParquetExecVisitor => {+ parquet_exec_visitor::parquet_exec_visitor().await?+ }+ ExampleKind::ParquetIdx => parquet_index::parquet_index().await?,+ ExampleKind::QueryHttpCsv => query_http_csv::query_http_csv().await?,+ ExampleKind::RemoteCatalog => remote_catalog::remote_catalog().await?,+ }++ Ok(())+}diff --git a/datafusion-examples/examples/advanced_parquet_index.rs b/datafusion-examples/examples/data_io/parquet_advanced_index.rssimilarity index 99%rename from datafusion-examples/examples/advanced_parquet_index.rsrename to datafusion-examples/examples/data_io/parquet_advanced_index.rsindex 67bfc5b1bc..af1e03fe4d 100644--- a/datafusion-examples/examples/advanced_parquet_index.rs+++ b/datafusion-examples/examples/data_io/parquet_advanced_index.rs@@ -155,8 +155,7 @@ use url::Url;////// [`ListingTable`]: datafusion::datasource::listing::ListingTable/// [Page Index](https://github.com/apache/parquet-format/blob/master/PageIndex.md)-#[tokio::main]-async fn main() -> Result<()> {+pub async fn parquet_advanced_index() -> Result<()> {// the object store is used to read the parquet files (in this case, it is// a local file system, but in a real system it could be S3, GCS, etc)let object_store: Arc<dyn ObjectStore> =@@ -239,6 +238,7 @@ pub struct IndexTableProvider {/// if true, use row selections in addition to row group selectionsuse_row_selections: AtomicBool,}+impl IndexTableProvider {/// Create a new IndexTableProvider/// * `object_store` - the object store implementation to use for reading files@@ -539,6 +539,7 @@ impl CachedParquetFileReaderFactory {metadata: HashMap::new(),}}+/// Add the pre-parsed information about the file to the factorfn with_file(mut self, indexed_file: &IndexedFile) -> Self {self.metadata.insert(diff --git a/datafusion-examples/examples/parquet_embedded_index.rs b/datafusion-examples/examples/data_io/parquet_embedded_index.rssimilarity index 99%rename from datafusion-examples/examples/parquet_embedded_index.rsrename to datafusion-examples/examples/data_io/parquet_embedded_index.rsindex bc0e5a072c..0275a3141f 100644--- a/datafusion-examples/examples/parquet_embedded_index.rs+++ b/datafusion-examples/examples/data_io/parquet_embedded_index.rs@@ -136,6 +136,31 @@ use std::path::{Path, PathBuf};use std::sync::Arc;use tempfile::TempDir;+/// Store a custom index inside a Parquet file and use it to speed up queries+pub async fn parquet_embedded_index() -> Result<()> {+ // 1. Create temp dir and write 3 Parquet files with different category sets+ let tmp = TempDir::new()?;+ let dir = tmp.path();+ write_file_with_index(&dir.join("a.parquet"), &["foo", "bar", "foo"])?;+ write_file_with_index(&dir.join("b.parquet"), &["baz", "qux"])?;+ write_file_with_index(&dir.join("c.parquet"), &["foo", "quux", "quux"])?;++ // 2. Register our custom TableProvider+ let field = Field::new("category", DataType::Utf8, false);+ let schema_ref = Arc::new(Schema::new(vec![field]));+ let provider = Arc::new(DistinctIndexTable::try_new(dir, schema_ref.clone())?);++ let ctx = SessionContext::new();+ ctx.register_table("t", provider)?;++ // 3. Run a query: only files containing 'foo' get scanned. The rest are pruned.+ // based on the distinct index.+ let df = ctx.sql("SELECT * FROM t WHERE category = 'foo'").await?;+ df.show().await?;++ Ok(())+}+/// An index of distinct values for a single column////// In this example the index is a simple set of strings, but in a real@@ -452,28 +477,3 @@ impl TableProvider for DistinctIndexTable {Ok(vec![TableProviderFilterPushDown::Inexact; fs.len()])}}--#[tokio::main]-async fn main() -> Result<()> {- // 1. Create temp dir and write 3 Parquet files with different category sets- let tmp = TempDir::new()?;- let dir = tmp.path();- write_file_with_index(&dir.join("a.parquet"), &["foo", "bar", "foo"])?;- write_file_with_index(&dir.join("b.parquet"), &["baz", "qux"])?;- write_file_with_index(&dir.join("c.parquet"), &["foo", "quux", "quux"])?;-- // 2. Register our custom TableProvider- let field = Field::new("category", DataType::Utf8, false);- let schema_ref = Arc::new(Schema::new(vec![field]));- let provider = Arc::new(DistinctIndexTable::try_new(dir, schema_ref.clone())?);-- let ctx = SessionContext::new();- ctx.register_table("t", provider)?;-- // 3. Run a query: only files containing 'foo' get scanned. The rest are pruned.- // based on the distinct index.- let df = ctx.sql("SELECT * FROM t WHERE category = 'foo'").await?;- df.show().await?;-- Ok(())-}diff --git a/datafusion-examples/examples/parquet_encrypted.rs b/datafusion-examples/examples/data_io/parquet_encrypted.rssimilarity index 97%rename from datafusion-examples/examples/parquet_encrypted.rsrename to datafusion-examples/examples/data_io/parquet_encrypted.rsindex 690d9f2a5f..3f26c436a9 100644--- a/datafusion-examples/examples/parquet_encrypted.rs+++ b/datafusion-examples/examples/data_io/parquet_encrypted.rs@@ -25,8 +25,8 @@ use datafusion::prelude::{ParquetReadOptions, SessionContext};use std::sync::Arc;use tempfile::TempDir;-#[tokio::main]-async fn main() -> datafusion::common::Result<()> {+/// Read and write encrypted Parquet files using DataFusion+pub async fn parquet_encrypted() -> datafusion::common::Result<()> {// The SessionContext is the main high level API for interacting with DataFusionlet ctx = SessionContext::new();diff --git a/datafusion-examples/examples/parquet_encrypted_with_kms.rs b/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rssimilarity index 99%rename from datafusion-examples/examples/parquet_encrypted_with_kms.rsrename to datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rsindex 45bfd18377..3dca002db4 100644--- a/datafusion-examples/examples/parquet_encrypted_with_kms.rs+++ b/datafusion-examples/examples/data_io/parquet_encrypted_with_kms.rs@@ -53,8 +53,7 @@ const ENCRYPTION_FACTORY_ID: &str = "example.mock_kms_encryption";/// which is not a secure way to store encryption keys./// For production use, it is recommended to use a key-management service (KMS) to encrypt/// data encryption keys.-#[tokio::main]-async fn main() -> Result<()> {+pub async fn parquet_encrypted_with_kms() -> Result<()> {let ctx = SessionContext::new();// Register an `EncryptionFactory` implementation to be used for Parquet encryptiondiff --git a/datafusion-examples/examples/parquet_exec_visitor.rs b/datafusion-examples/examples/data_io/parquet_exec_visitor.rssimilarity index 94%rename from datafusion-examples/examples/parquet_exec_visitor.rsrename to datafusion-examples/examples/data_io/parquet_exec_visitor.rsindex 84f92d4f45..e0611fa144 100644--- a/datafusion-examples/examples/parquet_exec_visitor.rs+++ b/datafusion-examples/examples/data_io/parquet_exec_visitor.rs@@ -30,8 +30,7 @@ use datafusion::physical_plan::{use futures::StreamExt;/// Example of collecting metrics after execution by visiting the `ExecutionPlan`-#[tokio::main]-async fn main() {+pub async fn parquet_exec_visitor() -> datafusion::common::Result<()> {let ctx = SessionContext::new();let test_data = datafusion::test_util::parquet_test_data();@@ -51,8 +50,8 @@ async fn main() {).await;- let df = ctx.sql("SELECT * FROM my_table").await.unwrap();- let plan = df.create_physical_plan().await.unwrap();+ let df = ctx.sql("SELECT * FROM my_table").await?;+ let plan = df.create_physical_plan().await?;// Create empty visitorlet mut visitor = ParquetExecVisitor {@@ -63,12 +62,12 @@ async fn main() {// Make sure you execute the plan to collect actual execution statistics.// For example, in this example the `file_scan_config` is known without executing// but the `bytes_scanned` would be None if we did not execute.- let mut batch_stream = execute_stream(plan.clone(), ctx.task_ctx()).unwrap();+ let mut batch_stream = execute_stream(plan.clone(), ctx.task_ctx())?;while let Some(batch) = batch_stream.next().await {println!("Batch rows: {}", batch.unwrap().num_rows());}- visit_execution_plan(plan.as_ref(), &mut visitor).unwrap();+ visit_execution_plan(plan.as_ref(), &mut visitor)?;println!("ParquetExecVisitor bytes_scanned: {:?}",@@ -78,6 +77,8 @@ async fn main() {"ParquetExecVisitor file_groups: {:?}",visitor.file_groups.unwrap());++ Ok(())}/// Define a struct with fields to hold the execution information you want todiff --git a/datafusion-examples/examples/parquet_index.rs b/datafusion-examples/examples/data_io/parquet_index.rssimilarity index 99%rename from datafusion-examples/examples/parquet_index.rsrename to datafusion-examples/examples/data_io/parquet_index.rsindex bc9e2a9226..4cca73b1f9 100644--- a/datafusion-examples/examples/parquet_index.rs+++ b/datafusion-examples/examples/data_io/parquet_index.rs@@ -102,8 +102,7 @@ use url::Url;/// ```////// [`ListingTable`]: datafusion::datasource::listing::ListingTable-#[tokio::main]-async fn main() -> Result<()> {+pub async fn parquet_index() -> Result<()> {// Demo data has three files, each with schema// * file_name (string)// * value (int32)diff --git a/datafusion-examples/examples/query-http-csv.rs b/datafusion-examples/examples/data_io/query_http_csv.rssimilarity index 91%rename from datafusion-examples/examples/query-http-csv.rsrename to datafusion-examples/examples/data_io/query_http_csv.rsindex fa3fd2ac06..be27f8ad21 100644--- a/datafusion-examples/examples/query-http-csv.rs+++ b/datafusion-examples/examples/data_io/query_http_csv.rs@@ -21,10 +21,8 @@ use object_store::http::HttpBuilder;use std::sync::Arc;use url::Url;-/// This example demonstrates executing a simple query against an Arrow data source (CSV) and-/// fetching results-#[tokio::main]-async fn main() -> Result<()> {+/// Configure `object_store` and run a query against files via HTTP+pub async fn query_http_csv() -> Result<()> {// create local execution contextlet ctx = SessionContext::new();diff --git a/datafusion-examples/examples/remote_catalog.rs b/datafusion-examples/examples/data_io/remote_catalog.rssimilarity index 98%rename from datafusion-examples/examples/remote_catalog.rsrename to datafusion-examples/examples/data_io/remote_catalog.rsindex 74575554ec..e4898e19d4 100644--- a/datafusion-examples/examples/remote_catalog.rs+++ b/datafusion-examples/examples/data_io/remote_catalog.rs@@ -46,8 +46,8 @@ use futures::TryStreamExt;use std::any::Any;use std::sync::Arc;-#[tokio::main]-async fn main() -> Result<()> {+/// Interfacing with a remote catalog (e.g. over a network)+pub async fn remote_catalog() -> Result<()> {// As always, we create a session context to interact with DataFusionlet ctx = SessionContext::new();