Skip to content

Apache DataFusion SQL Query Engine

Adds Partitioned CSV test to object store access tests (#18370)

Blake Orth6514ec7b6757
diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rs
index d1592c2147..f89ca9e049 100644
--- a/datafusion/core/tests/datasource/object_store_access.rs
+++ b/datafusion/core/tests/datasource/object_store_access.rs
@@ -28,6 +28,9 @@ use arrow::array::{ArrayRef, Int32Array, RecordBatch};
use async_trait::async_trait;
use bytes::Bytes;
use datafusion::prelude::{CsvReadOptions, ParquetReadOptions, SessionContext};
+use datafusion_catalog_listing::{ListingOptions, ListingTable, ListingTableConfig};
+use datafusion_datasource::ListingTableUrl;
+use datafusion_datasource_csv::CsvFormat;
use futures::stream::BoxStream;
use insta::assert_snapshot;
use object_store::memory::InMemory;
@@ -123,6 +126,163 @@ async fn query_multi_csv_file() {
);
}
+#[tokio::test]
+async fn query_partitioned_csv_file() {
+ let test = Test::new().with_partitioned_csv().await;
+ assert_snapshot!(
+ test.query("select * from csv_table_partitioned").await,
+ @r"
+ ------- Query Output (6 rows) -------
+ +---------+-------+-------+---+----+-----+
+ | d1 | d2 | d3 | a | b | c |
+ +---------+-------+-------+---+----+-----+
+ | 0.00001 | 1e-12 | true | 1 | 10 | 100 |
+ | 0.00003 | 5e-12 | false | 1 | 10 | 100 |
+ | 0.00002 | 2e-12 | true | 2 | 20 | 200 |
+ | 0.00003 | 5e-12 | false | 2 | 20 | 200 |
+ | 0.00003 | 3e-12 | true | 3 | 30 | 300 |
+ | 0.00003 | 5e-12 | false | 3 | 30 | 300 |
+ +---------+-------+-------+---+----+-----+
+ ------- Object Store Request Summary -------
+ RequestCountingObjectStore()
+ Total Requests: 13
+ - LIST (with delimiter) prefix=data
+ - LIST (with delimiter) prefix=data/a=1
+ - LIST (with delimiter) prefix=data/a=2
+ - LIST (with delimiter) prefix=data/a=3
+ - LIST (with delimiter) prefix=data/a=1/b=10
+ - LIST (with delimiter) prefix=data/a=2/b=20
+ - LIST (with delimiter) prefix=data/a=3/b=30
+ - LIST (with delimiter) prefix=data/a=1/b=10/c=100
+ - LIST (with delimiter) prefix=data/a=2/b=20/c=200
+ - LIST (with delimiter) prefix=data/a=3/b=30/c=300
+ - GET (opts) path=data/a=1/b=10/c=100/file_1.csv
+ - GET (opts) path=data/a=2/b=20/c=200/file_2.csv
+ - GET (opts) path=data/a=3/b=30/c=300/file_3.csv
+ "
+ );
+
+ assert_snapshot!(
+ test.query("select * from csv_table_partitioned WHERE a=2").await,
+ @r"
+ ------- Query Output (2 rows) -------
+ +---------+-------+-------+---+----+-----+
+ | d1 | d2 | d3 | a | b | c |
+ +---------+-------+-------+---+----+-----+
+ | 0.00002 | 2e-12 | true | 2 | 20 | 200 |
+ | 0.00003 | 5e-12 | false | 2 | 20 | 200 |
+ +---------+-------+-------+---+----+-----+
+ ------- Object Store Request Summary -------
+ RequestCountingObjectStore()
+ Total Requests: 4
+ - LIST (with delimiter) prefix=data/a=2
+ - LIST (with delimiter) prefix=data/a=2/b=20
+ - LIST (with delimiter) prefix=data/a=2/b=20/c=200
+ - GET (opts) path=data/a=2/b=20/c=200/file_2.csv
+ "
+ );
+
+ assert_snapshot!(
+ test.query("select * from csv_table_partitioned WHERE b=20").await,
+ @r"
+ ------- Query Output (2 rows) -------
+ +---------+-------+-------+---+----+-----+
+ | d1 | d2 | d3 | a | b | c |
+ +---------+-------+-------+---+----+-----+
+ | 0.00002 | 2e-12 | true | 2 | 20 | 200 |
+ | 0.00003 | 5e-12 | false | 2 | 20 | 200 |
+ +---------+-------+-------+---+----+-----+
+ ------- Object Store Request Summary -------
+ RequestCountingObjectStore()
+ Total Requests: 11
+ - LIST (with delimiter) prefix=data
+ - LIST (with delimiter) prefix=data/a=1
+ - LIST (with delimiter) prefix=data/a=2
+ - LIST (with delimiter) prefix=data/a=3
+ - LIST (with delimiter) prefix=data/a=1/b=10
+ - LIST (with delimiter) prefix=data/a=2/b=20
+ - LIST (with delimiter) prefix=data/a=3/b=30
+ - LIST (with delimiter) prefix=data/a=1/b=10/c=100
+ - LIST (with delimiter) prefix=data/a=2/b=20/c=200
+ - LIST (with delimiter) prefix=data/a=3/b=30/c=300
+ - GET (opts) path=data/a=2/b=20/c=200/file_2.csv
+ "
+ );
+
+ assert_snapshot!(
+ test.query("select * from csv_table_partitioned WHERE c=200").await,
+ @r"
+ ------- Query Output (2 rows) -------
+ +---------+-------+-------+---+----+-----+
+ | d1 | d2 | d3 | a | b | c |
+ +---------+-------+-------+---+----+-----+
+ | 0.00002 | 2e-12 | true | 2 | 20 | 200 |
+ | 0.00003 | 5e-12 | false | 2 | 20 | 200 |
+ +---------+-------+-------+---+----+-----+
+ ------- Object Store Request Summary -------
+ RequestCountingObjectStore()
+ Total Requests: 11
+ - LIST (with delimiter) prefix=data
+ - LIST (with delimiter) prefix=data/a=1
+ - LIST (with delimiter) prefix=data/a=2
+ - LIST (with delimiter) prefix=data/a=3
+ - LIST (with delimiter) prefix=data/a=1/b=10
+ - LIST (with delimiter) prefix=data/a=2/b=20
+ - LIST (with delimiter) prefix=data/a=3/b=30
+ - LIST (with delimiter) prefix=data/a=1/b=10/c=100
+ - LIST (with delimiter) prefix=data/a=2/b=20/c=200
+ - LIST (with delimiter) prefix=data/a=3/b=30/c=300
+ - GET (opts) path=data/a=2/b=20/c=200/file_2.csv
+ "
+ );
+
+ assert_snapshot!(
+ test.query("select * from csv_table_partitioned WHERE a=2 AND b=20").await,
+ @r"
+ ------- Query Output (2 rows) -------
+ +---------+-------+-------+---+----+-----+
+ | d1 | d2 | d3 | a | b | c |
+ +---------+-------+-------+---+----+-----+
+ | 0.00002 | 2e-12 | true | 2 | 20 | 200 |
+ | 0.00003 | 5e-12 | false | 2 | 20 | 200 |
+ +---------+-------+-------+---+----+-----+
+ ------- Object Store Request Summary -------
+ RequestCountingObjectStore()
+ Total Requests: 3
+ - LIST (with delimiter) prefix=data/a=2/b=20
+ - LIST (with delimiter) prefix=data/a=2/b=20/c=200
+ - GET (opts) path=data/a=2/b=20/c=200/file_2.csv
+ "
+ );
+
+ assert_snapshot!(
+ test.query("select * from csv_table_partitioned WHERE a<2 AND b=10 AND c=100").await,
+ @r"
+ ------- Query Output (2 rows) -------
+ +---------+-------+-------+---+----+-----+
+ | d1 | d2 | d3 | a | b | c |
+ +---------+-------+-------+---+----+-----+
+ | 0.00001 | 1e-12 | true | 1 | 10 | 100 |
+ | 0.00003 | 5e-12 | false | 1 | 10 | 100 |
+ +---------+-------+-------+---+----+-----+
+ ------- Object Store Request Summary -------
+ RequestCountingObjectStore()
+ Total Requests: 11
+ - LIST (with delimiter) prefix=data
+ - LIST (with delimiter) prefix=data/a=1
+ - LIST (with delimiter) prefix=data/a=2
+ - LIST (with delimiter) prefix=data/a=3
+ - LIST (with delimiter) prefix=data/a=1/b=10
+ - LIST (with delimiter) prefix=data/a=2/b=20
+ - LIST (with delimiter) prefix=data/a=3/b=30
+ - LIST (with delimiter) prefix=data/a=1/b=10/c=100
+ - LIST (with delimiter) prefix=data/a=2/b=20/c=200
+ - LIST (with delimiter) prefix=data/a=3/b=30/c=300
+ - GET (opts) path=data/a=1/b=10/c=100/file_1.csv
+ "
+ );
+}
+
#[tokio::test]
async fn create_single_parquet_file_default() {
// The default metadata size hint is 512KB
@@ -363,7 +523,7 @@ impl Test {
self
}
- /// Register a CSV file at the given path relative to the [`datafusion_test_data`] directory
+ /// Register a CSV file at the given path
async fn register_csv(self, table_name: &str, path: &str) -> Self {
let mut options = CsvReadOptions::new();
options.has_header = true;
@@ -375,8 +535,30 @@ impl Test {
self
}
- /// Register a Parquet file at the given path relative to the
- /// [`datafusion_test_data`] directory
+ /// Register a partitioned CSV table at the given path
+ async fn register_partitioned_csv(self, table_name: &str, path: &str) -> Self {
+ let file_format = Arc::new(CsvFormat::default().with_has_header(true));
+ let options = ListingOptions::new(file_format);
+
+ let url = format!("mem://{path}").parse().unwrap();
+ let table_url = ListingTableUrl::try_new(url, None).unwrap();
+
+ let session_state = self.session_context.state();
+ let mut config = ListingTableConfig::new(table_url).with_listing_options(options);
+ config = config
+ .infer_partitions_from_path(&session_state)
+ .await
+ .unwrap();
+ config = config.infer_schema(&session_state).await.unwrap();
+
+ let table = Arc::new(ListingTable::try_new(config).unwrap());
+ self.session_context
+ .register_table(table_name, table)
+ .unwrap();
+ self
+ }
+
+ /// Register a Parquet file at the given path
async fn register_parquet(self, table_name: &str, path: &str) -> Self {
let path = format!("mem://{path}");
let mut options: ParquetReadOptions<'_> = ParquetReadOptions::new();
@@ -425,6 +607,29 @@ impl Test {
self.register_csv("csv_table", "/data/").await
}
+ /// Register three CSV files in a partitioned directory structure, called
+ /// `csv_table_partitioned`
+ async fn with_partitioned_csv(mut self) -> Test {
+ for i in 1..4 {
+ // upload CSV data to object store
+ let csv_data1 = format!(
+ r#"d1,d2,d3
+0.0000{i},{i}e-12,true
+0.00003,5e-12,false
+"#
+ );
+ self = self
+ .with_bytes(
+ &format!("/data/a={i}/b={}/c={}/file_{i}.csv", i * 10, i * 100,),
+ csv_data1,
+ )
+ .await;
+ }
+ // register table
+ self.register_partitioned_csv("csv_table_partitioned", "/data/")
+ .await
+ }
+
/// Add a single parquet file that has two columns and two row groups named `parquet_table`
///
/// Column "a": Int32 with values 0-100] in row group 1