Apache DataFusion SQL Query Engine
Normalize partitioned and flat object listing (#18146)
Blake Orth28755b1d7eb5
diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rsindex 82cc368679..089457648d 100644--- a/datafusion/catalog-listing/src/helpers.rs+++ b/datafusion/catalog-listing/src/helpers.rs@@ -25,12 +25,11 @@ use datafusion_common::internal_err;use datafusion_common::{HashMap, Result, ScalarValue};use datafusion_datasource::ListingTableUrl;use datafusion_datasource::PartitionedFile;-use datafusion_expr::{BinaryExpr, Operator};+use datafusion_expr::{lit, utils, BinaryExpr, Operator};use arrow::{- array::{Array, ArrayRef, AsArray, StringBuilder},- compute::{and, cast, prep_null_mask_filter},- datatypes::{DataType, Field, Fields, Schema},+ array::AsArray,+ datatypes::{DataType, Field},record_batch::RecordBatch,};use datafusion_expr::execution_props::ExecutionProps;@@ -39,7 +38,7 @@ use futures::{stream::BoxStream, StreamExt, TryStreamExt};use log::{debug, trace};use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};-use datafusion_common::{Column, DFSchema, DataFusionError};+use datafusion_common::{Column, DFSchema};use datafusion_expr::{Expr, Volatility};use datafusion_physical_expr::create_physical_expr;use object_store::path::Path;@@ -239,105 +238,6 @@ pub async fn list_partitions(Ok(out)}-async fn prune_partitions(- table_path: &ListingTableUrl,- partitions: Vec<Partition>,- filters: &[Expr],- partition_cols: &[(String, DataType)],-) -> Result<Vec<Partition>> {- if filters.is_empty() {- // prune partitions which don't contain the partition columns- return Ok(partitions- .into_iter()- .filter(|p| {- let cols = partition_cols.iter().map(|x| x.0.as_str());- !parse_partitions_for_path(table_path, &p.path, cols)- .unwrap_or_default()- .is_empty()- })- .collect());- }-- let mut builders: Vec<_> = (0..partition_cols.len())- .map(|_| StringBuilder::with_capacity(partitions.len(), partitions.len() * 10))- .collect();-- for partition in &partitions {- let cols = partition_cols.iter().map(|x| x.0.as_str());- let parsed = parse_partitions_for_path(table_path, &partition.path, cols)- .unwrap_or_default();-- let mut builders = builders.iter_mut();- for (p, b) in parsed.iter().zip(&mut builders) {- b.append_value(p);- }- builders.for_each(|b| b.append_null());- }-- let arrays = partition_cols- .iter()- .zip(builders)- .map(|((_, d), mut builder)| {- let array = builder.finish();- cast(&array, d)- })- .collect::<Result<_, _>>()?;-- let fields: Fields = partition_cols- .iter()- .map(|(n, d)| Field::new(n, d.clone(), true))- .collect();- let schema = Arc::new(Schema::new(fields));-- let df_schema = DFSchema::from_unqualified_fields(- partition_cols- .iter()- .map(|(n, d)| Field::new(n, d.clone(), true))- .collect(),- Default::default(),- )?;-- let batch = RecordBatch::try_new(schema, arrays)?;-- // TODO: Plumb this down- let props = ExecutionProps::new();-- // Applies `filter` to `batch` returning `None` on error- let do_filter = |filter| -> Result<ArrayRef> {- let expr = create_physical_expr(filter, &df_schema, &props)?;- expr.evaluate(&batch)?.into_array(partitions.len())- };-- //.Compute the conjunction of the filters- let mask = filters- .iter()- .map(|f| do_filter(f).map(|a| a.as_boolean().clone()))- .reduce(|a, b| Ok(and(&a?, &b?)?));-- let mask = match mask {- Some(Ok(mask)) => mask,- Some(Err(err)) => return Err(err),- None => return Ok(partitions),- };-- // Don't retain partitions that evaluated to null- let prepared = match mask.null_count() {- 0 => mask,- _ => prep_null_mask_filter(&mask),- };-- // Sanity check- assert_eq!(prepared.len(), partitions.len());-- let filtered = partitions- .into_iter()- .zip(prepared.values())- .filter_map(|(p, f)| f.then_some(p))- .collect();-- Ok(filtered)-}-#[derive(Debug)]enum PartitionValue {Single(String),@@ -412,6 +312,62 @@ pub fn evaluate_partition_prefix<'a>(}}+fn filter_partitions(+ pf: PartitionedFile,+ filters: &[Expr],+ df_schema: &DFSchema,+) -> Result<Option<PartitionedFile>> {+ if pf.partition_values.is_empty() && !filters.is_empty() {+ return Ok(None);+ } else if filters.is_empty() {+ return Ok(Some(pf));+ }++ let arrays = pf+ .partition_values+ .iter()+ .map(|v| v.to_array())+ .collect::<Result<_, _>>()?;++ let batch = RecordBatch::try_new(Arc::clone(df_schema.inner()), arrays)?;++ let filter = utils::conjunction(filters.iter().cloned()).unwrap_or_else(|| lit(true));+ let props = ExecutionProps::new();+ let expr = create_physical_expr(&filter, df_schema, &props)?;++ // Since we're only operating on a single file, our batch and resulting "array" holds only one+ // value indicating if the input file matches the provided filters+ let matches = expr.evaluate(&batch)?.into_array(1)?;+ if matches.as_boolean().value(0) {+ return Ok(Some(pf));+ }++ Ok(None)+}++fn try_into_partitioned_file(+ object_meta: ObjectMeta,+ partition_cols: &[(String, DataType)],+ table_path: &ListingTableUrl,+) -> Result<PartitionedFile> {+ let cols = partition_cols.iter().map(|(name, _)| name.as_str());+ let parsed = parse_partitions_for_path(table_path, &object_meta.location, cols);++ let partition_values = parsed+ .into_iter()+ .flatten()+ .zip(partition_cols)+ .map(|(parsed, (_, datatype))| {+ ScalarValue::try_from_string(parsed.to_string(), datatype)+ })+ .collect::<Result<Vec<_>>>()?;++ let mut pf: PartitionedFile = object_meta.into();+ pf.partition_values = partition_values;++ Ok(pf)+}+/// Discover the partitions on the given path and prune out files/// that belong to irrelevant partitions using `filters` expressions./// `filters` should only contain expressions that can be evaluated@@ -424,7 +380,11 @@ pub async fn pruned_partition_list<'a>(file_extension: &'a str,partition_cols: &'a [(String, DataType)],) -> Result<BoxStream<'a, Result<PartitionedFile>>> {- // if no partition col => simply list all the files+ let objects = table_path+ .list_all_files(ctx, store, file_extension)+ .await?+ .try_filter(|object_meta| futures::future::ready(object_meta.size > 0));+if partition_cols.is_empty() {if !filters.is_empty() {return internal_err!(@@ -432,72 +392,29 @@ pub async fn pruned_partition_list<'a>(table_path);}- return Ok(Box::pin(- table_path- .list_all_files(ctx, store, file_extension)- .await?- .try_filter(|object_meta| futures::future::ready(object_meta.size > 0))- .map_ok(|object_meta| object_meta.into()),- ));- }-- let partition_prefix = evaluate_partition_prefix(partition_cols, filters);-- let partitions =- list_partitions(store, table_path, partition_cols.len(), partition_prefix)- .await?;- debug!("Listed {} partitions", partitions.len());- let pruned =- prune_partitions(table_path, partitions, filters, partition_cols).await?;-- debug!("Pruning yielded {} partitions", pruned.len());-- let stream = futures::stream::iter(pruned)- .map(move |partition: Partition| async move {- let cols = partition_cols.iter().map(|x| x.0.as_str());- let parsed = parse_partitions_for_path(table_path, &partition.path, cols);+ // if no partition col => simply list all the files+ Ok(objects.map_ok(|object_meta| object_meta.into()).boxed())+ } else {+ let df_schema = DFSchema::from_unqualified_fields(+ partition_cols+ .iter()+ .map(|(n, d)| Field::new(n, d.clone(), true))+ .collect(),+ Default::default(),+ )?;- let partition_values = parsed- .into_iter()- .flatten()- .zip(partition_cols)- .map(|(parsed, (_, datatype))| {- ScalarValue::try_from_string(parsed.to_string(), datatype)- })- .collect::<Result<Vec<_>>>()?;-- let files = match partition.files {- Some(files) => files,- None => {- trace!("Recursively listing partition {}", partition.path);- store.list(Some(&partition.path)).try_collect().await?- }- };- let files = files.into_iter().filter(move |o| {- let extension_match = o.location.as_ref().ends_with(file_extension);- // here need to scan subdirectories(`listing_table_ignore_subdirectory` = false)- let glob_match = table_path.contains(&o.location, false);- extension_match && glob_match- });-- let stream = futures::stream::iter(files.map(move |object_meta| {- Ok(PartitionedFile {- object_meta,- partition_values: partition_values.clone(),- range: None,- statistics: None,- extensions: None,- metadata_size_hint: None,- })- }));-- Ok::<_, DataFusionError>(stream)- })- .buffer_unordered(CONCURRENCY_LIMIT)- .try_flatten()- .boxed();- Ok(stream)+ Ok(objects+ .map_ok(|object_meta| {+ try_into_partitioned_file(object_meta, partition_cols, table_path)+ })+ .try_filter_map(move |pf| {+ futures::future::ready(+ pf.and_then(|pf| filter_partitions(pf, filters, &df_schema)),+ )+ })+ .boxed())+ }}/// Extract the partition values for the given `file_path` (in the given `table_path`)@@ -541,22 +458,11 @@ pub fn describe_partition(partition: &Partition) -> (&str, usize, Vec<&str>) {#[cfg(test)]mod tests {- use async_trait::async_trait;- use datafusion_common::config::TableOptions;use datafusion_datasource::file_groups::FileGroup;- use datafusion_execution::config::SessionConfig;- use datafusion_execution::runtime_env::RuntimeEnv;- use futures::FutureExt;- use object_store::memory::InMemory;- use std::any::Any;use std::ops::Not;use super::*;- use datafusion_expr::{- case, col, lit, AggregateUDF, Expr, LogicalPlan, ScalarUDF, WindowUDF,- };- use datafusion_physical_expr_common::physical_expr::PhysicalExpr;- use datafusion_physical_plan::ExecutionPlan;+ use datafusion_expr::{case, col, lit, Expr};#[test]fn test_split_files() {@@ -599,209 +505,6 @@ mod tests {assert_eq!(0, chunks.len());}- #[tokio::test]- async fn test_pruned_partition_list_empty() {- let (store, state) = make_test_store_and_state(&[- ("tablepath/mypartition=val1/notparquetfile", 100),- ("tablepath/mypartition=val1/ignoresemptyfile.parquet", 0),- ("tablepath/file.parquet", 100),- ("tablepath/notapartition/file.parquet", 100),- ("tablepath/notmypartition=val1/file.parquet", 100),- ]);- let filter = Expr::eq(col("mypartition"), lit("val1"));- let pruned = pruned_partition_list(- state.as_ref(),- store.as_ref(),- &ListingTableUrl::parse("file:///tablepath/").unwrap(),- &[filter],- ".parquet",- &[(String::from("mypartition"), DataType::Utf8)],- )- .await- .expect("partition pruning failed")- .collect::<Vec<_>>()- .await;-- assert_eq!(pruned.len(), 0);- }-- #[tokio::test]- async fn test_pruned_partition_list() {- let (store, state) = make_test_store_and_state(&[- ("tablepath/mypartition=val1/file.parquet", 100),- ("tablepath/mypartition=val2/file.parquet", 100),- ("tablepath/mypartition=val1/ignoresemptyfile.parquet", 0),- ("tablepath/mypartition=val1/other=val3/file.parquet", 100),- ("tablepath/notapartition/file.parquet", 100),- ("tablepath/notmypartition=val1/file.parquet", 100),- ]);- let filter = Expr::eq(col("mypartition"), lit("val1"));- let pruned = pruned_partition_list(- state.as_ref(),- store.as_ref(),- &ListingTableUrl::parse("file:///tablepath/").unwrap(),- &[filter],- ".parquet",- &[(String::from("mypartition"), DataType::Utf8)],- )- .await- .expect("partition pruning failed")- .try_collect::<Vec<_>>()- .await- .unwrap();-- assert_eq!(pruned.len(), 2);- let f1 = &pruned[0];- assert_eq!(- f1.object_meta.location.as_ref(),- "tablepath/mypartition=val1/file.parquet"- );- assert_eq!(&f1.partition_values, &[ScalarValue::from("val1")]);- let f2 = &pruned[1];- assert_eq!(- f2.object_meta.location.as_ref(),- "tablepath/mypartition=val1/other=val3/file.parquet"- );- assert_eq!(f2.partition_values, &[ScalarValue::from("val1"),]);- }-- #[tokio::test]- async fn test_pruned_partition_list_multi() {- let (store, state) = make_test_store_and_state(&[- ("tablepath/part1=p1v1/file.parquet", 100),- ("tablepath/part1=p1v2/part2=p2v1/file1.parquet", 100),- ("tablepath/part1=p1v2/part2=p2v1/file2.parquet", 100),- ("tablepath/part1=p1v3/part2=p2v1/file2.parquet", 100),- ("tablepath/part1=p1v2/part2=p2v2/file2.parquet", 100),- ]);- let filter1 = Expr::eq(col("part1"), lit("p1v2"));- let filter2 = Expr::eq(col("part2"), lit("p2v1"));- let pruned = pruned_partition_list(- state.as_ref(),- store.as_ref(),- &ListingTableUrl::parse("file:///tablepath/").unwrap(),- &[filter1, filter2],- ".parquet",- &[- (String::from("part1"), DataType::Utf8),- (String::from("part2"), DataType::Utf8),- ],- )- .await- .expect("partition pruning failed")- .try_collect::<Vec<_>>()- .await- .unwrap();-- assert_eq!(pruned.len(), 2);- let f1 = &pruned[0];- assert_eq!(- f1.object_meta.location.as_ref(),- "tablepath/part1=p1v2/part2=p2v1/file1.parquet"- );- assert_eq!(- &f1.partition_values,- &[ScalarValue::from("p1v2"), ScalarValue::from("p2v1"),]- );- let f2 = &pruned[1];- assert_eq!(- f2.object_meta.location.as_ref(),- "tablepath/part1=p1v2/part2=p2v1/file2.parquet"- );- assert_eq!(- &f2.partition_values,- &[ScalarValue::from("p1v2"), ScalarValue::from("p2v1")]- );- }-- #[tokio::test]- async fn test_list_partition() {- let (store, _) = make_test_store_and_state(&[- ("tablepath/part1=p1v1/file.parquet", 100),- ("tablepath/part1=p1v2/part2=p2v1/file1.parquet", 100),- ("tablepath/part1=p1v2/part2=p2v1/file2.parquet", 100),- ("tablepath/part1=p1v3/part2=p2v1/file3.parquet", 100),- ("tablepath/part1=p1v2/part2=p2v2/file4.parquet", 100),- ("tablepath/part1=p1v2/part2=p2v2/empty.parquet", 0),- ]);-- let partitions = list_partitions(- store.as_ref(),- &ListingTableUrl::parse("file:///tablepath/").unwrap(),- 0,- None,- )- .await- .expect("listing partitions failed");-- assert_eq!(- &partitions- .iter()- .map(describe_partition)- .collect::<Vec<_>>(),- &vec![- ("tablepath", 0, vec![]),- ("tablepath/part1=p1v1", 1, vec![]),- ("tablepath/part1=p1v2", 1, vec![]),- ("tablepath/part1=p1v3", 1, vec![]),- ]- );-- let partitions = list_partitions(- store.as_ref(),- &ListingTableUrl::parse("file:///tablepath/").unwrap(),- 1,- None,- )- .await- .expect("listing partitions failed");-- assert_eq!(- &partitions- .iter()- .map(describe_partition)- .collect::<Vec<_>>(),- &vec![- ("tablepath", 0, vec![]),- ("tablepath/part1=p1v1", 1, vec!["file.parquet"]),- ("tablepath/part1=p1v2", 1, vec![]),- ("tablepath/part1=p1v2/part2=p2v1", 2, vec![]),- ("tablepath/part1=p1v2/part2=p2v2", 2, vec![]),- ("tablepath/part1=p1v3", 1, vec![]),- ("tablepath/part1=p1v3/part2=p2v1", 2, vec![]),- ]- );-- let partitions = list_partitions(- store.as_ref(),- &ListingTableUrl::parse("file:///tablepath/").unwrap(),- 2,- None,- )- .await- .expect("listing partitions failed");-- assert_eq!(- &partitions- .iter()- .map(describe_partition)- .collect::<Vec<_>>(),- &vec![- ("tablepath", 0, vec![]),- ("tablepath/part1=p1v1", 1, vec!["file.parquet"]),- ("tablepath/part1=p1v2", 1, vec![]),- ("tablepath/part1=p1v3", 1, vec![]),- (- "tablepath/part1=p1v2/part2=p2v1",- 2,- vec!["file1.parquet", "file2.parquet"]- ),- ("tablepath/part1=p1v2/part2=p2v2", 2, vec!["file4.parquet"]),- ("tablepath/part1=p1v3/part2=p2v1", 2, vec!["file3.parquet"]),- ]- );- }-#[test]fn test_parse_partitions_for_path() {assert_eq!(@@ -1016,86 +719,4 @@ mod tests {Some(Path::from("a=1970-01-05")),);}-- pub fn make_test_store_and_state(- files: &[(&str, u64)],- ) -> (Arc<InMemory>, Arc<dyn Session>) {- let memory = InMemory::new();-- for (name, size) in files {- memory- .put(&Path::from(*name), vec![0; *size as usize].into())- .now_or_never()- .unwrap()- .unwrap();- }-- (Arc::new(memory), Arc::new(MockSession {}))- }-- struct MockSession {}-- #[async_trait]- impl Session for MockSession {- fn session_id(&self) -> &str {- unimplemented!()- }-- fn config(&self) -> &SessionConfig {- unimplemented!()- }-- async fn create_physical_plan(- &self,- _logical_plan: &LogicalPlan,- ) -> Result<Arc<dyn ExecutionPlan>> {- unimplemented!()- }-- fn create_physical_expr(- &self,- _expr: Expr,- _df_schema: &DFSchema,- ) -> Result<Arc<dyn PhysicalExpr>> {- unimplemented!()- }-- fn scalar_functions(&self) -> &std::collections::HashMap<String, Arc<ScalarUDF>> {- unimplemented!()- }-- fn aggregate_functions(- &self,- ) -> &std::collections::HashMap<String, Arc<AggregateUDF>> {- unimplemented!()- }-- fn window_functions(&self) -> &std::collections::HashMap<String, Arc<WindowUDF>> {- unimplemented!()- }-- fn runtime_env(&self) -> &Arc<RuntimeEnv> {- unimplemented!()- }-- fn execution_props(&self) -> &ExecutionProps {- unimplemented!()- }-- fn as_any(&self) -> &dyn Any {- unimplemented!()- }-- fn table_options(&self) -> &TableOptions {- unimplemented!()- }-- fn table_options_mut(&mut self) -> &mut TableOptions {- unimplemented!()- }-- fn task_ctx(&self) -> Arc<datafusion_execution::TaskContext> {- unimplemented!()- }- }}diff --git a/datafusion/core/tests/catalog_listing/mod.rs b/datafusion/core/tests/catalog_listing/mod.rsnew file mode 100644index 0000000000..cb6cac4fb0--- /dev/null+++ b/datafusion/core/tests/catalog_listing/mod.rs@@ -0,0 +1,18 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++mod pruned_partition_list;diff --git a/datafusion/core/tests/catalog_listing/pruned_partition_list.rs b/datafusion/core/tests/catalog_listing/pruned_partition_list.rsnew file mode 100644index 0000000000..3cdaa3bb9b--- /dev/null+++ b/datafusion/core/tests/catalog_listing/pruned_partition_list.rs@@ -0,0 +1,251 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::sync::Arc;++use arrow_schema::DataType;+use futures::{FutureExt, StreamExt as _, TryStreamExt as _};+use object_store::{memory::InMemory, path::Path, ObjectStore as _};++use datafusion::execution::SessionStateBuilder;+use datafusion_catalog_listing::helpers::{+ describe_partition, list_partitions, pruned_partition_list,+};+use datafusion_common::ScalarValue;+use datafusion_datasource::ListingTableUrl;+use datafusion_expr::{col, lit, Expr};+use datafusion_session::Session;++#[tokio::test]+async fn test_pruned_partition_list_empty() {+ let (store, state) = make_test_store_and_state(&[+ ("tablepath/mypartition=val1/notparquetfile", 100),+ ("tablepath/mypartition=val1/ignoresemptyfile.parquet", 0),+ ("tablepath/file.parquet", 100),+ ("tablepath/notapartition/file.parquet", 100),+ ("tablepath/notmypartition=val1/file.parquet", 100),+ ]);+ let filter = Expr::eq(col("mypartition"), lit("val1"));+ let pruned = pruned_partition_list(+ state.as_ref(),+ store.as_ref(),+ &ListingTableUrl::parse("file:///tablepath/").unwrap(),+ &[filter],+ ".parquet",+ &[(String::from("mypartition"), DataType::Utf8)],+ )+ .await+ .expect("partition pruning failed")+ .collect::<Vec<_>>()+ .await;++ assert_eq!(pruned.len(), 0);+}++#[tokio::test]+async fn test_pruned_partition_list() {+ let (store, state) = make_test_store_and_state(&[+ ("tablepath/mypartition=val1/file.parquet", 100),+ ("tablepath/mypartition=val2/file.parquet", 100),+ ("tablepath/mypartition=val1/ignoresemptyfile.parquet", 0),+ ("tablepath/mypartition=val1/other=val3/file.parquet", 100),+ ("tablepath/notapartition/file.parquet", 100),+ ("tablepath/notmypartition=val1/file.parquet", 100),+ ]);+ let filter = Expr::eq(col("mypartition"), lit("val1"));+ let pruned = pruned_partition_list(+ state.as_ref(),+ store.as_ref(),+ &ListingTableUrl::parse("file:///tablepath/").unwrap(),+ &[filter],+ ".parquet",+ &[(String::from("mypartition"), DataType::Utf8)],+ )+ .await+ .expect("partition pruning failed")+ .try_collect::<Vec<_>>()+ .await+ .unwrap();++ assert_eq!(pruned.len(), 2);+ let f1 = &pruned[0];+ assert_eq!(+ f1.object_meta.location.as_ref(),+ "tablepath/mypartition=val1/file.parquet"+ );+ assert_eq!(&f1.partition_values, &[ScalarValue::from("val1")]);+ let f2 = &pruned[1];+ assert_eq!(+ f2.object_meta.location.as_ref(),+ "tablepath/mypartition=val1/other=val3/file.parquet"+ );+ assert_eq!(f2.partition_values, &[ScalarValue::from("val1"),]);+}++#[tokio::test]+async fn test_pruned_partition_list_multi() {+ let (store, state) = make_test_store_and_state(&[+ ("tablepath/part1=p1v1/file.parquet", 100),+ ("tablepath/part1=p1v2/part2=p2v1/file1.parquet", 100),+ ("tablepath/part1=p1v2/part2=p2v1/file2.parquet", 100),+ ("tablepath/part1=p1v3/part2=p2v1/file2.parquet", 100),+ ("tablepath/part1=p1v2/part2=p2v2/file2.parquet", 100),+ ]);+ let filter1 = Expr::eq(col("part1"), lit("p1v2"));+ let filter2 = Expr::eq(col("part2"), lit("p2v1"));+ let pruned = pruned_partition_list(+ state.as_ref(),+ store.as_ref(),+ &ListingTableUrl::parse("file:///tablepath/").unwrap(),+ &[filter1, filter2],+ ".parquet",+ &[+ (String::from("part1"), DataType::Utf8),+ (String::from("part2"), DataType::Utf8),+ ],+ )+ .await+ .expect("partition pruning failed")+ .try_collect::<Vec<_>>()+ .await+ .unwrap();++ assert_eq!(pruned.len(), 2);+ let f1 = &pruned[0];+ assert_eq!(+ f1.object_meta.location.as_ref(),+ "tablepath/part1=p1v2/part2=p2v1/file1.parquet"+ );+ assert_eq!(+ &f1.partition_values,+ &[ScalarValue::from("p1v2"), ScalarValue::from("p2v1"),]+ );+ let f2 = &pruned[1];+ assert_eq!(+ f2.object_meta.location.as_ref(),+ "tablepath/part1=p1v2/part2=p2v1/file2.parquet"+ );+ assert_eq!(+ &f2.partition_values,+ &[ScalarValue::from("p1v2"), ScalarValue::from("p2v1")]+ );+}++#[tokio::test]+async fn test_list_partition() {+ let (store, _) = make_test_store_and_state(&[+ ("tablepath/part1=p1v1/file.parquet", 100),+ ("tablepath/part1=p1v2/part2=p2v1/file1.parquet", 100),+ ("tablepath/part1=p1v2/part2=p2v1/file2.parquet", 100),+ ("tablepath/part1=p1v3/part2=p2v1/file3.parquet", 100),+ ("tablepath/part1=p1v2/part2=p2v2/file4.parquet", 100),+ ("tablepath/part1=p1v2/part2=p2v2/empty.parquet", 0),+ ]);++ let partitions = list_partitions(+ store.as_ref(),+ &ListingTableUrl::parse("file:///tablepath/").unwrap(),+ 0,+ None,+ )+ .await+ .expect("listing partitions failed");++ assert_eq!(+ &partitions+ .iter()+ .map(describe_partition)+ .collect::<Vec<_>>(),+ &vec![+ ("tablepath", 0, vec![]),+ ("tablepath/part1=p1v1", 1, vec![]),+ ("tablepath/part1=p1v2", 1, vec![]),+ ("tablepath/part1=p1v3", 1, vec![]),+ ]+ );++ let partitions = list_partitions(+ store.as_ref(),+ &ListingTableUrl::parse("file:///tablepath/").unwrap(),+ 1,+ None,+ )+ .await+ .expect("listing partitions failed");++ assert_eq!(+ &partitions+ .iter()+ .map(describe_partition)+ .collect::<Vec<_>>(),+ &vec![+ ("tablepath", 0, vec![]),+ ("tablepath/part1=p1v1", 1, vec!["file.parquet"]),+ ("tablepath/part1=p1v2", 1, vec![]),+ ("tablepath/part1=p1v2/part2=p2v1", 2, vec![]),+ ("tablepath/part1=p1v2/part2=p2v2", 2, vec![]),+ ("tablepath/part1=p1v3", 1, vec![]),+ ("tablepath/part1=p1v3/part2=p2v1", 2, vec![]),+ ]+ );++ let partitions = list_partitions(+ store.as_ref(),+ &ListingTableUrl::parse("file:///tablepath/").unwrap(),+ 2,+ None,+ )+ .await+ .expect("listing partitions failed");++ assert_eq!(+ &partitions+ .iter()+ .map(describe_partition)+ .collect::<Vec<_>>(),+ &vec![+ ("tablepath", 0, vec![]),+ ("tablepath/part1=p1v1", 1, vec!["file.parquet"]),+ ("tablepath/part1=p1v2", 1, vec![]),+ ("tablepath/part1=p1v3", 1, vec![]),+ (+ "tablepath/part1=p1v2/part2=p2v1",+ 2,+ vec!["file1.parquet", "file2.parquet"]+ ),+ ("tablepath/part1=p1v2/part2=p2v2", 2, vec!["file4.parquet"]),+ ("tablepath/part1=p1v3/part2=p2v1", 2, vec!["file3.parquet"]),+ ]+ );+}++pub fn make_test_store_and_state(+ files: &[(&str, u64)],+) -> (Arc<InMemory>, Arc<dyn Session>) {+ let memory = InMemory::new();++ for (name, size) in files {+ memory+ .put(&Path::from(*name), vec![0; *size as usize].into())+ .now_or_never()+ .unwrap()+ .unwrap();+ }++ let state = SessionStateBuilder::new().build();+ (Arc::new(memory), Arc::new(state))+}diff --git a/datafusion/core/tests/core_integration.rs b/datafusion/core/tests/core_integration.rsindex edcf039e4e..cc4dfcf720 100644--- a/datafusion/core/tests/core_integration.rs+++ b/datafusion/core/tests/core_integration.rs@@ -57,6 +57,9 @@ mod serde;/// Run all tests that are found in the `catalog` directorymod catalog;+/// Run all tests that are found in the `catalog_listing` directory+mod catalog_listing;+/// Run all tests that are found in the `tracing` directorymod tracing;diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rsindex f89ca9e049..33129150db 100644--- a/datafusion/core/tests/datasource/object_store_access.rs+++ b/datafusion/core/tests/datasource/object_store_access.rs@@ -145,17 +145,8 @@ async fn query_partitioned_csv_file() {+---------+-------+-------+---+----+-----+------- Object Store Request Summary -------RequestCountingObjectStore()- Total Requests: 13- - LIST (with delimiter) prefix=data- - LIST (with delimiter) prefix=data/a=1- - LIST (with delimiter) prefix=data/a=2- - LIST (with delimiter) prefix=data/a=3- - LIST (with delimiter) prefix=data/a=1/b=10- - LIST (with delimiter) prefix=data/a=2/b=20- - LIST (with delimiter) prefix=data/a=3/b=30- - LIST (with delimiter) prefix=data/a=1/b=10/c=100- - LIST (with delimiter) prefix=data/a=2/b=20/c=200- - LIST (with delimiter) prefix=data/a=3/b=30/c=300+ Total Requests: 4+ - LIST prefix=data- GET (opts) path=data/a=1/b=10/c=100/file_1.csv- GET (opts) path=data/a=2/b=20/c=200/file_2.csv- GET (opts) path=data/a=3/b=30/c=300/file_3.csv@@ -174,10 +165,8 @@ async fn query_partitioned_csv_file() {+---------+-------+-------+---+----+-----+------- Object Store Request Summary -------RequestCountingObjectStore()- Total Requests: 4- - LIST (with delimiter) prefix=data/a=2- - LIST (with delimiter) prefix=data/a=2/b=20- - LIST (with delimiter) prefix=data/a=2/b=20/c=200+ Total Requests: 2+ - LIST prefix=data- GET (opts) path=data/a=2/b=20/c=200/file_2.csv");@@ -194,17 +183,8 @@ async fn query_partitioned_csv_file() {+---------+-------+-------+---+----+-----+------- Object Store Request Summary -------RequestCountingObjectStore()- Total Requests: 11- - LIST (with delimiter) prefix=data- - LIST (with delimiter) prefix=data/a=1- - LIST (with delimiter) prefix=data/a=2- - LIST (with delimiter) prefix=data/a=3- - LIST (with delimiter) prefix=data/a=1/b=10- - LIST (with delimiter) prefix=data/a=2/b=20- - LIST (with delimiter) prefix=data/a=3/b=30- - LIST (with delimiter) prefix=data/a=1/b=10/c=100- - LIST (with delimiter) prefix=data/a=2/b=20/c=200- - LIST (with delimiter) prefix=data/a=3/b=30/c=300+ Total Requests: 2+ - LIST prefix=data- GET (opts) path=data/a=2/b=20/c=200/file_2.csv");@@ -221,17 +201,8 @@ async fn query_partitioned_csv_file() {+---------+-------+-------+---+----+-----+------- Object Store Request Summary -------RequestCountingObjectStore()- Total Requests: 11- - LIST (with delimiter) prefix=data- - LIST (with delimiter) prefix=data/a=1- - LIST (with delimiter) prefix=data/a=2- - LIST (with delimiter) prefix=data/a=3- - LIST (with delimiter) prefix=data/a=1/b=10- - LIST (with delimiter) prefix=data/a=2/b=20- - LIST (with delimiter) prefix=data/a=3/b=30- - LIST (with delimiter) prefix=data/a=1/b=10/c=100- - LIST (with delimiter) prefix=data/a=2/b=20/c=200- - LIST (with delimiter) prefix=data/a=3/b=30/c=300+ Total Requests: 2+ - LIST prefix=data- GET (opts) path=data/a=2/b=20/c=200/file_2.csv");@@ -248,9 +219,8 @@ async fn query_partitioned_csv_file() {+---------+-------+-------+---+----+-----+------- Object Store Request Summary -------RequestCountingObjectStore()- Total Requests: 3- - LIST (with delimiter) prefix=data/a=2/b=20- - LIST (with delimiter) prefix=data/a=2/b=20/c=200+ Total Requests: 2+ - LIST prefix=data- GET (opts) path=data/a=2/b=20/c=200/file_2.csv");@@ -267,17 +237,8 @@ async fn query_partitioned_csv_file() {+---------+-------+-------+---+----+-----+------- Object Store Request Summary -------RequestCountingObjectStore()- Total Requests: 11- - LIST (with delimiter) prefix=data- - LIST (with delimiter) prefix=data/a=1- - LIST (with delimiter) prefix=data/a=2- - LIST (with delimiter) prefix=data/a=3- - LIST (with delimiter) prefix=data/a=1/b=10- - LIST (with delimiter) prefix=data/a=2/b=20- - LIST (with delimiter) prefix=data/a=3/b=30- - LIST (with delimiter) prefix=data/a=1/b=10/c=100- - LIST (with delimiter) prefix=data/a=2/b=20/c=200- - LIST (with delimiter) prefix=data/a=3/b=30/c=300+ Total Requests: 2+ - LIST prefix=data- GET (opts) path=data/a=1/b=10/c=100/file_1.csv");