Apache DataFusion SQL Query Engine
bench: measure projected Parquet scans across file boundaries (#27016)
Chao Suncaaf0fae8a42
diff --git a/datafusion/core/benches/sql_query_with_io.rs b/datafusion/core/benches/sql_query_with_io.rsindex 88aa4d3c67..c57fc7780c 100644--- a/datafusion/core/benches/sql_query_with_io.rs+++ b/datafusion/core/benches/sql_query_with_io.rs@@ -16,6 +16,7 @@ use datafusion::{prelude::{SessionConfig, SessionContext},};use datafusion_execution::runtime_env::RuntimeEnv;+use datafusion_physical_plan::{collect, execution_plan::reset_plan_states};use itertools::Itertools;use object_store::{ObjectStore, ObjectStoreExt,@@ -67,7 +68,7 @@ fn create_parquet_file(rng: &mut StdRng, id_offset: usize) -> Bytes {let mut data = Vec::<u8>::new();let mut writer = ArrowWriter::try_new(&mut data, schema, Default::default()).unwrap();writer.write(&batch).unwrap();- writer.close().unwrap();+ assert_eq!(writer.close().unwrap().num_row_groups(), 1);data.into()}@@ -101,15 +102,19 @@ whereArc::new(ThrottledStore::new(object_store, config))}-async fn setup_context(object_store: Arc<dyn ObjectStore>) -> SessionContext {- let config = SessionConfig::new().with_target_partitions(THREADS);+async fn setup_context(+ object_store: Arc<dyn ObjectStore>,+ config: SessionConfig,+) -> SessionContext {let rt = Arc::new(RuntimeEnv::default());rt.register_object_store(&Url::parse("data://my_store").unwrap(), object_store);let context = SessionContext::new_with_config_rt(config, rt);for table_id in 0..TABLES {let table_name = table_name(table_id);- let file_format = ParquetFormat::default().with_enable_pruning(true);+ let file_format = ParquetFormat::default()+ .with_options(context.copied_table_options().parquet)+ .with_enable_pruning(true);let options = ListingOptions::new(Arc::new(file_format)).with_table_partition_cols(vec![(String::from("partition"),@@ -167,7 +172,8 @@ fn bench_query(fn criterion_benchmark(c: &mut Criterion) {env_logger::try_init().ok();- let store = make_store_slow(InMemory::new());+ let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());+ let slow_store = make_store_slow(Arc::clone(&store));let rt = tokio::runtime::Builder::new_multi_thread().worker_threads(THREADS).enable_time()@@ -175,7 +181,10 @@ fn criterion_benchmark(c: &mut Criterion) {.unwrap();rt.block_on(setup_files(Arc::clone(&store)));- let ctx = rt.block_on(setup_context(store));+ let ctx = rt.block_on(setup_context(+ Arc::clone(&slow_store),+ SessionConfig::new().with_target_partitions(THREADS),+ ));let table0_name = table_name(0);let select_query = format!("SELECT * FROM {table0_name}");@@ -244,6 +253,37 @@ fn criterion_benchmark(c: &mut Criterion) {&format!("{join_query} WHERE {table0_name}.partition = 0"),PARTITION_FILES * FILE_ROWS,);++ // Each file contains one row group. Using one scan partition makes crossing+ // a file boundary the only opportunity to read ahead into another group.+ let mut group = c.benchmark_group("parquet_cross_file_projection");+ group.sample_size(10);+ group.sampling_mode(SamplingMode::Flat);+ group.measurement_time(Duration::from_secs(25));+ for (name, store) in [("in_memory", store), ("100ms_get", slow_store)] {+ let config = SessionConfig::from_env().unwrap().with_target_partitions(1);+ let ctx = rt.block_on(setup_context(store, config));+ let plan = rt.block_on(async {+ ctx.sql("SELECT sum(id) FROM table_0")+ .await+ .unwrap()+ .create_physical_plan()+ .await+ .unwrap()+ });+ let rows_per_partition = (PARTITION_FILES * FILE_ROWS) as u64;+ let expected =+ TABLE_PARTITIONS as u64 * rows_per_partition * (rows_per_partition - 1) / 2;+ let scan = || async {+ let plan = reset_plan_states(Arc::clone(&plan)).unwrap();+ let batches = collect(plan, ctx.task_ctx()).await.unwrap();+ let sum =+ datafusion_common::cast::as_uint64_array(batches[0].column(0)).unwrap();+ assert_eq!(sum.value(0), expected);+ };+ group.bench_function(name, |b| b.to_async(&rt).iter(scan));+ }+ group.finish();}criterion_group!(benches, criterion_benchmark);diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rsindex d9dffa33bd..f89879709a 100644--- a/datafusion/core/tests/datasource/object_store_access.rs+++ b/datafusion/core/tests/datasource/object_store_access.rs@@ -961,6 +961,92 @@ async fn query_single_parquet_file() {);}+#[tokio::test]+async fn projected_parquet_ranges_across_files() -> datafusion_common::Result<()> {+ // Keep both files in one scan partition, and read just the footer when+ // looking up metadata so those requests are distinct from column chunks.+ let mut test = Test::new().with_parquet_metadata_size_hint(Some(8));+ test.query("SET datafusion.execution.target_partitions = 1")+ .await;+ // Let the scan enforce LIMIT. Exact row counts would let the planner+ // discard the second file before the scan starts.+ test.query("SET datafusion.execution.collect_statistics = false")+ .await;+ let mut selected_chunks = vec![];+ for (path, value) in [("data/a.parquet", 1), ("data/b.parquet", 3)] {+ let unused: ArrayRef = Arc::new(Int32Array::from(vec![0, 0]));+ let selected: ArrayRef = Arc::new(Int32Array::from(vec![value, value + 1]));+ let batch =+ RecordBatch::try_from_iter([("unused", unused), ("selected", selected)])?;+ let mut bytes = vec![];+ // A single data page gives the projected column a single payload range.+ let props = parquet::file::properties::WriterProperties::builder()+ .set_dictionary_enabled(false)+ .build();+ let mut writer = parquet::arrow::ArrowWriter::try_new(+ &mut bytes,+ batch.schema(),+ Some(props),+ )?;+ writer.write(&batch)?;+ let metadata = writer.close()?;+ let (start, len) = metadata.row_group(0).column(1).byte_range();+ selected_chunks.push((Path::from(path), bytes.len() as u64, start..start + len));+ test = test.with_bytes(path, bytes).await;+ }+ let test = test.register_parquet("t", "/data/").await;++ for (query, files_to_read) in [+ ("SELECT selected FROM t", 2),+ ("SELECT selected FROM t LIMIT 1", 1),+ ] {+ test.object_store.clear_requests();+ let batches = test.session_context.sql(query).await?.collect().await?;+ let mut values = vec![];+ for batch in &batches {+ values.extend_from_slice(+ datafusion_common::cast::as_int32_array(batch.column(0))?.values(),+ );+ }+ values.sort_unstable();+ if files_to_read == 2 {+ assert_eq!(values, [1, 2, 3, 4]);+ } else {+ assert!(matches!(values.as_slice(), [1] | [3]));+ }++ let requests = test.object_store.recorded_requests();+ let mut files_read = 0;+ for (file, size, selected) in &selected_chunks {+ let mut data_ranges = vec![];+ for request in &requests {+ match request {+ RequestDetails::GetRanges { path, ranges } if path == file => {+ data_ranges.extend_from_slice(ranges);+ }+ RequestDetails::GetOpts { path, get_options }+ if path == file && !get_options.head =>+ {+ data_ranges.push(match &get_options.range {+ Some(range) => range.as_range(*size).unwrap(),+ None => 0..*size,+ });+ }+ _ => {}+ }+ }+ // Ignore footer reads. The selected column is the last data chunk.+ data_ranges.retain(|range| range.start < selected.end);+ if !data_ranges.is_empty() {+ assert_eq!(data_ranges, std::slice::from_ref(selected), "{file}");+ files_read += 1;+ }+ }+ assert_eq!(files_read, files_to_read);+ }+ Ok(())+}+#[tokio::test]async fn query_single_parquet_file_with_single_predicate() {let test = Test::new().with_single_file_parquet().await;