Skip to content

Apache DataFusion SQL Query Engine

bench: measure projected Parquet scans across file boundaries (#27016)

Chao Suncaaf0fae8a42
diff --git a/datafusion/core/benches/sql_query_with_io.rs b/datafusion/core/benches/sql_query_with_io.rs
index 88aa4d3c67..c57fc7780c 100644
--- a/datafusion/core/benches/sql_query_with_io.rs
+++ b/datafusion/core/benches/sql_query_with_io.rs
@@ -16,6 +16,7 @@ use datafusion::{
prelude::{SessionConfig, SessionContext},
};
use datafusion_execution::runtime_env::RuntimeEnv;
+use datafusion_physical_plan::{collect, execution_plan::reset_plan_states};
use itertools::Itertools;
use object_store::{
ObjectStore, ObjectStoreExt,
@@ -67,7 +68,7 @@ fn create_parquet_file(rng: &mut StdRng, id_offset: usize) -> Bytes {
let mut data = Vec::<u8>::new();
let mut writer = ArrowWriter::try_new(&mut data, schema, Default::default()).unwrap();
writer.write(&batch).unwrap();
- writer.close().unwrap();
+ assert_eq!(writer.close().unwrap().num_row_groups(), 1);
data.into()
}
@@ -101,15 +102,19 @@ where
Arc::new(ThrottledStore::new(object_store, config))
}
-async fn setup_context(object_store: Arc<dyn ObjectStore>) -> SessionContext {
- let config = SessionConfig::new().with_target_partitions(THREADS);
+async fn setup_context(
+ object_store: Arc<dyn ObjectStore>,
+ config: SessionConfig,
+) -> SessionContext {
let rt = Arc::new(RuntimeEnv::default());
rt.register_object_store(&Url::parse("data://my_store").unwrap(), object_store);
let context = SessionContext::new_with_config_rt(config, rt);
for table_id in 0..TABLES {
let table_name = table_name(table_id);
- let file_format = ParquetFormat::default().with_enable_pruning(true);
+ let file_format = ParquetFormat::default()
+ .with_options(context.copied_table_options().parquet)
+ .with_enable_pruning(true);
let options = ListingOptions::new(Arc::new(file_format))
.with_table_partition_cols(vec![(
String::from("partition"),
@@ -167,7 +172,8 @@ fn bench_query(
fn criterion_benchmark(c: &mut Criterion) {
env_logger::try_init().ok();
- let store = make_store_slow(InMemory::new());
+ let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+ let slow_store = make_store_slow(Arc::clone(&store));
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(THREADS)
.enable_time()
@@ -175,7 +181,10 @@ fn criterion_benchmark(c: &mut Criterion) {
.unwrap();
rt.block_on(setup_files(Arc::clone(&store)));
- let ctx = rt.block_on(setup_context(store));
+ let ctx = rt.block_on(setup_context(
+ Arc::clone(&slow_store),
+ SessionConfig::new().with_target_partitions(THREADS),
+ ));
let table0_name = table_name(0);
let select_query = format!("SELECT * FROM {table0_name}");
@@ -244,6 +253,37 @@ fn criterion_benchmark(c: &mut Criterion) {
&format!("{join_query} WHERE {table0_name}.partition = 0"),
PARTITION_FILES * FILE_ROWS,
);
+
+ // Each file contains one row group. Using one scan partition makes crossing
+ // a file boundary the only opportunity to read ahead into another group.
+ let mut group = c.benchmark_group("parquet_cross_file_projection");
+ group.sample_size(10);
+ group.sampling_mode(SamplingMode::Flat);
+ group.measurement_time(Duration::from_secs(25));
+ for (name, store) in [("in_memory", store), ("100ms_get", slow_store)] {
+ let config = SessionConfig::from_env().unwrap().with_target_partitions(1);
+ let ctx = rt.block_on(setup_context(store, config));
+ let plan = rt.block_on(async {
+ ctx.sql("SELECT sum(id) FROM table_0")
+ .await
+ .unwrap()
+ .create_physical_plan()
+ .await
+ .unwrap()
+ });
+ let rows_per_partition = (PARTITION_FILES * FILE_ROWS) as u64;
+ let expected =
+ TABLE_PARTITIONS as u64 * rows_per_partition * (rows_per_partition - 1) / 2;
+ let scan = || async {
+ let plan = reset_plan_states(Arc::clone(&plan)).unwrap();
+ let batches = collect(plan, ctx.task_ctx()).await.unwrap();
+ let sum =
+ datafusion_common::cast::as_uint64_array(batches[0].column(0)).unwrap();
+ assert_eq!(sum.value(0), expected);
+ };
+ group.bench_function(name, |b| b.to_async(&rt).iter(scan));
+ }
+ group.finish();
}
criterion_group!(benches, criterion_benchmark);
diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rs
index d9dffa33bd..f89879709a 100644
--- a/datafusion/core/tests/datasource/object_store_access.rs
+++ b/datafusion/core/tests/datasource/object_store_access.rs
@@ -961,6 +961,92 @@ async fn query_single_parquet_file() {
);
}
+#[tokio::test]
+async fn projected_parquet_ranges_across_files() -> datafusion_common::Result<()> {
+ // Keep both files in one scan partition, and read just the footer when
+ // looking up metadata so those requests are distinct from column chunks.
+ let mut test = Test::new().with_parquet_metadata_size_hint(Some(8));
+ test.query("SET datafusion.execution.target_partitions = 1")
+ .await;
+ // Let the scan enforce LIMIT. Exact row counts would let the planner
+ // discard the second file before the scan starts.
+ test.query("SET datafusion.execution.collect_statistics = false")
+ .await;
+ let mut selected_chunks = vec![];
+ for (path, value) in [("data/a.parquet", 1), ("data/b.parquet", 3)] {
+ let unused: ArrayRef = Arc::new(Int32Array::from(vec![0, 0]));
+ let selected: ArrayRef = Arc::new(Int32Array::from(vec![value, value + 1]));
+ let batch =
+ RecordBatch::try_from_iter([("unused", unused), ("selected", selected)])?;
+ let mut bytes = vec![];
+ // A single data page gives the projected column a single payload range.
+ let props = parquet::file::properties::WriterProperties::builder()
+ .set_dictionary_enabled(false)
+ .build();
+ let mut writer = parquet::arrow::ArrowWriter::try_new(
+ &mut bytes,
+ batch.schema(),
+ Some(props),
+ )?;
+ writer.write(&batch)?;
+ let metadata = writer.close()?;
+ let (start, len) = metadata.row_group(0).column(1).byte_range();
+ selected_chunks.push((Path::from(path), bytes.len() as u64, start..start + len));
+ test = test.with_bytes(path, bytes).await;
+ }
+ let test = test.register_parquet("t", "/data/").await;
+
+ for (query, files_to_read) in [
+ ("SELECT selected FROM t", 2),
+ ("SELECT selected FROM t LIMIT 1", 1),
+ ] {
+ test.object_store.clear_requests();
+ let batches = test.session_context.sql(query).await?.collect().await?;
+ let mut values = vec![];
+ for batch in &batches {
+ values.extend_from_slice(
+ datafusion_common::cast::as_int32_array(batch.column(0))?.values(),
+ );
+ }
+ values.sort_unstable();
+ if files_to_read == 2 {
+ assert_eq!(values, [1, 2, 3, 4]);
+ } else {
+ assert!(matches!(values.as_slice(), [1] | [3]));
+ }
+
+ let requests = test.object_store.recorded_requests();
+ let mut files_read = 0;
+ for (file, size, selected) in &selected_chunks {
+ let mut data_ranges = vec![];
+ for request in &requests {
+ match request {
+ RequestDetails::GetRanges { path, ranges } if path == file => {
+ data_ranges.extend_from_slice(ranges);
+ }
+ RequestDetails::GetOpts { path, get_options }
+ if path == file && !get_options.head =>
+ {
+ data_ranges.push(match &get_options.range {
+ Some(range) => range.as_range(*size).unwrap(),
+ None => 0..*size,
+ });
+ }
+ _ => {}
+ }
+ }
+ // Ignore footer reads. The selected column is the last data chunk.
+ data_ranges.retain(|range| range.start < selected.end);
+ if !data_ranges.is_empty() {
+ assert_eq!(data_ranges, std::slice::from_ref(selected), "{file}");
+ files_read += 1;
+ }
+ }
+ assert_eq!(files_read, files_to_read);
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn query_single_parquet_file_with_single_predicate() {
let test = Test::new().with_single_file_parquet().await;