Skip to content

Apache DataFusion SQL Query Engine

feat: preserve bitmap-backed Parquet row selections (#25731)

Huaijinc3ef34688741
diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rs
index 9ba0f80394..b699ae2e76 100644
--- a/datafusion/core/tests/parquet/external_access_plan.rs
+++ b/datafusion/core/tests/parquet/external_access_plan.rs
@@ -23,6 +23,7 @@ use std::sync::Arc;
use crate::parquet::utils::MetricsFinder;
use crate::parquet::{Scenario, create_data_batch};
+use arrow::buffer::BooleanBuffer;
use arrow::datatypes::SchemaRef;
use arrow::util::pretty::pretty_format_batches;
use datafusion::common::Result;
@@ -189,33 +190,82 @@ async fn row_selection_extension() {
#[tokio::test]
async fn row_selection_extension_spanning_row_groups() {
- // A selection whose selectors straddle the row group boundary (row 4 is the
- // last row of group 0, rows 5-6 are the first rows of group 1).
- let parquet_metrics = TestFull {
- access_plan: None,
- row_selection: Some(ParquetRowSelection::new(RowSelection::from(vec![
+ // Row 4 is the last row of group 0; rows 5-6 begin group 1.
+ for selection in [
+ RowSelection::from(vec![
RowSelector::skip(4),
RowSelector::select(3),
RowSelector::skip(3),
- ]))),
- expected_rows: 3,
+ ]),
+ RowSelection::from_boolean_buffer(BooleanBuffer::from(vec![
+ false, false, false, false, true, true, true, false, false, false,
+ ])),
+ ] {
+ let parquet_metrics = TestFull {
+ access_plan: None,
+ row_selection: Some(ParquetRowSelection::new(selection)),
+ expected_rows: 3,
+ expected_output: Some(&[
+ "+------+------------+",
+ "| utf8 | large_utf8 |",
+ "+------+------------+",
+ "| | |",
+ "| e | e |",
+ "| f | f |",
+ "+------+------------+",
+ ]),
+ predicate: None,
+ }
+ .run()
+ .await
+ .unwrap();
+
+ let bytes_scanned = metric_value(&parquet_metrics, "bytes_scanned").unwrap();
+ assert_ne!(bytes_scanned, 0, "metrics : {parquet_metrics:#?}");
+ }
+}
+
+#[tokio::test]
+async fn bitmap_row_selection_extension_with_predicate() {
+ // Pruning removes row group 0, leaving the bitmap-selected rows in group 1.
+ let parquet_metrics = TestFull {
+ access_plan: None,
+ row_selection: Some(ParquetRowSelection::new(RowSelection::from_boolean_buffer(
+ BooleanBuffer::from(vec![
+ false, false, true, false, true, true, true, false, false, false,
+ ]),
+ ))),
+ expected_rows: 2,
expected_output: Some(&[
"+------+------------+",
"| utf8 | large_utf8 |",
"+------+------------+",
- "| | |",
"| e | e |",
"| f | f |",
"+------+------------+",
]),
- predicate: None,
+ predicate: Some(col("utf8").eq(lit("e"))),
}
.run()
.await
.unwrap();
- let bytes_scanned = metric_value(&parquet_metrics, "bytes_scanned").unwrap();
- assert_ne!(bytes_scanned, 0, "metrics : {parquet_metrics:#?}");
+ // Verify that statistics pruned row group 0.
+ let row_groups_pruned_statistics = parquet_metrics
+ .sum_by_name("row_groups_pruned_statistics")
+ .unwrap();
+ if let MetricValue::PruningMetrics {
+ pruning_metrics, ..
+ } = row_groups_pruned_statistics
+ {
+ assert_eq!(
+ pruning_metrics.pruned(),
+ 1,
+ "metrics : {parquet_metrics:#?}"
+ );
+ } else {
+ unreachable!("metrics `row_groups_pruned_statistics` should exist")
+ }
}
#[tokio::test]
@@ -316,9 +366,9 @@ async fn mixed_bitmap_and_selector_selections() {
// Each representation must keep coordinates local to its row group.
TestFull {
access_plan: Some(ParquetAccessPlan::new(vec![
- RowGroupAccess::Selection(RowSelection::from(
- arrow::buffer::BooleanBuffer::from(vec![true, false, true, false, false]),
- )),
+ RowGroupAccess::Selection(RowSelection::from(BooleanBuffer::from(vec![
+ true, false, true, false, false,
+ ]))),
RowGroupAccess::Selection(select_two_rows()),
])),
row_selection: None,
diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rs
index b72612134d..961a91c0b4 100644
--- a/datafusion/datasource-parquet/src/access_plan.rs
+++ b/datafusion/datasource-parquet/src/access_plan.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use arrow::array::BooleanBufferBuilder;
use arrow::datatypes::Schema;
use datafusion_common::{Result, assert_eq_or_internal_err, exec_err};
use datafusion_physical_expr::expressions::Column;
@@ -297,6 +298,54 @@ impl ParquetAccessPlan {
pub fn try_new_from_overall_row_selection(
selection: RowSelection,
row_group_meta_data: &[RowGroupMetaData],
+ ) -> Result<Self> {
+ if selection.as_mask().is_some() {
+ Self::try_new_from_overall_row_selection_mask(selection, row_group_meta_data)
+ } else {
+ Self::try_new_from_overall_row_selection_selectors(
+ selection,
+ row_group_meta_data,
+ )
+ }
+ }
+
+ fn try_new_from_overall_row_selection_mask(
+ mut selection: RowSelection,
+ row_group_meta_data: &[RowGroupMetaData],
+ ) -> Result<Self> {
+ let selection_rows = selection.total_row_count();
+ let file_rows = row_group_meta_data
+ .iter()
+ .map(|rg| rg.num_rows() as usize)
+ .sum::<usize>();
+ if selection_rows != file_rows {
+ return exec_err!(
+ "Invalid Parquet RowSelection. File has {file_rows} rows, \
+ but selection specifies {selection_rows} rows."
+ );
+ }
+
+ // For masks, split_off uses bitmap slices without materializing
+ // selectors. Use row_count() to cache each partial group's count
+ // for later preparation.
+ let row_groups = row_group_meta_data
+ .iter()
+ .map(|rg| {
+ let row_count = rg.num_rows() as usize;
+ let group_selection = selection.split_off(row_count);
+ match group_selection.row_count() {
+ 0 => RowGroupAccess::Skip,
+ selected if selected == row_count => RowGroupAccess::Scan,
+ _ => RowGroupAccess::Selection(group_selection),
+ }
+ })
+ .collect();
+ Ok(Self::new(row_groups))
+ }
+
+ fn try_new_from_overall_row_selection_selectors(
+ selection: RowSelection,
+ row_group_meta_data: &[RowGroupMetaData],
) -> Result<Self> {
// Keep this as a single pass over the selector stream rather than
// repeatedly calling `RowSelection::split_off` per row group. The
@@ -392,6 +441,22 @@ impl ParquetAccessPlan {
RowGroupAccess::Skip => RowGroupAccess::Skip,
RowGroupAccess::Scan => RowGroupAccess::Selection(selection),
RowGroupAccess::Selection(existing_selection) => {
+ // Parquet preserves bitmap backing only when both operands
+ // are masks. Promote selector-backed page pruning to retain
+ // an external index's bitmap and use a bitwise intersection.
+ // Revisit this conversion once Parquet optimizes mixed-backed
+ // intersections: https://github.com/apache/arrow-rs/issues/10423
+ let selection = if existing_selection.as_mask().is_some()
+ && selection.as_mask().is_none()
+ {
+ let mut mask = BooleanBufferBuilder::new(selection.total_row_count());
+ for selector in selection.iter() {
+ mask.append_n(selector.row_count, !selector.skip);
+ }
+ RowSelection::from(mask.finish())
+ } else {
+ selection
+ };
RowGroupAccess::Selection(existing_selection.intersection(&selection))
}
}
@@ -838,6 +903,7 @@ impl PreparedAccessPlan {
#[cfg(test)]
mod test {
use super::*;
+ use arrow::buffer::BooleanBuffer;
use datafusion_common::assert_contains;
use parquet::basic::LogicalType;
use parquet::file::metadata::ColumnChunkMetaData;
@@ -862,7 +928,7 @@ mod test {
// Check both input representations retain the same conversion behavior.
let selectors =
RowSelection::from(vec![RowSelector::skip(10), RowSelector::select(20)]);
- let bitmap = RowSelection::from(arrow::buffer::BooleanBuffer::from(
+ let bitmap = RowSelection::from(BooleanBuffer::from(
(0..30).map(|i| i >= 10).collect::<Vec<_>>(),
));
for selection in [selectors, bitmap] {
@@ -986,6 +1052,110 @@ mod test {
}
}
+ #[test]
+ fn test_scan_selection_preserves_mask_backing() {
+ let mask = BooleanBuffer::from(vec![
+ true, true, false, false, true, true, false, false, true, true,
+ ]);
+ let selectors =
+ RowSelection::from(vec![RowSelector::select(5), RowSelector::skip(5)]);
+ // Both selector-backed page pruning and bitmap intersections retain
+ // the existing mask, including when no rows survive.
+ for incoming in [
+ selectors,
+ RowSelection::from(BooleanBuffer::from(vec![
+ true, true, true, true, true, false, false, false, false, false,
+ ])),
+ RowSelection::from(vec![RowSelector::skip(10)]),
+ ] {
+ let empty = incoming.row_count() == 0;
+ let mut plan = ParquetAccessPlan::new(vec![RowGroupAccess::Selection(
+ RowSelection::from(mask.clone()),
+ )]);
+ plan.scan_selection(0, incoming);
+ let RowGroupAccess::Selection(selection) = &plan.inner()[0] else {
+ panic!("expected selection");
+ };
+ let expected = if empty {
+ BooleanBuffer::new_unset(10)
+ } else {
+ BooleanBuffer::from(vec![
+ true, true, false, false, true, false, false, false, false, false,
+ ])
+ };
+ assert_eq!(selection.as_mask(), Some(&expected));
+ }
+ }
+
+ #[test]
+ fn test_scan_selection_preserves_selector_backing() {
+ let mut plan = ParquetAccessPlan::new(vec![RowGroupAccess::Selection(
+ RowSelection::from(vec![RowSelector::select(6), RowSelector::skip(4)]),
+ )]);
+ plan.scan_selection(
+ 0,
+ RowSelection::from(vec![
+ RowSelector::skip(2),
+ RowSelector::select(5),
+ RowSelector::skip(3),
+ ]),
+ );
+ let RowGroupAccess::Selection(selection) = &plan.inner()[0] else {
+ panic!("expected selection");
+ };
+ assert!(selection.as_mask().is_none());
+ assert_eq!(
+ selection,
+ &RowSelection::from(vec![
+ RowSelector::skip(2),
+ RowSelector::select(4),
+ RowSelector::skip(4),
+ ])
+ );
+ }
+
+ #[test]
+ fn test_new_from_overall_mask_preserves_bitmap_backing() {
+ // Include all-selected, all-skipped, and fragmented groups. Start at
+ // a non-byte-aligned offset to exercise slicing an existing bitmap.
+ let mut bits = vec![false; 3];
+ bits.extend(vec![true; 10]);
+ bits.extend(vec![false; 20]);
+ bits.extend((0..30).map(|i| i % 2 == 1));
+ bits.extend(vec![true; 40]);
+ let mask = BooleanBuffer::from(bits).slice(3, 100);
+ for cache_row_count in [false, true] {
+ let selection = RowSelection::from(mask.clone());
+ if cache_row_count {
+ // Exercise split_off's propagation of an already cached count.
+ assert_eq!(selection.row_count(), 65);
+ }
+ let plan = ParquetAccessPlan::try_new_from_overall_row_selection(
+ selection,
+ &ROW_GROUP_METADATA,
+ )
+ .unwrap();
+ assert_eq!(plan.inner()[0], RowGroupAccess::Scan);
+ assert_eq!(plan.inner()[1], RowGroupAccess::Skip);
+ assert_eq!(plan.inner()[3], RowGroupAccess::Scan);
+ let RowGroupAccess::Selection(selection) = &plan.inner()[2] else {
+ panic!("expected selection");
+ };
+ assert_eq!(selection.row_count(), 15);
+ let group_mask = mask.slice(30, 30);
+ assert!(selection.as_mask().unwrap().ptr_eq(&group_mask));
+
+ // The local selection must also survive preparation and reversal.
+ let prepared = plan.prepare(&ROW_GROUP_METADATA).unwrap().reverse();
+ assert_eq!(prepared.row_group_indexes(), vec![3, 2, 0]);
+ assert!(prepared.row_groups[0].selection.selection().is_none());
+ assert!(prepared.row_groups[2].selection.selection().is_none());
+ let selection = prepared.row_groups[1].selection.selection().unwrap();
+ assert_eq!(selection.row_count(), 15);
+ assert!(selection.as_mask().unwrap().ptr_eq(&group_mask));
+ }
+ }
+
#[test]
fn test_new_from_overall_row_selection() {
let row_selection = RowSelection::from(vec![
@@ -1022,19 +1192,26 @@ mod test {
#[test]
fn test_new_from_overall_row_selection_invalid_row_count() {
- let row_selection = RowSelection::from(vec![RowSelector::select(99)]);
-
- let err = ParquetAccessPlan::try_new_from_overall_row_selection(
- row_selection,
- &ROW_GROUP_METADATA,
- )
- .unwrap_err()
- .to_string();
-
- assert_contains!(
- err,
- "Invalid Parquet RowSelection. File has 100 rows, but selection specifies 99 rows"
- );
+ for selection_rows in [99, 101] {
+ for selection in [
+ RowSelection::from(vec![RowSelector::select(selection_rows)]),
+ RowSelection::from(BooleanBuffer::new_set(selection_rows)),
+ ] {
+ let err = ParquetAccessPlan::try_new_from_overall_row_selection(
+ selection,
+ &ROW_GROUP_METADATA,
+ )
+ .unwrap_err()
+ .to_string();
+ assert_contains!(
+ err,
+ format!(
+ "Invalid Parquet RowSelection. File has 100 rows, \
+ but selection specifies {selection_rows} rows"
+ )
+ );
+ }
+ }
}
#[test]
diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs
index ccd9e098df..3cd9e795c1 100644
--- a/datafusion/datasource-parquet/src/opener/mod.rs
+++ b/datafusion/datasource-parquet/src/opener/mod.rs
@@ -2144,26 +2144,32 @@ async fn load_page_index<T: AsyncFileReader>(
mod test {
use super::*;
use super::{ConstantColumns, ParquetMorselizer, constant_columns_from_stats};
- use crate::metadata::DFParquetMetadata;
+ use crate::metadata::{CachedParquetMetaData, DFParquetMetadata};
+ use crate::source::ParquetSource;
use crate::{
CachedParquetFileReaderFactory, DefaultParquetFileReaderFactory,
ParquetFileReaderFactory, ParquetFileSchemaProvider, ParquetRowSelection,
RowGroupAccess,
};
- use arrow::array::{AsArray, RecordBatch, record_batch};
- use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+ use arrow::array::{Array, AsArray, RecordBatch, record_batch};
+ use arrow::buffer::BooleanBuffer;
+ use arrow::datatypes::{DataType, Field, Int32Type, Schema, SchemaRef};
use bytes::{BufMut, BytesMut};
use datafusion_common::{
ColumnStatistics, ScalarValue, Statistics, assert_contains, internal_err,
stats::Precision,
};
+ use datafusion_datasource::file::FileSource;
+ use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
use datafusion_datasource::morsel::{Morsel, Morselizer};
use datafusion_datasource::{PartitionedFile, TableSchema, TableSchemaBuilder};
use datafusion_execution::cache::cache_manager::{
CachedFileMetadataEntry, FileMetadataCache,
};
use datafusion_execution::cache::default_cache::DefaultCache;
+ use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_expr::{Expr, col, lit};
+ use datafusion_functions::core::expr_fn::get_field;
use datafusion_physical_expr::{
PhysicalExpr,
expressions::{Column, DynamicFilterPhysicalExpr, Literal},
@@ -2173,15 +2179,23 @@ mod test {
use datafusion_physical_expr_adapter::{
DefaultPhysicalExprAdapterFactory, replace_columns_with_literals,
};
- use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
+ use datafusion_physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue};
use datafusion_pruning::MAX_IN_LIST_SIZE;
use futures::stream::BoxStream;
use futures::{StreamExt, TryStreamExt};
use object_store::{ObjectStore, ObjectStoreExt, memory::InMemory, path::Path};
+ use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
use parquet::arrow::{ArrowSchemaConverter, ArrowWriter, parquet_to_arrow_schema};
use parquet::basic::ConvertedType;
- use parquet::file::metadata::{ColumnChunkMetaData, FileMetaData, ParquetMetaData};
- use parquet::file::properties::{EnabledStatistics, WriterProperties};
+ use parquet::data_type::{ByteArray, ByteArrayType, Int32Type as ParquetInt32Type};
+ #[cfg(feature = "parquet_encryption")]
+ use parquet::encryption::encrypt::FileEncryptionProperties;
+ use parquet::file::metadata::{
+ ColumnChunkMetaData, FileMetaData, KeyValue, ParquetMetaData,
+ };
+ use parquet::file::properties::{EnabledStatistics, WriterProperties, WriterVersion};
+ use parquet::file::writer::SerializedFileWriter;
+ use parquet::schema::parser::parse_message_type;
use parquet::schema::types::{SchemaDescPtr, SchemaDescriptor};
use std::collections::VecDeque;
use std::sync::Arc;
@@ -2217,8 +2231,6 @@ mod test {
#[test]
fn create_initial_plan_from_parquet_row_selection_extension() {
- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
-
let mut extensions = datafusion_datasource::FileExtensions::new();
extensions.insert(ParquetRowSelection::new(RowSelection::from(vec![
RowSelector::select(10),
@@ -2242,8 +2254,6 @@ mod test {
#[test]
fn create_initial_plan_rejects_multiple_access_extensions() {
- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
-
let mut extensions = datafusion_datasource::FileExtensions::new();
extensions.insert(ParquetAccessPlan::new_all(3));
extensions.insert(ParquetRowSelection::new(RowSelection::from(vec![
@@ -2370,7 +2380,6 @@ mod test {
plan: ParquetAccessPlan,
projection: Option<&[usize]>,
) -> bool {
- use crate::RowGroupAccessPlanFilter;
let arrow_schema: SchemaRef = Arc::new(
parquet_to_arrow_schema(metadata.file_metadata().schema_descr(), None)
.unwrap(),
@@ -2779,7 +2788,6 @@ mod test {
async fn collect_int32_values(
mut stream: BoxStream<'static, Result<RecordBatch>>,
) -> Vec<i32> {
- use arrow::array::Array;
let mut values = vec![];
while let Some(Ok(batch)) = stream.next().await {
let array = batch
@@ -2818,8 +2826,6 @@ mod test {
batch: RecordBatch,
logical_file_schema: &SchemaRef,
) -> (usize, Statistics) {
- use parquet::file::metadata::ParquetMetaDataReader;
-
let mut out = BytesMut::new().writer();
{
let mut writer =
@@ -2864,7 +2870,6 @@ mod test {
}
fn counter_metric_value(metrics: &ExecutionPlanMetricsSet, name: &str) -> usize {
- use datafusion_physical_plan::metrics::MetricValue;
metrics
.clone_inner()
.sum_by_name(name)
@@ -3380,7 +3385,6 @@ mod test {
/// row-filtered" — both yield zero rows — so the tests around
/// constant-column substitution assert on this metric instead.
fn pruned_row_groups_statistics(metrics: &ExecutionPlanMetricsSet) -> usize {
- use datafusion_physical_plan::metrics::MetricValue;
metrics
.clone_inner()
.iter()
@@ -3917,10 +3921,6 @@ mod test {
extra_field: bool,
props: WriterProperties,
) -> (PartitionedFile, SchemaRef) {
- use parquet::data_type::{ByteArray, ByteArrayType, Int32Type};
- use parquet::file::writer::SerializedFileWriter;
- use parquet::schema::parser::parse_message_type;
-
let extra = if extra_field {
"required int32 extra;"
} else {
@@ -3943,7 +3943,7 @@ mod test {
let mut row_group = writer.next_row_group().unwrap();
let mut column = row_group.next_column().unwrap().unwrap();
column
- .typed::<Int32Type>()
+ .typed::<ParquetInt32Type>()
.write_batch(&[1, 2, 3], None, None)
.unwrap();
column.close().unwrap();
@@ -3965,7 +3965,7 @@ mod test {
if extra_field {
let mut column = row_group.next_column().unwrap().unwrap();
column
- .typed::<Int32Type>()
+ .typed::<ParquetInt32Type>()
.write_batch(&[10, 20, 30], None, None)
.unwrap();
column.close().unwrap();
@@ -3981,7 +3981,6 @@ mod test {
}
fn invalid_arrow_hint() -> parquet::file::properties::WriterPropertiesBuilder {
- use parquet::file::metadata::KeyValue;
WriterProperties::builder()
.set_key_value_metadata(Some(vec![KeyValue::new(
"ARROW:schema".to_string(),
@@ -3994,12 +3993,6 @@ mod test {
#[tokio::test]
async fn test_schema_provider_lazy_nested_scans() {
- use crate::source::ParquetSource;
- use datafusion_datasource::file::FileSource;
- use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
- use datafusion_execution::object_store::ObjectStoreUrl;
- use datafusion_functions::core::expr_fn::get_field;
-
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let (file1, schema) = write_enum_file(
store.as_ref(),
@@ -4064,8 +4057,6 @@ mod test {
#[tokio::test]
async fn test_schema_provider_preserves_metadata_and_precedence() {
- use crate::metadata::{CachedParquetMetaData, DFParquetMetadata};
-
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let (file, schema) = write_enum_file(
store.as_ref(),
@@ -4187,9 +4178,6 @@ mod test {
#[cfg(feature = "parquet_encryption")]
#[tokio::test]
async fn test_schema_provider_encrypted_scan() {
- use datafusion_physical_plan::metrics::MetricValue;
- use parquet::encryption::encrypt::FileEncryptionProperties;
-
let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
let footer_key = b"0123456789012345".to_vec();
let column_key = b"1234567890123450".to_vec();
@@ -4259,8 +4247,6 @@ mod test {
#[tokio::test]
async fn test_reverse_scan_row_groups() {
- use parquet::file::properties::WriterProperties;
-
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
// Create multiple batches to ensure multiple row groups
@@ -4357,9 +4343,108 @@ mod test {
}
#[tokio::test]
- async fn test_reverse_scan_with_row_selection() {
- use parquet::file::properties::WriterProperties;
+ async fn test_dynamic_pruning_preserves_external_row_selection() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ // After RG0, tighten the dynamic filter to a < 50. RG1 can then be
+ // pruned, but RG2 survives and must retain its external selection.
+ let batch = record_batch!((
+ "a",
+ Int32,
+ vec![0, 1, 2, 3, 100, 101, 102, 103, 10, 11, 12, 13]
+ ))
+ .unwrap();
+ let schema = batch.schema();
+ let data_len = write_parquet_batches(
+ Arc::clone(&store),
+ "test.parquet",
+ vec![batch],
+ Some(
+ WriterProperties::builder()
+ .set_max_row_group_row_count(Some(4))
+ .build(),
+ ),
+ )
+ .await;
+ let file = PartitionedFile::new("test.parquet".to_string(), data_len as u64);
+
+ for selection in [
+ None,
+ Some(RowSelection::from(vec![
+ RowSelector::select(1),
+ RowSelector::skip(1),
+ RowSelector::select(1),
+ RowSelector::skip(1),
+ ])),
+ Some(RowSelection::from(BooleanBuffer::from(vec![
+ true, false, true, false,
+ ]))),
+ ] {
+ let has_selection = selection.is_some();
+ let mut file = file.clone();
+ if let Some(selection) = selection {
+ file = file.with_extension(ParquetAccessPlan::new(vec![
+ RowGroupAccess::Scan,
+ RowGroupAccess::Scan,
+ RowGroupAccess::Selection(selection),
+ ]));
+ }
+ let dynamic = Arc::new(DynamicFilterPhysicalExpr::new(
+ vec![Arc::new(Column::new("a", 0))],
+ logical2physical(&col("a").lt(lit(200i32)), &schema),
+ ));
+ let metrics = ExecutionPlanMetricsSet::new();
+ let opener = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_schema(Arc::clone(&schema))
+ .with_projection_indices(&[0])
+ .with_predicate(Arc::clone(&dynamic) as Arc<dyn PhysicalExpr>)
+ .with_row_group_stats_pruning(true)
+ .with_preserve_order(true)
+ .with_metrics(metrics.clone())
+ .build();
+ let mut stream = open_file(&opener, file).await.unwrap();
+ let first = stream.next().await.unwrap().unwrap();
+ assert_eq!(
+ first
+ .column(0)
+ .as_primitive::<Int32Type>()
+ .values()
+ .as_ref(),
+ &[0, 1, 2, 3]
+ );
+ dynamic
+ .update(logical2physical(&col("a").lt(lit(50i32)), &schema))
+ .unwrap();
+
+ // Apply the final predicate as the consumer would. Runtime
+ // pruning is best-effort, so RG1 may still be read when the
+ // selection disables it. RG2's unselected rows also satisfy
+ // this predicate and must never reach the consumer.
+ let remaining: Vec<i32> = collect_int32_values(stream)
+ .await
+ .into_iter()
+ .filter(|&value| value < 50)
+ .collect();
+ if has_selection {
+ // Do not assert that runtime pruning is disabled: enabling it
+ // later must preserve these rows through decoder rebuilds.
+ // Replacing RG2's selection with None would also return 11/13,
+ // which satisfy the predicate but not the external selection.
+ assert_eq!(remaining, vec![10, 12]);
+ } else {
+ assert_eq!(remaining, vec![10, 11, 12, 13]);
+ // Control case: prove the late update exercises runtime
+ // pruning rather than only the per-row filter.
+ assert_eq!(
+ counter_metric_value(&metrics, "row_groups_pruned_dynamic_filter"),
+ 1
+ );
+ }
+ }
+ }
+ #[tokio::test]
+ async fn test_reverse_scan_with_row_selection() {
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
// Create 3 batches with DIFFERENT selection patterns
@@ -4387,9 +4472,6 @@ mod test {
let schema = batch1.schema();
- use crate::ParquetAccessPlan;
- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
-
let mut access_plan = ParquetAccessPlan::new_all(3);
// Row group 0: skip first 2, select last 2 (should get: 3, 4)
access_plan.scan_selection(
@@ -4450,8 +4532,6 @@ mod test {
#[tokio::test]
async fn test_reverse_scan_with_non_contiguous_row_groups() {
- use parquet::file::properties::WriterProperties;
-
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
// Create 4 batches (4 row groups)
@@ -4474,9 +4554,6 @@ mod test {
let schema = batch0.schema();
- use crate::ParquetAccessPlan;
- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
-
// KEY: Skip RG1 (non-contiguous!)
// Only scan row groups: [0, 2, 3]
let mut access_plan = ParquetAccessPlan::new(vec![
@@ -4553,8 +4630,6 @@ mod test {
/// since neither pushdown nor row-group pruning is active.
#[tokio::test]
async fn test_page_pruning_predicate_respects_enable_page_index() {
- use parquet::file::properties::WriterProperties;
-
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
// 100 rows with values 1..=100, written as a single row group with 10 rows per page
@@ -4697,8 +4772,6 @@ mod test {
#[test]
fn should_load_page_index_with_row_selection() {
- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
-
let mut plan = ParquetAccessPlan::new_all(1);
plan.scan_selection(
0,
@@ -4770,9 +4843,7 @@ mod test {
})
.collect::<Vec<_>>(),
);
- let mask = RowSelection::from_boolean_buffer(
- arrow::buffer::BooleanBuffer::from(bits),
- );
+ let mask = RowSelection::from_boolean_buffer(BooleanBuffer::from(bits));
for selection in [selectors, mask] {
let plan =
ParquetAccessPlan::new(vec![RowGroupAccess::Selection(selection)]);
@@ -4786,8 +4857,6 @@ mod test {
#[test]
fn should_load_page_index_with_row_selection_checks_projection() {
- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
-
let metadata = page_index_metadata(&[("a", false), ("b", true)], 1);
let plan = ParquetAccessPlan::new(vec![RowGroupAccess::Selection(
RowSelection::from(vec![RowSelector::skip(9), RowSelector::select(1)]),
@@ -4927,12 +4996,6 @@ mod test {
#[tokio::test]
async fn test_page_index_with_external_row_selection() {
- use arrow::datatypes::Int32Type;
- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
- use parquet::file::properties::{
- EnabledStatistics, WriterProperties, WriterVersion,
- };
-
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
let values: Vec<i32> = (0..10_000).collect();
let output_values: Vec<i32> = (10_000..20_000).collect();
@@ -5082,8 +5145,6 @@ mod test {
#[tokio::test]
async fn test_page_index_skipped_when_row_groups_fully_matched() {
- use parquet::file::properties::WriterProperties;
-
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
let values: Vec<i32> = (1..=100).collect();
let batch = record_batch!((
@@ -5129,8 +5190,6 @@ mod test {
#[tokio::test]
async fn test_page_index_skipped_with_cached_reader_factory() {
- use parquet::file::properties::WriterProperties;
-
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
let metadata_cache: Arc<FileMetadataCache> =
Arc::new(DefaultCache::<Path, CachedFileMetadataEntry>::new(
@@ -5197,8 +5256,6 @@ mod test {
#[tokio::test]
async fn test_page_index_loaded_when_not_fully_matched() {
- use parquet::file::properties::WriterProperties;
-
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
let values: Vec<i32> = (1..=100).collect();
let batch = record_batch!((
@@ -5245,8 +5302,6 @@ mod test {
async fn fully_matched_split_test_file(
store: Arc<dyn ObjectStore>,
) -> (SchemaRef, PartitionedFile) {
- use parquet::file::properties::WriterProperties;
-
let batch0 =
record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();
let batch1 =
@@ -5274,6 +5329,72 @@ mod test {
(schema, file)
}
+ #[tokio::test]
+ async fn test_filter_toggles_preserve_external_row_selection() {
+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
+ // For a > 5, the groups are fully matched, partially matched, then
+ // fully matched. Both boundaries must rebuild the decoder to toggle
+ // its row filter while preserving the remaining external selections.
+ let batch = record_batch!((
+ "a",
+ Int32,
+ vec![10, 11, 12, 13, 0, 20, 21, 22, 30, 31, 32, 33]
+ ))
+ .unwrap();
+ let schema = batch.schema();
+ let data_len = write_parquet_batches(
+ Arc::clone(&store),
+ "test.parquet",
+ vec![batch],
+ Some(
+ WriterProperties::builder()
+ .set_max_row_group_row_count(Some(4))
+ .build(),
+ ),
+ )
+ .await;
+ let file = PartitionedFile::new("test.parquet".to_string(), data_len as u64);
+
+ for selection in [
+ RowSelection::from(vec![
+ RowSelector::select(1),
+ RowSelector::skip(1),
+ RowSelector::select(1),
+ RowSelector::skip(1),
+ ]),
+ RowSelection::from(BooleanBuffer::from(vec![true, false, true, false])),
+ ] {
+ let file = file.clone().with_extension(ParquetAccessPlan::new(vec![
+ RowGroupAccess::Selection(selection.clone()),
+ RowGroupAccess::Selection(selection.clone()),
+ RowGroupAccess::Selection(selection),
+ ]));
+ let metrics = ExecutionPlanMetricsSet::new();
+ let opener = ParquetMorselizerBuilder::new()
+ .with_store(Arc::clone(&store))
+ .with_schema(Arc::clone(&schema))
+ .with_projection_indices(&[0])
+ .with_predicate(logical2physical(&col("a").gt(lit(5i32)), &schema))
+ .with_pushdown_filters(true)
+ .with_row_group_stats_pruning(true)
+ .with_preserve_order(true)
+ .with_metrics(metrics.clone())
+ .build();
+
+ let values =
+ collect_int32_values(open_file(&opener, file).await.unwrap()).await;
+ // 0 tests that the filter was enabled; unselected values that
+ // also pass a > 5 detect lost selections at either boundary.
+ assert_eq!(values, vec![10, 12, 21, 30, 32]);
+ // One skip at initialization and one after re-enabling the filter
+ // for RG1 prove that the fully-matched optimization was exercised.
+ assert_eq!(
+ counter_metric_value(&metrics, "row_filter_skipped_fully_matched"),
+ 2
+ );
+ }
+ }
+
#[tokio::test]
async fn test_fully_matched_runs_respect_global_limit() {
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
@@ -5358,7 +5479,7 @@ mod test {
/// (e.g. `row_number`) plumbed through `TableSchema`/`ParquetOpener`.
mod virtual_columns {
use super::*;
- use arrow::array::{Array, Int64Array, StringArray};
+ use arrow::array::{Int64Array, StringArray};
use arrow::datatypes::FieldRef;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::ScalarUDF;