Apache DataFusion SQL Query Engine
feat: preserve bitmap-backed Parquet row selections (#25731)
Huaijinc3ef34688741
diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rsindex 9ba0f80394..b699ae2e76 100644--- a/datafusion/core/tests/parquet/external_access_plan.rs+++ b/datafusion/core/tests/parquet/external_access_plan.rs@@ -23,6 +23,7 @@ use std::sync::Arc;use crate::parquet::utils::MetricsFinder;use crate::parquet::{Scenario, create_data_batch};+use arrow::buffer::BooleanBuffer;use arrow::datatypes::SchemaRef;use arrow::util::pretty::pretty_format_batches;use datafusion::common::Result;@@ -189,33 +190,82 @@ async fn row_selection_extension() {#[tokio::test]async fn row_selection_extension_spanning_row_groups() {- // A selection whose selectors straddle the row group boundary (row 4 is the- // last row of group 0, rows 5-6 are the first rows of group 1).- let parquet_metrics = TestFull {- access_plan: None,- row_selection: Some(ParquetRowSelection::new(RowSelection::from(vec![+ // Row 4 is the last row of group 0; rows 5-6 begin group 1.+ for selection in [+ RowSelection::from(vec![RowSelector::skip(4),RowSelector::select(3),RowSelector::skip(3),- ]))),- expected_rows: 3,+ ]),+ RowSelection::from_boolean_buffer(BooleanBuffer::from(vec![+ false, false, false, false, true, true, true, false, false, false,+ ])),+ ] {+ let parquet_metrics = TestFull {+ access_plan: None,+ row_selection: Some(ParquetRowSelection::new(selection)),+ expected_rows: 3,+ expected_output: Some(&[+ "+------+------------+",+ "| utf8 | large_utf8 |",+ "+------+------------+",+ "| | |",+ "| e | e |",+ "| f | f |",+ "+------+------------+",+ ]),+ predicate: None,+ }+ .run()+ .await+ .unwrap();++ let bytes_scanned = metric_value(&parquet_metrics, "bytes_scanned").unwrap();+ assert_ne!(bytes_scanned, 0, "metrics : {parquet_metrics:#?}");+ }+}++#[tokio::test]+async fn bitmap_row_selection_extension_with_predicate() {+ // Pruning removes row group 0, leaving the bitmap-selected rows in group 1.+ let parquet_metrics = TestFull {+ access_plan: None,+ row_selection: Some(ParquetRowSelection::new(RowSelection::from_boolean_buffer(+ BooleanBuffer::from(vec![+ false, false, true, false, true, true, true, false, false, false,+ ]),+ ))),+ expected_rows: 2,expected_output: Some(&["+------+------------+","| utf8 | large_utf8 |","+------+------------+",- "| | |","| e | e |","| f | f |","+------+------------+",]),- predicate: None,+ predicate: Some(col("utf8").eq(lit("e"))),}.run().await.unwrap();- let bytes_scanned = metric_value(&parquet_metrics, "bytes_scanned").unwrap();- assert_ne!(bytes_scanned, 0, "metrics : {parquet_metrics:#?}");+ // Verify that statistics pruned row group 0.+ let row_groups_pruned_statistics = parquet_metrics+ .sum_by_name("row_groups_pruned_statistics")+ .unwrap();+ if let MetricValue::PruningMetrics {+ pruning_metrics, ..+ } = row_groups_pruned_statistics+ {+ assert_eq!(+ pruning_metrics.pruned(),+ 1,+ "metrics : {parquet_metrics:#?}"+ );+ } else {+ unreachable!("metrics `row_groups_pruned_statistics` should exist")+ }}#[tokio::test]@@ -316,9 +366,9 @@ async fn mixed_bitmap_and_selector_selections() {// Each representation must keep coordinates local to its row group.TestFull {access_plan: Some(ParquetAccessPlan::new(vec![- RowGroupAccess::Selection(RowSelection::from(- arrow::buffer::BooleanBuffer::from(vec![true, false, true, false, false]),- )),+ RowGroupAccess::Selection(RowSelection::from(BooleanBuffer::from(vec![+ true, false, true, false, false,+ ]))),RowGroupAccess::Selection(select_two_rows()),])),row_selection: None,diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rsindex b72612134d..961a91c0b4 100644--- a/datafusion/datasource-parquet/src/access_plan.rs+++ b/datafusion/datasource-parquet/src/access_plan.rs@@ -15,6 +15,7 @@// specific language governing permissions and limitations// under the License.+use arrow::array::BooleanBufferBuilder;use arrow::datatypes::Schema;use datafusion_common::{Result, assert_eq_or_internal_err, exec_err};use datafusion_physical_expr::expressions::Column;@@ -297,6 +298,54 @@ impl ParquetAccessPlan {pub fn try_new_from_overall_row_selection(selection: RowSelection,row_group_meta_data: &[RowGroupMetaData],+ ) -> Result<Self> {+ if selection.as_mask().is_some() {+ Self::try_new_from_overall_row_selection_mask(selection, row_group_meta_data)+ } else {+ Self::try_new_from_overall_row_selection_selectors(+ selection,+ row_group_meta_data,+ )+ }+ }++ fn try_new_from_overall_row_selection_mask(+ mut selection: RowSelection,+ row_group_meta_data: &[RowGroupMetaData],+ ) -> Result<Self> {+ let selection_rows = selection.total_row_count();+ let file_rows = row_group_meta_data+ .iter()+ .map(|rg| rg.num_rows() as usize)+ .sum::<usize>();+ if selection_rows != file_rows {+ return exec_err!(+ "Invalid Parquet RowSelection. File has {file_rows} rows, \+ but selection specifies {selection_rows} rows."+ );+ }++ // For masks, split_off uses bitmap slices without materializing+ // selectors. Use row_count() to cache each partial group's count+ // for later preparation.+ let row_groups = row_group_meta_data+ .iter()+ .map(|rg| {+ let row_count = rg.num_rows() as usize;+ let group_selection = selection.split_off(row_count);+ match group_selection.row_count() {+ 0 => RowGroupAccess::Skip,+ selected if selected == row_count => RowGroupAccess::Scan,+ _ => RowGroupAccess::Selection(group_selection),+ }+ })+ .collect();+ Ok(Self::new(row_groups))+ }++ fn try_new_from_overall_row_selection_selectors(+ selection: RowSelection,+ row_group_meta_data: &[RowGroupMetaData],) -> Result<Self> {// Keep this as a single pass over the selector stream rather than// repeatedly calling `RowSelection::split_off` per row group. The@@ -392,6 +441,22 @@ impl ParquetAccessPlan {RowGroupAccess::Skip => RowGroupAccess::Skip,RowGroupAccess::Scan => RowGroupAccess::Selection(selection),RowGroupAccess::Selection(existing_selection) => {+ // Parquet preserves bitmap backing only when both operands+ // are masks. Promote selector-backed page pruning to retain+ // an external index's bitmap and use a bitwise intersection.+ // Revisit this conversion once Parquet optimizes mixed-backed+ // intersections: https://github.com/apache/arrow-rs/issues/10423+ let selection = if existing_selection.as_mask().is_some()+ && selection.as_mask().is_none()+ {+ let mut mask = BooleanBufferBuilder::new(selection.total_row_count());+ for selector in selection.iter() {+ mask.append_n(selector.row_count, !selector.skip);+ }+ RowSelection::from(mask.finish())+ } else {+ selection+ };RowGroupAccess::Selection(existing_selection.intersection(&selection))}}@@ -838,6 +903,7 @@ impl PreparedAccessPlan {#[cfg(test)]mod test {use super::*;+ use arrow::buffer::BooleanBuffer;use datafusion_common::assert_contains;use parquet::basic::LogicalType;use parquet::file::metadata::ColumnChunkMetaData;@@ -862,7 +928,7 @@ mod test {// Check both input representations retain the same conversion behavior.let selectors =RowSelection::from(vec![RowSelector::skip(10), RowSelector::select(20)]);- let bitmap = RowSelection::from(arrow::buffer::BooleanBuffer::from(+ let bitmap = RowSelection::from(BooleanBuffer::from((0..30).map(|i| i >= 10).collect::<Vec<_>>(),));for selection in [selectors, bitmap] {@@ -986,6 +1052,110 @@ mod test {}}+ #[test]+ fn test_scan_selection_preserves_mask_backing() {+ let mask = BooleanBuffer::from(vec![+ true, true, false, false, true, true, false, false, true, true,+ ]);+ let selectors =+ RowSelection::from(vec![RowSelector::select(5), RowSelector::skip(5)]);+ // Both selector-backed page pruning and bitmap intersections retain+ // the existing mask, including when no rows survive.+ for incoming in [+ selectors,+ RowSelection::from(BooleanBuffer::from(vec![+ true, true, true, true, true, false, false, false, false, false,+ ])),+ RowSelection::from(vec![RowSelector::skip(10)]),+ ] {+ let empty = incoming.row_count() == 0;+ let mut plan = ParquetAccessPlan::new(vec![RowGroupAccess::Selection(+ RowSelection::from(mask.clone()),+ )]);+ plan.scan_selection(0, incoming);+ let RowGroupAccess::Selection(selection) = &plan.inner()[0] else {+ panic!("expected selection");+ };+ let expected = if empty {+ BooleanBuffer::new_unset(10)+ } else {+ BooleanBuffer::from(vec![+ true, true, false, false, true, false, false, false, false, false,+ ])+ };+ assert_eq!(selection.as_mask(), Some(&expected));+ }+ }++ #[test]+ fn test_scan_selection_preserves_selector_backing() {+ let mut plan = ParquetAccessPlan::new(vec![RowGroupAccess::Selection(+ RowSelection::from(vec![RowSelector::select(6), RowSelector::skip(4)]),+ )]);+ plan.scan_selection(+ 0,+ RowSelection::from(vec![+ RowSelector::skip(2),+ RowSelector::select(5),+ RowSelector::skip(3),+ ]),+ );+ let RowGroupAccess::Selection(selection) = &plan.inner()[0] else {+ panic!("expected selection");+ };+ assert!(selection.as_mask().is_none());+ assert_eq!(+ selection,+ &RowSelection::from(vec![+ RowSelector::skip(2),+ RowSelector::select(4),+ RowSelector::skip(4),+ ])+ );+ }++ #[test]+ fn test_new_from_overall_mask_preserves_bitmap_backing() {+ // Include all-selected, all-skipped, and fragmented groups. Start at+ // a non-byte-aligned offset to exercise slicing an existing bitmap.+ let mut bits = vec![false; 3];+ bits.extend(vec![true; 10]);+ bits.extend(vec![false; 20]);+ bits.extend((0..30).map(|i| i % 2 == 1));+ bits.extend(vec![true; 40]);+ let mask = BooleanBuffer::from(bits).slice(3, 100);+ for cache_row_count in [false, true] {+ let selection = RowSelection::from(mask.clone());+ if cache_row_count {+ // Exercise split_off's propagation of an already cached count.+ assert_eq!(selection.row_count(), 65);+ }+ let plan = ParquetAccessPlan::try_new_from_overall_row_selection(+ selection,+ &ROW_GROUP_METADATA,+ )+ .unwrap();+ assert_eq!(plan.inner()[0], RowGroupAccess::Scan);+ assert_eq!(plan.inner()[1], RowGroupAccess::Skip);+ assert_eq!(plan.inner()[3], RowGroupAccess::Scan);+ let RowGroupAccess::Selection(selection) = &plan.inner()[2] else {+ panic!("expected selection");+ };+ assert_eq!(selection.row_count(), 15);+ let group_mask = mask.slice(30, 30);+ assert!(selection.as_mask().unwrap().ptr_eq(&group_mask));++ // The local selection must also survive preparation and reversal.+ let prepared = plan.prepare(&ROW_GROUP_METADATA).unwrap().reverse();+ assert_eq!(prepared.row_group_indexes(), vec![3, 2, 0]);+ assert!(prepared.row_groups[0].selection.selection().is_none());+ assert!(prepared.row_groups[2].selection.selection().is_none());+ let selection = prepared.row_groups[1].selection.selection().unwrap();+ assert_eq!(selection.row_count(), 15);+ assert!(selection.as_mask().unwrap().ptr_eq(&group_mask));+ }+ }+#[test]fn test_new_from_overall_row_selection() {let row_selection = RowSelection::from(vec![@@ -1022,19 +1192,26 @@ mod test {#[test]fn test_new_from_overall_row_selection_invalid_row_count() {- let row_selection = RowSelection::from(vec![RowSelector::select(99)]);-- let err = ParquetAccessPlan::try_new_from_overall_row_selection(- row_selection,- &ROW_GROUP_METADATA,- )- .unwrap_err()- .to_string();-- assert_contains!(- err,- "Invalid Parquet RowSelection. File has 100 rows, but selection specifies 99 rows"- );+ for selection_rows in [99, 101] {+ for selection in [+ RowSelection::from(vec![RowSelector::select(selection_rows)]),+ RowSelection::from(BooleanBuffer::new_set(selection_rows)),+ ] {+ let err = ParquetAccessPlan::try_new_from_overall_row_selection(+ selection,+ &ROW_GROUP_METADATA,+ )+ .unwrap_err()+ .to_string();+ assert_contains!(+ err,+ format!(+ "Invalid Parquet RowSelection. File has 100 rows, \+ but selection specifies {selection_rows} rows"+ )+ );+ }+ }}#[test]diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rsindex ccd9e098df..3cd9e795c1 100644--- a/datafusion/datasource-parquet/src/opener/mod.rs+++ b/datafusion/datasource-parquet/src/opener/mod.rs@@ -2144,26 +2144,32 @@ async fn load_page_index<T: AsyncFileReader>(mod test {use super::*;use super::{ConstantColumns, ParquetMorselizer, constant_columns_from_stats};- use crate::metadata::DFParquetMetadata;+ use crate::metadata::{CachedParquetMetaData, DFParquetMetadata};+ use crate::source::ParquetSource;use crate::{CachedParquetFileReaderFactory, DefaultParquetFileReaderFactory,ParquetFileReaderFactory, ParquetFileSchemaProvider, ParquetRowSelection,RowGroupAccess,};- use arrow::array::{AsArray, RecordBatch, record_batch};- use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+ use arrow::array::{Array, AsArray, RecordBatch, record_batch};+ use arrow::buffer::BooleanBuffer;+ use arrow::datatypes::{DataType, Field, Int32Type, Schema, SchemaRef};use bytes::{BufMut, BytesMut};use datafusion_common::{ColumnStatistics, ScalarValue, Statistics, assert_contains, internal_err,stats::Precision,};+ use datafusion_datasource::file::FileSource;+ use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_datasource::morsel::{Morsel, Morselizer};use datafusion_datasource::{PartitionedFile, TableSchema, TableSchemaBuilder};use datafusion_execution::cache::cache_manager::{CachedFileMetadataEntry, FileMetadataCache,};use datafusion_execution::cache::default_cache::DefaultCache;+ use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_expr::{Expr, col, lit};+ use datafusion_functions::core::expr_fn::get_field;use datafusion_physical_expr::{PhysicalExpr,expressions::{Column, DynamicFilterPhysicalExpr, Literal},@@ -2173,15 +2179,23 @@ mod test {use datafusion_physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, replace_columns_with_literals,};- use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;+ use datafusion_physical_plan::metrics::{ExecutionPlanMetricsSet, MetricValue};use datafusion_pruning::MAX_IN_LIST_SIZE;use futures::stream::BoxStream;use futures::{StreamExt, TryStreamExt};use object_store::{ObjectStore, ObjectStoreExt, memory::InMemory, path::Path};+ use parquet::arrow::arrow_reader::{RowSelection, RowSelector};use parquet::arrow::{ArrowSchemaConverter, ArrowWriter, parquet_to_arrow_schema};use parquet::basic::ConvertedType;- use parquet::file::metadata::{ColumnChunkMetaData, FileMetaData, ParquetMetaData};- use parquet::file::properties::{EnabledStatistics, WriterProperties};+ use parquet::data_type::{ByteArray, ByteArrayType, Int32Type as ParquetInt32Type};+ #[cfg(feature = "parquet_encryption")]+ use parquet::encryption::encrypt::FileEncryptionProperties;+ use parquet::file::metadata::{+ ColumnChunkMetaData, FileMetaData, KeyValue, ParquetMetaData,+ };+ use parquet::file::properties::{EnabledStatistics, WriterProperties, WriterVersion};+ use parquet::file::writer::SerializedFileWriter;+ use parquet::schema::parser::parse_message_type;use parquet::schema::types::{SchemaDescPtr, SchemaDescriptor};use std::collections::VecDeque;use std::sync::Arc;@@ -2217,8 +2231,6 @@ mod test {#[test]fn create_initial_plan_from_parquet_row_selection_extension() {- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};-let mut extensions = datafusion_datasource::FileExtensions::new();extensions.insert(ParquetRowSelection::new(RowSelection::from(vec![RowSelector::select(10),@@ -2242,8 +2254,6 @@ mod test {#[test]fn create_initial_plan_rejects_multiple_access_extensions() {- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};-let mut extensions = datafusion_datasource::FileExtensions::new();extensions.insert(ParquetAccessPlan::new_all(3));extensions.insert(ParquetRowSelection::new(RowSelection::from(vec![@@ -2370,7 +2380,6 @@ mod test {plan: ParquetAccessPlan,projection: Option<&[usize]>,) -> bool {- use crate::RowGroupAccessPlanFilter;let arrow_schema: SchemaRef = Arc::new(parquet_to_arrow_schema(metadata.file_metadata().schema_descr(), None).unwrap(),@@ -2779,7 +2788,6 @@ mod test {async fn collect_int32_values(mut stream: BoxStream<'static, Result<RecordBatch>>,) -> Vec<i32> {- use arrow::array::Array;let mut values = vec![];while let Some(Ok(batch)) = stream.next().await {let array = batch@@ -2818,8 +2826,6 @@ mod test {batch: RecordBatch,logical_file_schema: &SchemaRef,) -> (usize, Statistics) {- use parquet::file::metadata::ParquetMetaDataReader;-let mut out = BytesMut::new().writer();{let mut writer =@@ -2864,7 +2870,6 @@ mod test {}fn counter_metric_value(metrics: &ExecutionPlanMetricsSet, name: &str) -> usize {- use datafusion_physical_plan::metrics::MetricValue;metrics.clone_inner().sum_by_name(name)@@ -3380,7 +3385,6 @@ mod test {/// row-filtered" — both yield zero rows — so the tests around/// constant-column substitution assert on this metric instead.fn pruned_row_groups_statistics(metrics: &ExecutionPlanMetricsSet) -> usize {- use datafusion_physical_plan::metrics::MetricValue;metrics.clone_inner().iter()@@ -3917,10 +3921,6 @@ mod test {extra_field: bool,props: WriterProperties,) -> (PartitionedFile, SchemaRef) {- use parquet::data_type::{ByteArray, ByteArrayType, Int32Type};- use parquet::file::writer::SerializedFileWriter;- use parquet::schema::parser::parse_message_type;-let extra = if extra_field {"required int32 extra;"} else {@@ -3943,7 +3943,7 @@ mod test {let mut row_group = writer.next_row_group().unwrap();let mut column = row_group.next_column().unwrap().unwrap();column- .typed::<Int32Type>()+ .typed::<ParquetInt32Type>().write_batch(&[1, 2, 3], None, None).unwrap();column.close().unwrap();@@ -3965,7 +3965,7 @@ mod test {if extra_field {let mut column = row_group.next_column().unwrap().unwrap();column- .typed::<Int32Type>()+ .typed::<ParquetInt32Type>().write_batch(&[10, 20, 30], None, None).unwrap();column.close().unwrap();@@ -3981,7 +3981,6 @@ mod test {}fn invalid_arrow_hint() -> parquet::file::properties::WriterPropertiesBuilder {- use parquet::file::metadata::KeyValue;WriterProperties::builder().set_key_value_metadata(Some(vec![KeyValue::new("ARROW:schema".to_string(),@@ -3994,12 +3993,6 @@ mod test {#[tokio::test]async fn test_schema_provider_lazy_nested_scans() {- use crate::source::ParquetSource;- use datafusion_datasource::file::FileSource;- use datafusion_datasource::file_scan_config::FileScanConfigBuilder;- use datafusion_execution::object_store::ObjectStoreUrl;- use datafusion_functions::core::expr_fn::get_field;-let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());let (file1, schema) = write_enum_file(store.as_ref(),@@ -4064,8 +4057,6 @@ mod test {#[tokio::test]async fn test_schema_provider_preserves_metadata_and_precedence() {- use crate::metadata::{CachedParquetMetaData, DFParquetMetadata};-let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());let (file, schema) = write_enum_file(store.as_ref(),@@ -4187,9 +4178,6 @@ mod test {#[cfg(feature = "parquet_encryption")]#[tokio::test]async fn test_schema_provider_encrypted_scan() {- use datafusion_physical_plan::metrics::MetricValue;- use parquet::encryption::encrypt::FileEncryptionProperties;-let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());let footer_key = b"0123456789012345".to_vec();let column_key = b"1234567890123450".to_vec();@@ -4259,8 +4247,6 @@ mod test {#[tokio::test]async fn test_reverse_scan_row_groups() {- use parquet::file::properties::WriterProperties;-let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;// Create multiple batches to ensure multiple row groups@@ -4357,9 +4343,108 @@ mod test {}#[tokio::test]- async fn test_reverse_scan_with_row_selection() {- use parquet::file::properties::WriterProperties;+ async fn test_dynamic_pruning_preserves_external_row_selection() {+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ // After RG0, tighten the dynamic filter to a < 50. RG1 can then be+ // pruned, but RG2 survives and must retain its external selection.+ let batch = record_batch!((+ "a",+ Int32,+ vec![0, 1, 2, 3, 100, 101, 102, 103, 10, 11, 12, 13]+ ))+ .unwrap();+ let schema = batch.schema();+ let data_len = write_parquet_batches(+ Arc::clone(&store),+ "test.parquet",+ vec![batch],+ Some(+ WriterProperties::builder()+ .set_max_row_group_row_count(Some(4))+ .build(),+ ),+ )+ .await;+ let file = PartitionedFile::new("test.parquet".to_string(), data_len as u64);++ for selection in [+ None,+ Some(RowSelection::from(vec![+ RowSelector::select(1),+ RowSelector::skip(1),+ RowSelector::select(1),+ RowSelector::skip(1),+ ])),+ Some(RowSelection::from(BooleanBuffer::from(vec![+ true, false, true, false,+ ]))),+ ] {+ let has_selection = selection.is_some();+ let mut file = file.clone();+ if let Some(selection) = selection {+ file = file.with_extension(ParquetAccessPlan::new(vec![+ RowGroupAccess::Scan,+ RowGroupAccess::Scan,+ RowGroupAccess::Selection(selection),+ ]));+ }+ let dynamic = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("a", 0))],+ logical2physical(&col("a").lt(lit(200i32)), &schema),+ ));+ let metrics = ExecutionPlanMetricsSet::new();+ let opener = ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_schema(Arc::clone(&schema))+ .with_projection_indices(&[0])+ .with_predicate(Arc::clone(&dynamic) as Arc<dyn PhysicalExpr>)+ .with_row_group_stats_pruning(true)+ .with_preserve_order(true)+ .with_metrics(metrics.clone())+ .build();+ let mut stream = open_file(&opener, file).await.unwrap();+ let first = stream.next().await.unwrap().unwrap();+ assert_eq!(+ first+ .column(0)+ .as_primitive::<Int32Type>()+ .values()+ .as_ref(),+ &[0, 1, 2, 3]+ );+ dynamic+ .update(logical2physical(&col("a").lt(lit(50i32)), &schema))+ .unwrap();++ // Apply the final predicate as the consumer would. Runtime+ // pruning is best-effort, so RG1 may still be read when the+ // selection disables it. RG2's unselected rows also satisfy+ // this predicate and must never reach the consumer.+ let remaining: Vec<i32> = collect_int32_values(stream)+ .await+ .into_iter()+ .filter(|&value| value < 50)+ .collect();+ if has_selection {+ // Do not assert that runtime pruning is disabled: enabling it+ // later must preserve these rows through decoder rebuilds.+ // Replacing RG2's selection with None would also return 11/13,+ // which satisfy the predicate but not the external selection.+ assert_eq!(remaining, vec![10, 12]);+ } else {+ assert_eq!(remaining, vec![10, 11, 12, 13]);+ // Control case: prove the late update exercises runtime+ // pruning rather than only the per-row filter.+ assert_eq!(+ counter_metric_value(&metrics, "row_groups_pruned_dynamic_filter"),+ 1+ );+ }+ }+ }+ #[tokio::test]+ async fn test_reverse_scan_with_row_selection() {let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;// Create 3 batches with DIFFERENT selection patterns@@ -4387,9 +4472,6 @@ mod test {let schema = batch1.schema();- use crate::ParquetAccessPlan;- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};-let mut access_plan = ParquetAccessPlan::new_all(3);// Row group 0: skip first 2, select last 2 (should get: 3, 4)access_plan.scan_selection(@@ -4450,8 +4532,6 @@ mod test {#[tokio::test]async fn test_reverse_scan_with_non_contiguous_row_groups() {- use parquet::file::properties::WriterProperties;-let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;// Create 4 batches (4 row groups)@@ -4474,9 +4554,6 @@ mod test {let schema = batch0.schema();- use crate::ParquetAccessPlan;- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};-// KEY: Skip RG1 (non-contiguous!)// Only scan row groups: [0, 2, 3]let mut access_plan = ParquetAccessPlan::new(vec![@@ -4553,8 +4630,6 @@ mod test {/// since neither pushdown nor row-group pruning is active.#[tokio::test]async fn test_page_pruning_predicate_respects_enable_page_index() {- use parquet::file::properties::WriterProperties;-let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;// 100 rows with values 1..=100, written as a single row group with 10 rows per page@@ -4697,8 +4772,6 @@ mod test {#[test]fn should_load_page_index_with_row_selection() {- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};-let mut plan = ParquetAccessPlan::new_all(1);plan.scan_selection(0,@@ -4770,9 +4843,7 @@ mod test {}).collect::<Vec<_>>(),);- let mask = RowSelection::from_boolean_buffer(- arrow::buffer::BooleanBuffer::from(bits),- );+ let mask = RowSelection::from_boolean_buffer(BooleanBuffer::from(bits));for selection in [selectors, mask] {let plan =ParquetAccessPlan::new(vec![RowGroupAccess::Selection(selection)]);@@ -4786,8 +4857,6 @@ mod test {#[test]fn should_load_page_index_with_row_selection_checks_projection() {- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};-let metadata = page_index_metadata(&[("a", false), ("b", true)], 1);let plan = ParquetAccessPlan::new(vec![RowGroupAccess::Selection(RowSelection::from(vec![RowSelector::skip(9), RowSelector::select(1)]),@@ -4927,12 +4996,6 @@ mod test {#[tokio::test]async fn test_page_index_with_external_row_selection() {- use arrow::datatypes::Int32Type;- use parquet::arrow::arrow_reader::{RowSelection, RowSelector};- use parquet::file::properties::{- EnabledStatistics, WriterProperties, WriterVersion,- };-let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;let values: Vec<i32> = (0..10_000).collect();let output_values: Vec<i32> = (10_000..20_000).collect();@@ -5082,8 +5145,6 @@ mod test {#[tokio::test]async fn test_page_index_skipped_when_row_groups_fully_matched() {- use parquet::file::properties::WriterProperties;-let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;let values: Vec<i32> = (1..=100).collect();let batch = record_batch!((@@ -5129,8 +5190,6 @@ mod test {#[tokio::test]async fn test_page_index_skipped_with_cached_reader_factory() {- use parquet::file::properties::WriterProperties;-let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;let metadata_cache: Arc<FileMetadataCache> =Arc::new(DefaultCache::<Path, CachedFileMetadataEntry>::new(@@ -5197,8 +5256,6 @@ mod test {#[tokio::test]async fn test_page_index_loaded_when_not_fully_matched() {- use parquet::file::properties::WriterProperties;-let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;let values: Vec<i32> = (1..=100).collect();let batch = record_batch!((@@ -5245,8 +5302,6 @@ mod test {async fn fully_matched_split_test_file(store: Arc<dyn ObjectStore>,) -> (SchemaRef, PartitionedFile) {- use parquet::file::properties::WriterProperties;-let batch0 =record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();let batch1 =@@ -5274,6 +5329,72 @@ mod test {(schema, file)}+ #[tokio::test]+ async fn test_filter_toggles_preserve_external_row_selection() {+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;+ // For a > 5, the groups are fully matched, partially matched, then+ // fully matched. Both boundaries must rebuild the decoder to toggle+ // its row filter while preserving the remaining external selections.+ let batch = record_batch!((+ "a",+ Int32,+ vec![10, 11, 12, 13, 0, 20, 21, 22, 30, 31, 32, 33]+ ))+ .unwrap();+ let schema = batch.schema();+ let data_len = write_parquet_batches(+ Arc::clone(&store),+ "test.parquet",+ vec![batch],+ Some(+ WriterProperties::builder()+ .set_max_row_group_row_count(Some(4))+ .build(),+ ),+ )+ .await;+ let file = PartitionedFile::new("test.parquet".to_string(), data_len as u64);++ for selection in [+ RowSelection::from(vec![+ RowSelector::select(1),+ RowSelector::skip(1),+ RowSelector::select(1),+ RowSelector::skip(1),+ ]),+ RowSelection::from(BooleanBuffer::from(vec![true, false, true, false])),+ ] {+ let file = file.clone().with_extension(ParquetAccessPlan::new(vec![+ RowGroupAccess::Selection(selection.clone()),+ RowGroupAccess::Selection(selection.clone()),+ RowGroupAccess::Selection(selection),+ ]));+ let metrics = ExecutionPlanMetricsSet::new();+ let opener = ParquetMorselizerBuilder::new()+ .with_store(Arc::clone(&store))+ .with_schema(Arc::clone(&schema))+ .with_projection_indices(&[0])+ .with_predicate(logical2physical(&col("a").gt(lit(5i32)), &schema))+ .with_pushdown_filters(true)+ .with_row_group_stats_pruning(true)+ .with_preserve_order(true)+ .with_metrics(metrics.clone())+ .build();++ let values =+ collect_int32_values(open_file(&opener, file).await.unwrap()).await;+ // 0 tests that the filter was enabled; unselected values that+ // also pass a > 5 detect lost selections at either boundary.+ assert_eq!(values, vec![10, 12, 21, 30, 32]);+ // One skip at initialization and one after re-enabling the filter+ // for RG1 prove that the fully-matched optimization was exercised.+ assert_eq!(+ counter_metric_value(&metrics, "row_filter_skipped_fully_matched"),+ 2+ );+ }+ }+#[tokio::test]async fn test_fully_matched_runs_respect_global_limit() {let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;@@ -5358,7 +5479,7 @@ mod test {/// (e.g. `row_number`) plumbed through `TableSchema`/`ParquetOpener`.mod virtual_columns {use super::*;- use arrow::array::{Array, Int64Array, StringArray};+ use arrow::array::{Int64Array, StringArray};use arrow::datatypes::FieldRef;use datafusion_common::config::ConfigOptions;use datafusion_expr::ScalarUDF;