Apache DataFusion SQL Query Engine
feat: derive per-column pruning guarantees from tuple IN lists (#25469)
Yash Bhisikar2717e25625ff
diff --git a/datafusion/functions/src/core/named_struct.rs b/datafusion/functions/src/core/named_struct.rsindex 0d9e133ada..00b4297026 100644--- a/datafusion/functions/src/core/named_struct.rs+++ b/datafusion/functions/src/core/named_struct.rs@@ -25,6 +25,7 @@ use datafusion_expr::{};use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};use datafusion_macros::user_doc;+use std::collections::HashSet;use std::sync::Arc;#[user_doc(@@ -191,9 +192,14 @@ impl ScalarUDFImpl for NamedStructFunc {}let mut fields = Vec::with_capacity(literal_args.len() / 2);+ let mut names = HashSet::new();for (i, chunk) in literal_args.chunks(2).enumerate() {match chunk {[Some(ScalarValue::Utf8(Some(name))), _] => {+ // Field access resolves only the first occurrence of a name.+ if !names.insert(name) {+ return None;+ }fields.push((vec![ScalarValue::Utf8(Some(name.clone()))],i * 2 + 1, // index of the value argumentdiff --git a/datafusion/functions/src/core/struct.rs b/datafusion/functions/src/core/struct.rsindex 164b9d2032..d689f3aac4 100644--- a/datafusion/functions/src/core/struct.rs+++ b/datafusion/functions/src/core/struct.rs@@ -15,11 +15,13 @@// specific language governing permissions and limitations// under the License.+use super::getfield::GetFieldFunc;use arrow::array::StructArray;use arrow::datatypes::{DataType, Field, FieldRef};-use datafusion_common::{Result, exec_err, internal_err};+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};use datafusion_expr::{- ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,+ StructFieldMapping,};use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};use datafusion_macros::user_doc;@@ -150,4 +152,16 @@ impl ScalarUDFImpl for StructFunc {fn documentation(&self) -> Option<&Documentation> {self.doc()}++ fn struct_field_mapping(+ &self,+ literal_args: &[Option<ScalarValue>],+ ) -> Option<StructFieldMapping> {+ Some(StructFieldMapping {+ field_accessor: Arc::new(ScalarUDF::from(GetFieldFunc::new())),+ fields: (0..literal_args.len())+ .map(|i| (vec![ScalarValue::Utf8(Some(format!("c{i}")))], i))+ .collect(),+ })+ }}diff --git a/datafusion/physical-expr/src/utils/guarantee.rs b/datafusion/physical-expr/src/utils/guarantee.rsindex 4224943eb7..556cdf0625 100644--- a/datafusion/physical-expr/src/utils/guarantee.rs+++ b/datafusion/physical-expr/src/utils/guarantee.rs@@ -20,8 +20,9 @@use crate::utils::split_disjunction;use crate::{PhysicalExpr, split_conjunction};+use arrow::array::{Array, RecordBatch};use datafusion_common::{Column, HashMap, ScalarValue};-use datafusion_expr::Operator;+use datafusion_expr::{Operator, Volatility};use std::collections::HashSet;use std::fmt::{self, Display, Formatter};use std::sync::Arc;@@ -135,6 +136,16 @@ impl LiteralGuarantee {inlist.guarantee,inlist.list.iter().map(|lit| lit.value()),)+ } else if let Some(projected) = project_struct_in_list(inlist) {+ projected+ .into_iter()+ .fold(builder, |builder, (col, values)| {+ builder.aggregate_multi_conjunct(+ col,+ Guarantee::In,+ &values,+ )+ })} else {builder}@@ -310,11 +321,11 @@ impl<'a> GuaranteeBuilder<'a> {/// * `AND (a != 1 OR a != 2 OR a != 3)`: a is not in (1, 2, or 3)/// * `AND (a NOT IN (1,2,3))`: a is not in (1, 2, or 3)#[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key- fn aggregate_multi_conjunct(+ fn aggregate_multi_conjunct<'b>(mut self,col: &'a crate::expressions::Column,guarantee: Guarantee,- new_values: impl IntoIterator<Item = &'a ScalarValue>,+ new_values: impl IntoIterator<Item = &'b ScalarValue>,) -> Self {let key = (col, guarantee);if let Some(index) = self.map.get(&key) {@@ -377,6 +388,85 @@ impl<'a> GuaranteeBuilder<'a> {}}+/// Project necessary per-column guarantees; the original predicate retains tuple correlation.+fn project_struct_in_list(+ inlist: &crate::expressions::InListExpr,+) -> Option<Vec<(&crate::expressions::Column, Vec<ScalarValue>)>> {+ if inlist.negated() || inlist.is_empty() {+ return None;+ }+ let expr = inlist.expr().downcast_ref::<crate::ScalarFunctionExpr>()?;+ let literal_args = expr+ .args()+ .iter()+ .map(|arg| {+ arg.downcast_ref::<crate::expressions::Literal>()+ .map(|lit| lit.value().clone())+ })+ .collect::<Vec<_>>();+ let mapping = expr.fun().struct_field_mapping(&literal_args)?;+ if mapping.field_accessor.signature().volatility != Volatility::Immutable {+ return None;+ }+ let tuples = inlist+ .list()+ .iter()+ .map(|value| {+ let literal = value.downcast_ref::<crate::expressions::Literal>()?;+ let ScalarValue::Struct(array) = literal.value() else {+ return None;+ };+ (array.len() == 1).then_some(literal.value())+ })+ .collect::<Option<Vec<_>>>()?;+ // Null tuples cannot make a positive IN predicate true.+ let tuples = ScalarValue::iter_to_array(+ tuples.into_iter().filter(|tuple| !tuple.is_null()).cloned(),+ )+ .ok()?;+ let batch = RecordBatch::try_from_iter([("tuple", tuples)]).ok()?;++ let mut projected = Vec::new();+ for (accessor_args, source_index) in mapping.fields {+ let column = expr+ .args()+ .get(source_index)?+ .downcast_ref::<crate::expressions::Column>()?;+ let mut args: Vec<Arc<dyn PhysicalExpr>> =+ vec![Arc::new(crate::expressions::Column::new("tuple", 0))];+ args.extend(accessor_args.into_iter().map(crate::expressions::lit));+ let accessor = crate::ScalarFunctionExpr::try_new(+ Arc::clone(&mapping.field_accessor),+ args,+ batch.schema_ref(),+ Arc::new(expr.config_options().clone()),+ )+ .ok()?;+ let array = accessor+ .evaluate(&batch)+ .ok()?+ .into_array_of_size(batch.num_rows())+ .ok()?;+ let mut values = Vec::new();+ for index in 0..array.len() {+ let value = ScalarValue::try_from_array(array.as_ref(), index).ok()?;+ let value = match value {+ ScalarValue::Dictionary(_, value) => *value,+ value => value,+ };+ if value.is_null() {+ values.clear();+ break;+ }+ values.push(value);+ }+ if !values.is_empty() {+ projected.push((column, values));+ }+ }+ Some(projected)+}+/// Represents a single `col [not]in literal` expressionstruct ColOpLit<'a> {col: &'a crate::expressions::Column,@@ -442,7 +532,6 @@ impl<'a> ColInList<'a> {////// Returns None otherwisefn try_new(inlist: &'a crate::expressions::InListExpr) -> Option<Self> {- // Only support single-column inlist currently, multi-column inlist is not supportedlet col = inlist.expr().downcast_ref::<crate::expressions::Column>()?;let literals = inlist@@ -844,6 +933,125 @@ mod test {);}+ #[test]+ fn test_struct_inlist_guarantees() {+ use crate::expressions::InListExpr;+ use arrow::array::{ArrayRef, Int32Array, StringArray, StructArray};+ use arrow::buffer::NullBuffer;++ let make_expr = |strings: ArrayRef, nulls: Option<NullBuffer>, negated| {+ let schema = Schema::new(vec![+ Field::new("a", strings.data_type().clone(), true),+ Field::new("b", DataType::Int32, true),+ ]);+ let expr = logical2physical(+ &datafusion_functions::core::r#struct().call(vec![col("a"), col("b")]),+ &schema,+ );+ let DataType::Struct(fields) = expr.data_type(&schema).unwrap() else {+ unreachable!()+ };+ let values = Arc::new(StructArray::new(+ fields,+ vec![strings, Arc::new(Int32Array::from(vec![1, 2, 3]))],+ nulls,+ ));+ Arc::new(+ InListExpr::try_new_from_array(expr, values, negated, &schema).unwrap(),+ ) as Arc<dyn PhysicalExpr>+ };+ let strings: ArrayRef = Arc::new(StringArray::from(vec!["foo", "foo", "bar"]));+ assert_eq!(+ LiteralGuarantee::analyze(&make_expr(Arc::clone(&strings), None, false)),+ vec![+ in_guarantee("a", ["foo", "bar"]),+ in_guarantee("b", [1, 2, 3])+ ]+ );+ assert!(+ LiteralGuarantee::analyze(&make_expr(Arc::clone(&strings), None, true))+ .is_empty()+ );+ assert!(+ LiteralGuarantee::analyze(&make_expr(+ Arc::clone(&strings),+ Some(vec![false, false, false].into()),+ false+ ))+ .is_empty()+ );+ assert_eq!(+ LiteralGuarantee::analyze(&make_expr(+ strings,+ Some(vec![false, true, true].into()),+ false+ )),+ vec![in_guarantee("a", ["foo", "bar"]), in_guarantee("b", [2, 3])]+ );++ let strings = StringArray::from(vec![Some("foo"), None, Some("bar")]);+ let dictionary =+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));+ for data_type in [DataType::Utf8, dictionary] {+ let strings = arrow::compute::cast(&strings, &data_type).unwrap();+ assert_eq!(+ LiteralGuarantee::analyze(&make_expr(strings, None, false)),+ vec![in_guarantee("b", [1, 2, 3])]+ );+ }+ let strings = arrow::compute::cast(+ &StringArray::from(vec!["foo", "foo", "bar"]),+ &DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),+ )+ .unwrap();+ assert_eq!(+ LiteralGuarantee::analyze(&make_expr(strings, None, false)),+ vec![+ in_guarantee("a", ["foo", "bar"]),+ in_guarantee("b", [1, 2, 3])+ ]+ );++ // Named fields map to nonconsecutive arguments, in a different column order.+ let tuple = RecordBatch::try_from_iter([+ ("right", Arc::new(Int32Array::from(vec![1])) as ArrayRef),+ ("left", Arc::new(StringArray::from(vec!["foo"])) as ArrayRef),+ ])+ .unwrap();+ let expr = datafusion_functions::core::named_struct().call(vec![+ lit("right"),+ col("b"),+ lit("left"),+ col("a"),+ ]);+ test_analyze(+ expr.in_list(+ vec![lit(ScalarValue::Struct(Arc::new(StructArray::from(tuple))))],+ false,+ ),+ vec![in_guarantee("a", ["foo"]), in_guarantee("b", [1])],+ );++ let tuples = [(1, 10), (2, 20)]+ .into_iter()+ .map(|(left, right)| {+ let tuple = RecordBatch::try_from_iter([+ ("x", Arc::new(Int32Array::from(vec![left])) as ArrayRef),+ ("x", Arc::new(Int32Array::from(vec![right])) as ArrayRef),+ ])+ .unwrap();+ lit(ScalarValue::Struct(Arc::new(StructArray::from(tuple))))+ })+ .collect();+ let expr = datafusion_functions::core::named_struct().call(vec![+ lit("x"),+ col("b"),+ lit("x"),+ col("c"),+ ]);+ test_analyze(expr.in_list(tuples, false), vec![]);+ }+#[test]fn test_inlist_conjunction() {// b IN (1, 2, 3) AND b IN (2, 3, 4)diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.sltindex 3ad551f571..edfef76846 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt@@ -346,24 +346,33 @@ COPY () TO 'test_files/scratch/push_down_filter_parquet/join_build.parquet'STORED AS PARQUET;+# Three row groups: exact matches, Bloom-absent values within the min/max ranges,+# and crossed pairs that survive per-column pruning but fail the tuple filter.statement okCOPY (SELECT * FROM (VALUES('aa', 'ba', 1.0),('ab', 'bb', 2.0),- ('ac', 'bc', 3.0),- ('ad', 'bd', 4.0)+ ('aa', 'baz', 3.0),+ ('ab', 'baz', 4.0),+ ('aa', 'bb', 5.0),+ ('ab', 'ba', 6.0)) AS v(a, b, e)) TO 'test_files/scratch/push_down_filter_parquet/join_probe.parquet'-STORED AS PARQUET;+STORED AS PARQUET+OPTIONS (+ 'format.max_row_group_size' '2',+ 'format.bloom_filter_on_write' 'true',+ 'format.bloom_filter_fpp' '0.000001'+);statement ok-CREATE EXTERNAL TABLE join_build (a VARCHAR, b VARCHAR, c DOUBLE)+CREATE EXTERNAL TABLE join_build (a VARCHAR NOT NULL, b VARCHAR NOT NULL, c DOUBLE)STORED AS PARQUETLOCATION 'test_files/scratch/push_down_filter_parquet/join_build.parquet';statement ok-CREATE EXTERNAL TABLE join_probe (a VARCHAR, b VARCHAR, e DOUBLE)+CREATE EXTERNAL TABLE join_probe (a VARCHAR NOT NULL, b VARCHAR NOT NULL, e DOUBLE)STORED AS PARQUETLOCATION 'test_files/scratch/push_down_filter_parquet/join_probe.parquet';@@ -389,7 +398,7 @@ FROM join_probe p INNER JOIN join_build AS buildPlan with Metrics01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@3, b@4, c@2, e@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)]-03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)]+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[a in (aa, ab), b in (ba, bb)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=3 total → 3 matched, row_groups_pruned_bloom_filter=3 total → 2 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=23.61% (606/2.57 K)]statement okreset datafusion.explain.analyze_categories;@@ -874,7 +883,7 @@ ON lj_build.a = lj_probe.a AND lj_build.b = lj_probe.b;Plan with Metrics01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)]-03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)]+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[a in (aa, ab), b in (ba, bb)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)]# LEFT SEMI JOIN: only matching build rows are returned; probe scan still# receives the dynamic filter.@@ -890,7 +899,7 @@ WHERE EXISTS (Plan with Metrics01)HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=4, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)]-03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.11% (154/1.02 K)]+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[a in (aa, ab), b in (ba, bb)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.11% (154/1.02 K)]statement okreset datafusion.explain.analyze_categories;