Skip to content

Apache DataFusion SQL Query Engine

feat: derive per-column pruning guarantees from tuple IN lists (#25469)

Yash Bhisikar2717e25625ff
diff --git a/datafusion/functions/src/core/named_struct.rs b/datafusion/functions/src/core/named_struct.rs
index 0d9e133ada..00b4297026 100644
--- a/datafusion/functions/src/core/named_struct.rs
+++ b/datafusion/functions/src/core/named_struct.rs
@@ -25,6 +25,7 @@ use datafusion_expr::{
};
use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
use datafusion_macros::user_doc;
+use std::collections::HashSet;
use std::sync::Arc;
#[user_doc(
@@ -191,9 +192,14 @@ impl ScalarUDFImpl for NamedStructFunc {
}
let mut fields = Vec::with_capacity(literal_args.len() / 2);
+ let mut names = HashSet::new();
for (i, chunk) in literal_args.chunks(2).enumerate() {
match chunk {
[Some(ScalarValue::Utf8(Some(name))), _] => {
+ // Field access resolves only the first occurrence of a name.
+ if !names.insert(name) {
+ return None;
+ }
fields.push((
vec![ScalarValue::Utf8(Some(name.clone()))],
i * 2 + 1, // index of the value argument
diff --git a/datafusion/functions/src/core/struct.rs b/datafusion/functions/src/core/struct.rs
index 164b9d2032..d689f3aac4 100644
--- a/datafusion/functions/src/core/struct.rs
+++ b/datafusion/functions/src/core/struct.rs
@@ -15,11 +15,13 @@
// specific language governing permissions and limitations
// under the License.
+use super::getfield::GetFieldFunc;
use arrow::array::StructArray;
use arrow::datatypes::{DataType, Field, FieldRef};
-use datafusion_common::{Result, exec_err, internal_err};
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
use datafusion_expr::{
- ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,
+ StructFieldMapping,
};
use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
use datafusion_macros::user_doc;
@@ -150,4 +152,16 @@ impl ScalarUDFImpl for StructFunc {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
+
+ fn struct_field_mapping(
+ &self,
+ literal_args: &[Option<ScalarValue>],
+ ) -> Option<StructFieldMapping> {
+ Some(StructFieldMapping {
+ field_accessor: Arc::new(ScalarUDF::from(GetFieldFunc::new())),
+ fields: (0..literal_args.len())
+ .map(|i| (vec![ScalarValue::Utf8(Some(format!("c{i}")))], i))
+ .collect(),
+ })
+ }
}
diff --git a/datafusion/physical-expr/src/utils/guarantee.rs b/datafusion/physical-expr/src/utils/guarantee.rs
index 4224943eb7..556cdf0625 100644
--- a/datafusion/physical-expr/src/utils/guarantee.rs
+++ b/datafusion/physical-expr/src/utils/guarantee.rs
@@ -20,8 +20,9 @@
use crate::utils::split_disjunction;
use crate::{PhysicalExpr, split_conjunction};
+use arrow::array::{Array, RecordBatch};
use datafusion_common::{Column, HashMap, ScalarValue};
-use datafusion_expr::Operator;
+use datafusion_expr::{Operator, Volatility};
use std::collections::HashSet;
use std::fmt::{self, Display, Formatter};
use std::sync::Arc;
@@ -135,6 +136,16 @@ impl LiteralGuarantee {
inlist.guarantee,
inlist.list.iter().map(|lit| lit.value()),
)
+ } else if let Some(projected) = project_struct_in_list(inlist) {
+ projected
+ .into_iter()
+ .fold(builder, |builder, (col, values)| {
+ builder.aggregate_multi_conjunct(
+ col,
+ Guarantee::In,
+ &values,
+ )
+ })
} else {
builder
}
@@ -310,11 +321,11 @@ impl<'a> GuaranteeBuilder<'a> {
/// * `AND (a != 1 OR a != 2 OR a != 3)`: a is not in (1, 2, or 3)
/// * `AND (a NOT IN (1,2,3))`: a is not in (1, 2, or 3)
#[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key
- fn aggregate_multi_conjunct(
+ fn aggregate_multi_conjunct<'b>(
mut self,
col: &'a crate::expressions::Column,
guarantee: Guarantee,
- new_values: impl IntoIterator<Item = &'a ScalarValue>,
+ new_values: impl IntoIterator<Item = &'b ScalarValue>,
) -> Self {
let key = (col, guarantee);
if let Some(index) = self.map.get(&key) {
@@ -377,6 +388,85 @@ impl<'a> GuaranteeBuilder<'a> {
}
}
+/// Project necessary per-column guarantees; the original predicate retains tuple correlation.
+fn project_struct_in_list(
+ inlist: &crate::expressions::InListExpr,
+) -> Option<Vec<(&crate::expressions::Column, Vec<ScalarValue>)>> {
+ if inlist.negated() || inlist.is_empty() {
+ return None;
+ }
+ let expr = inlist.expr().downcast_ref::<crate::ScalarFunctionExpr>()?;
+ let literal_args = expr
+ .args()
+ .iter()
+ .map(|arg| {
+ arg.downcast_ref::<crate::expressions::Literal>()
+ .map(|lit| lit.value().clone())
+ })
+ .collect::<Vec<_>>();
+ let mapping = expr.fun().struct_field_mapping(&literal_args)?;
+ if mapping.field_accessor.signature().volatility != Volatility::Immutable {
+ return None;
+ }
+ let tuples = inlist
+ .list()
+ .iter()
+ .map(|value| {
+ let literal = value.downcast_ref::<crate::expressions::Literal>()?;
+ let ScalarValue::Struct(array) = literal.value() else {
+ return None;
+ };
+ (array.len() == 1).then_some(literal.value())
+ })
+ .collect::<Option<Vec<_>>>()?;
+ // Null tuples cannot make a positive IN predicate true.
+ let tuples = ScalarValue::iter_to_array(
+ tuples.into_iter().filter(|tuple| !tuple.is_null()).cloned(),
+ )
+ .ok()?;
+ let batch = RecordBatch::try_from_iter([("tuple", tuples)]).ok()?;
+
+ let mut projected = Vec::new();
+ for (accessor_args, source_index) in mapping.fields {
+ let column = expr
+ .args()
+ .get(source_index)?
+ .downcast_ref::<crate::expressions::Column>()?;
+ let mut args: Vec<Arc<dyn PhysicalExpr>> =
+ vec![Arc::new(crate::expressions::Column::new("tuple", 0))];
+ args.extend(accessor_args.into_iter().map(crate::expressions::lit));
+ let accessor = crate::ScalarFunctionExpr::try_new(
+ Arc::clone(&mapping.field_accessor),
+ args,
+ batch.schema_ref(),
+ Arc::new(expr.config_options().clone()),
+ )
+ .ok()?;
+ let array = accessor
+ .evaluate(&batch)
+ .ok()?
+ .into_array_of_size(batch.num_rows())
+ .ok()?;
+ let mut values = Vec::new();
+ for index in 0..array.len() {
+ let value = ScalarValue::try_from_array(array.as_ref(), index).ok()?;
+ let value = match value {
+ ScalarValue::Dictionary(_, value) => *value,
+ value => value,
+ };
+ if value.is_null() {
+ values.clear();
+ break;
+ }
+ values.push(value);
+ }
+ if !values.is_empty() {
+ projected.push((column, values));
+ }
+ }
+ Some(projected)
+}
+
/// Represents a single `col [not]in literal` expression
struct ColOpLit<'a> {
col: &'a crate::expressions::Column,
@@ -442,7 +532,6 @@ impl<'a> ColInList<'a> {
///
/// Returns None otherwise
fn try_new(inlist: &'a crate::expressions::InListExpr) -> Option<Self> {
- // Only support single-column inlist currently, multi-column inlist is not supported
let col = inlist.expr().downcast_ref::<crate::expressions::Column>()?;
let literals = inlist
@@ -844,6 +933,125 @@ mod test {
);
}
+ #[test]
+ fn test_struct_inlist_guarantees() {
+ use crate::expressions::InListExpr;
+ use arrow::array::{ArrayRef, Int32Array, StringArray, StructArray};
+ use arrow::buffer::NullBuffer;
+
+ let make_expr = |strings: ArrayRef, nulls: Option<NullBuffer>, negated| {
+ let schema = Schema::new(vec![
+ Field::new("a", strings.data_type().clone(), true),
+ Field::new("b", DataType::Int32, true),
+ ]);
+ let expr = logical2physical(
+ &datafusion_functions::core::r#struct().call(vec![col("a"), col("b")]),
+ &schema,
+ );
+ let DataType::Struct(fields) = expr.data_type(&schema).unwrap() else {
+ unreachable!()
+ };
+ let values = Arc::new(StructArray::new(
+ fields,
+ vec![strings, Arc::new(Int32Array::from(vec![1, 2, 3]))],
+ nulls,
+ ));
+ Arc::new(
+ InListExpr::try_new_from_array(expr, values, negated, &schema).unwrap(),
+ ) as Arc<dyn PhysicalExpr>
+ };
+ let strings: ArrayRef = Arc::new(StringArray::from(vec!["foo", "foo", "bar"]));
+ assert_eq!(
+ LiteralGuarantee::analyze(&make_expr(Arc::clone(&strings), None, false)),
+ vec![
+ in_guarantee("a", ["foo", "bar"]),
+ in_guarantee("b", [1, 2, 3])
+ ]
+ );
+ assert!(
+ LiteralGuarantee::analyze(&make_expr(Arc::clone(&strings), None, true))
+ .is_empty()
+ );
+ assert!(
+ LiteralGuarantee::analyze(&make_expr(
+ Arc::clone(&strings),
+ Some(vec![false, false, false].into()),
+ false
+ ))
+ .is_empty()
+ );
+ assert_eq!(
+ LiteralGuarantee::analyze(&make_expr(
+ strings,
+ Some(vec![false, true, true].into()),
+ false
+ )),
+ vec![in_guarantee("a", ["foo", "bar"]), in_guarantee("b", [2, 3])]
+ );
+
+ let strings = StringArray::from(vec![Some("foo"), None, Some("bar")]);
+ let dictionary =
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+ for data_type in [DataType::Utf8, dictionary] {
+ let strings = arrow::compute::cast(&strings, &data_type).unwrap();
+ assert_eq!(
+ LiteralGuarantee::analyze(&make_expr(strings, None, false)),
+ vec![in_guarantee("b", [1, 2, 3])]
+ );
+ }
+ let strings = arrow::compute::cast(
+ &StringArray::from(vec!["foo", "foo", "bar"]),
+ &DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
+ )
+ .unwrap();
+ assert_eq!(
+ LiteralGuarantee::analyze(&make_expr(strings, None, false)),
+ vec![
+ in_guarantee("a", ["foo", "bar"]),
+ in_guarantee("b", [1, 2, 3])
+ ]
+ );
+
+ // Named fields map to nonconsecutive arguments, in a different column order.
+ let tuple = RecordBatch::try_from_iter([
+ ("right", Arc::new(Int32Array::from(vec![1])) as ArrayRef),
+ ("left", Arc::new(StringArray::from(vec!["foo"])) as ArrayRef),
+ ])
+ .unwrap();
+ let expr = datafusion_functions::core::named_struct().call(vec![
+ lit("right"),
+ col("b"),
+ lit("left"),
+ col("a"),
+ ]);
+ test_analyze(
+ expr.in_list(
+ vec![lit(ScalarValue::Struct(Arc::new(StructArray::from(tuple))))],
+ false,
+ ),
+ vec![in_guarantee("a", ["foo"]), in_guarantee("b", [1])],
+ );
+
+ let tuples = [(1, 10), (2, 20)]
+ .into_iter()
+ .map(|(left, right)| {
+ let tuple = RecordBatch::try_from_iter([
+ ("x", Arc::new(Int32Array::from(vec![left])) as ArrayRef),
+ ("x", Arc::new(Int32Array::from(vec![right])) as ArrayRef),
+ ])
+ .unwrap();
+ lit(ScalarValue::Struct(Arc::new(StructArray::from(tuple))))
+ })
+ .collect();
+ let expr = datafusion_functions::core::named_struct().call(vec![
+ lit("x"),
+ col("b"),
+ lit("x"),
+ col("c"),
+ ]);
+ test_analyze(expr.in_list(tuples, false), vec![]);
+ }
+
#[test]
fn test_inlist_conjunction() {
// b IN (1, 2, 3) AND b IN (2, 3, 4)
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
index 3ad551f571..edfef76846 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
@@ -346,24 +346,33 @@ COPY (
) TO 'test_files/scratch/push_down_filter_parquet/join_build.parquet'
STORED AS PARQUET;
+# Three row groups: exact matches, Bloom-absent values within the min/max ranges,
+# and crossed pairs that survive per-column pruning but fail the tuple filter.
statement ok
COPY (
SELECT * FROM (VALUES
('aa', 'ba', 1.0),
('ab', 'bb', 2.0),
- ('ac', 'bc', 3.0),
- ('ad', 'bd', 4.0)
+ ('aa', 'baz', 3.0),
+ ('ab', 'baz', 4.0),
+ ('aa', 'bb', 5.0),
+ ('ab', 'ba', 6.0)
) AS v(a, b, e)
) TO 'test_files/scratch/push_down_filter_parquet/join_probe.parquet'
-STORED AS PARQUET;
+STORED AS PARQUET
+OPTIONS (
+ 'format.max_row_group_size' '2',
+ 'format.bloom_filter_on_write' 'true',
+ 'format.bloom_filter_fpp' '0.000001'
+);
statement ok
-CREATE EXTERNAL TABLE join_build (a VARCHAR, b VARCHAR, c DOUBLE)
+CREATE EXTERNAL TABLE join_build (a VARCHAR NOT NULL, b VARCHAR NOT NULL, c DOUBLE)
STORED AS PARQUET
LOCATION 'test_files/scratch/push_down_filter_parquet/join_build.parquet';
statement ok
-CREATE EXTERNAL TABLE join_probe (a VARCHAR, b VARCHAR, e DOUBLE)
+CREATE EXTERNAL TABLE join_probe (a VARCHAR NOT NULL, b VARCHAR NOT NULL, e DOUBLE)
STORED AS PARQUET
LOCATION 'test_files/scratch/push_down_filter_parquet/join_probe.parquet';
@@ -389,7 +398,7 @@ FROM join_probe p INNER JOIN join_build AS build
Plan with Metrics
01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@3, b@4, c@2, e@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)]
-03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)]
+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[a in (aa, ab), b in (ba, bb)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=3 total → 3 matched, row_groups_pruned_bloom_filter=3 total → 2 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=23.61% (606/2.57 K)]
statement ok
reset datafusion.explain.analyze_categories;
@@ -874,7 +883,7 @@ ON lj_build.a = lj_probe.a AND lj_build.b = lj_probe.b;
Plan with Metrics
01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)]
-03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)]
+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[a in (aa, ab), b in (ba, bb)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)]
# LEFT SEMI JOIN: only matching build rows are returned; probe scan still
# receives the dynamic filter.
@@ -890,7 +899,7 @@ WHERE EXISTS (
Plan with Metrics
01)HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=4, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)]
-03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.11% (154/1.02 K)]
+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[a in (aa, ab), b in (ba, bb)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.11% (154/1.02 K)]
statement ok
reset datafusion.explain.analyze_categories;