Skip to content

Apache DataFusion SQL Query Engine

Parquet: Push down supported list predicates (array_has/any/all) during decoding (#19545)

kosiew566bcde9e129
diff --git a/Cargo.lock b/Cargo.lock
index b0c26e27b4..ce09369b0a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2111,12 +2111,14 @@ dependencies = [
"async-trait",
"bytes",
"chrono",
+ "criterion",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions-aggregate-common",
+ "datafusion-functions-nested",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
@@ -2129,6 +2131,7 @@ dependencies = [
"object_store",
"parking_lot",
"parquet",
+ "tempfile",
"tokio",
]
diff --git a/datafusion/datasource-parquet/Cargo.toml b/datafusion/datasource-parquet/Cargo.toml
index a5f6f56ac6..b865422366 100644
--- a/datafusion/datasource-parquet/Cargo.toml
+++ b/datafusion/datasource-parquet/Cargo.toml
@@ -56,6 +56,9 @@ tokio = { workspace = true }
[dev-dependencies]
chrono = { workspace = true }
+criterion = { workspace = true }
+datafusion-functions-nested = { workspace = true }
+tempfile = { workspace = true }
# Note: add additional linter rules in lib.rs.
# Rust does not support workspace + new linter rules in subcrates yet
@@ -73,3 +76,7 @@ parquet_encryption = [
"datafusion-common/parquet_encryption",
"datafusion-execution/parquet_encryption",
]
+
+[[bench]]
+name = "parquet_nested_filter_pushdown"
+harness = false
diff --git a/datafusion/datasource-parquet/benches/parquet_nested_filter_pushdown.rs b/datafusion/datasource-parquet/benches/parquet_nested_filter_pushdown.rs
new file mode 100644
index 0000000000..ed92031f86
--- /dev/null
+++ b/datafusion/datasource-parquet/benches/parquet_nested_filter_pushdown.rs
@@ -0,0 +1,238 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, LazyLock};
+
+use arrow::array::{
+ BinaryBuilder, BooleanArray, ListBuilder, RecordBatch, StringBuilder,
+};
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
+use datafusion_datasource_parquet::{ParquetFileMetrics, build_row_filter};
+use datafusion_expr::{Expr, col};
+use datafusion_functions_nested::expr_fn::array_has;
+use datafusion_physical_expr::planner::logical2physical;
+use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use parquet::arrow::{ArrowWriter, ProjectionMask};
+use parquet::file::properties::WriterProperties;
+use tempfile::TempDir;
+
+const ROW_GROUP_SIZE: usize = 10_000;
+const TOTAL_ROW_GROUPS: usize = 10;
+const TOTAL_ROWS: usize = ROW_GROUP_SIZE * TOTAL_ROW_GROUPS;
+const TARGET_VALUE: &str = "target_value";
+const COLUMN_NAME: &str = "list_col";
+const PAYLOAD_COLUMN_NAME: &str = "payload";
+// Large binary payload to emphasize decoding overhead when pushdown is disabled.
+const PAYLOAD_BYTES: usize = 8 * 1024;
+
+struct BenchmarkDataset {
+ _tempdir: TempDir,
+ file_path: PathBuf,
+}
+
+impl BenchmarkDataset {
+ fn path(&self) -> &Path {
+ &self.file_path
+ }
+}
+
+static DATASET: LazyLock<BenchmarkDataset> = LazyLock::new(|| {
+ create_dataset().expect("failed to prepare parquet benchmark dataset")
+});
+
+fn parquet_nested_filter_pushdown(c: &mut Criterion) {
+ let dataset_path = DATASET.path().to_owned();
+ let mut group = c.benchmark_group("parquet_nested_filter_pushdown");
+ group.throughput(Throughput::Elements(TOTAL_ROWS as u64));
+
+ group.bench_function("no_pushdown", |b| {
+ let file_schema = setup_reader(&dataset_path);
+ let predicate = logical2physical(&create_predicate(), &file_schema);
+ b.iter(|| {
+ let matched = scan_with_predicate(&dataset_path, &predicate, false)
+ .expect("baseline parquet scan with filter succeeded");
+ assert_eq!(matched, ROW_GROUP_SIZE);
+ });
+ });
+
+ group.bench_function("with_pushdown", |b| {
+ let file_schema = setup_reader(&dataset_path);
+ let predicate = logical2physical(&create_predicate(), &file_schema);
+ b.iter(|| {
+ let matched = scan_with_predicate(&dataset_path, &predicate, true)
+ .expect("pushdown parquet scan with filter succeeded");
+ assert_eq!(matched, ROW_GROUP_SIZE);
+ });
+ });
+
+ group.finish();
+}
+
+fn setup_reader(path: &Path) -> SchemaRef {
+ let file = std::fs::File::open(path).expect("failed to open file");
+ let builder =
+ ParquetRecordBatchReaderBuilder::try_new(file).expect("failed to build reader");
+ Arc::clone(builder.schema())
+}
+
+fn create_predicate() -> Expr {
+ array_has(
+ col(COLUMN_NAME),
+ Expr::Literal(ScalarValue::Utf8(Some(TARGET_VALUE.to_string())), None),
+ )
+}
+
+fn scan_with_predicate(
+ path: &Path,
+ predicate: &Arc<dyn datafusion_physical_expr::PhysicalExpr>,
+ pushdown: bool,
+) -> datafusion_common::Result<usize> {
+ let file = std::fs::File::open(path)?;
+ let builder = ParquetRecordBatchReaderBuilder::try_new(file)?;
+ let metadata = builder.metadata().clone();
+ let file_schema = builder.schema();
+ let projection = ProjectionMask::all();
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let file_metrics = ParquetFileMetrics::new(0, &path.display().to_string(), &metrics);
+
+ let builder = if pushdown {
+ if let Some(row_filter) =
+ build_row_filter(predicate, file_schema, &metadata, false, &file_metrics)?
+ {
+ builder.with_row_filter(row_filter)
+ } else {
+ builder
+ }
+ } else {
+ builder
+ };
+
+ let reader = builder.with_projection(projection).build()?;
+
+ let mut matched_rows = 0usize;
+ for batch in reader {
+ let batch = batch?;
+ matched_rows += count_matches(predicate, &batch)?;
+ }
+
+ if pushdown {
+ let pruned_rows = file_metrics.pushdown_rows_pruned.value();
+ assert_eq!(
+ pruned_rows,
+ TOTAL_ROWS - matched_rows,
+ "row-level pushdown should prune 90% of rows"
+ );
+ }
+
+ Ok(matched_rows)
+}
+
+fn count_matches(
+ expr: &Arc<dyn datafusion_physical_expr::PhysicalExpr>,
+ batch: &RecordBatch,
+) -> datafusion_common::Result<usize> {
+ let values = expr.evaluate(batch)?.into_array(batch.num_rows())?;
+ let bools = values
+ .as_any()
+ .downcast_ref::<BooleanArray>()
+ .expect("boolean filter result");
+
+ Ok(bools.iter().filter(|v| matches!(v, Some(true))).count())
+}
+
+fn create_dataset() -> datafusion_common::Result<BenchmarkDataset> {
+ let tempdir = TempDir::new()?;
+ let file_path = tempdir.path().join("nested_lists.parquet");
+
+ let field = Arc::new(Field::new("item", DataType::Utf8, true));
+ let schema = Arc::new(Schema::new(vec![
+ Field::new(COLUMN_NAME, DataType::List(field), false),
+ Field::new(PAYLOAD_COLUMN_NAME, DataType::Binary, false),
+ ]));
+
+ let writer_props = WriterProperties::builder()
+ .set_max_row_group_size(ROW_GROUP_SIZE)
+ .build();
+
+ let mut writer = ArrowWriter::try_new(
+ std::fs::File::create(&file_path)?,
+ Arc::clone(&schema),
+ Some(writer_props),
+ )?;
+
+ // Create sorted row groups with distinct values so that min/max statistics
+ // allow skipping most groups when applying a selective predicate.
+ let sorted_values = [
+ "alpha",
+ "bravo",
+ "charlie",
+ "delta",
+ "echo",
+ "foxtrot",
+ "golf",
+ "hotel",
+ "india",
+ TARGET_VALUE,
+ ];
+
+ for value in sorted_values {
+ let batch = build_list_batch(&schema, value, ROW_GROUP_SIZE)?;
+ writer.write(&batch)?;
+ }
+
+ writer.close()?;
+
+ // Ensure the writer respected the requested row group size
+ let reader =
+ ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(&file_path)?)?;
+ assert_eq!(reader.metadata().row_groups().len(), TOTAL_ROW_GROUPS);
+
+ Ok(BenchmarkDataset {
+ _tempdir: tempdir,
+ file_path,
+ })
+}
+
+fn build_list_batch(
+ schema: &SchemaRef,
+ value: &str,
+ len: usize,
+) -> datafusion_common::Result<RecordBatch> {
+ let mut builder = ListBuilder::new(StringBuilder::new());
+ let mut payload_builder = BinaryBuilder::new();
+ let payload = vec![1u8; PAYLOAD_BYTES];
+ for _ in 0..len {
+ builder.values().append_value(value);
+ builder.append(true);
+ payload_builder.append_value(&payload);
+ }
+
+ let array = builder.finish();
+ let payload_array = payload_builder.finish();
+ Ok(RecordBatch::try_new(
+ Arc::clone(schema),
+ vec![Arc::new(array), Arc::new(payload_array)],
+ )?)
+}
+
+criterion_group!(benches, parquet_nested_filter_pushdown);
+criterion_main!(benches);
diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rs
index eb4cc9e9ad..d7e92f70af 100644
--- a/datafusion/datasource-parquet/src/mod.rs
+++ b/datafusion/datasource-parquet/src/mod.rs
@@ -32,6 +32,7 @@ mod row_filter;
mod row_group_filter;
mod sort;
pub mod source;
+mod supported_predicates;
mod writer;
pub use access_plan::{ParquetAccessPlan, RowGroupAccess};
diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rs
index ba3b29be40..04c11b8875 100644
--- a/datafusion/datasource-parquet/src/row_filter.rs
+++ b/datafusion/datasource-parquet/src/row_filter.rs
@@ -58,6 +58,11 @@
//! 8. Build the `RowFilter` with the sorted predicates followed by
//! the unsorted predicates. Within each partition, predicates are
//! still be sorted by size.
+//!
+//! List-aware predicates (for example, `array_has`, `array_has_all`, and
+//! `array_has_any`) can be evaluated directly during Parquet decoding. Struct
+//! columns and other nested projections that are not explicitly supported will
+//! continue to be evaluated after the batches are materialized.
use std::cmp::Ordering;
use std::collections::BTreeSet;
@@ -70,6 +75,7 @@ use arrow::record_batch::RecordBatch;
use parquet::arrow::ProjectionMask;
use parquet::arrow::arrow_reader::{ArrowPredicate, RowFilter};
use parquet::file::metadata::ParquetMetaData;
+use parquet::schema::types::SchemaDescriptor;
use datafusion_common::Result;
use datafusion_common::cast::as_boolean_array;
@@ -81,6 +87,7 @@ use datafusion_physical_expr::{PhysicalExpr, split_conjunction};
use datafusion_physical_plan::metrics;
use super::ParquetFileMetrics;
+use super::supported_predicates::supports_list_predicates;
/// A "compiled" predicate passed to `ParquetRecordBatchStream` to perform
/// row-level filtering during parquet decoding.
@@ -91,12 +98,14 @@ use super::ParquetFileMetrics;
///
/// An expression can be evaluated as a `DatafusionArrowPredicate` if it:
/// * Does not reference any projected columns
-/// * Does not reference columns with non-primitive types (e.g. structs / lists)
+/// * References either primitive columns or list columns used by
+/// supported predicates (such as `array_has_all` or NULL checks). Struct
+/// columns are still evaluated after decoding.
#[derive(Debug)]
pub(crate) struct DatafusionArrowPredicate {
/// the filter expression
physical_expr: Arc<dyn PhysicalExpr>,
- /// Path to the columns in the parquet schema required to evaluate the
+ /// Path to the leaf columns in the parquet schema required to evaluate the
/// expression
projection_mask: ProjectionMask,
/// how many rows were filtered out by this predicate
@@ -121,9 +130,12 @@ impl DatafusionArrowPredicate {
Ok(Self {
physical_expr,
- projection_mask: ProjectionMask::roots(
+ // Use leaf indices: when nested columns are involved, we must specify
+ // leaf (primitive) column indices in the Parquet schema so the decoder
+ // can properly project and filter nested structures.
+ projection_mask: ProjectionMask::leaves(
metadata.file_metadata().schema_descr(),
- candidate.projection,
+ candidate.projection.leaf_indices.iter().copied(),
),
rows_pruned,
rows_matched,
@@ -177,12 +189,23 @@ pub(crate) struct FilterCandidate {
/// Can this filter use an index (e.g. a page index) to prune rows?
can_use_index: bool,
/// Column indices into the parquet file schema required to evaluate this filter.
- projection: Vec<usize>,
+ projection: LeafProjection,
/// The Arrow schema containing only the columns required by this filter,
/// projected from the file's Arrow schema.
filter_schema: SchemaRef,
}
+/// Projection specification for nested columns using Parquet leaf column indices.
+///
+/// For nested types like List and Struct, Parquet stores data in leaf columns
+/// (the primitive fields). This struct tracks which leaf columns are needed
+/// to evaluate a filter expression.
+#[derive(Debug, Clone)]
+struct LeafProjection {
+ /// Leaf column indices in the Parquet schema descriptor.
+ leaf_indices: Vec<usize>,
+}
+
/// Helper to build a `FilterCandidate`.
///
/// This will do several things:
@@ -212,23 +235,29 @@ impl FilterCandidateBuilder {
/// * `Ok(None)` if the expression cannot be used as an ArrowFilter
/// * `Err(e)` if an error occurs while building the candidate
pub fn build(self, metadata: &ParquetMetaData) -> Result<Option<FilterCandidate>> {
- let Some(required_column_indices) =
- pushdown_columns(&self.expr, &self.file_schema)?
+ let Some(required_columns) = pushdown_columns(&self.expr, &self.file_schema)?
else {
return Ok(None);
};
- let projected_schema =
- Arc::new(self.file_schema.project(&required_column_indices)?);
+ let root_indices: Vec<_> =
+ required_columns.required_columns.into_iter().collect();
+ let leaf_indices = leaf_indices_for_roots(
+ &root_indices,
+ metadata.file_metadata().schema_descr(),
+ required_columns.nested,
+ );
- let required_bytes = size_of_columns(&required_column_indices, metadata)?;
- let can_use_index = columns_sorted(&required_column_indices, metadata)?;
+ let projected_schema = Arc::new(self.file_schema.project(&root_indices)?);
+
+ let required_bytes = size_of_columns(&leaf_indices, metadata)?;
+ let can_use_index = columns_sorted(&leaf_indices, metadata)?;
Ok(Some(FilterCandidate {
expr: self.expr,
required_bytes,
can_use_index,
- projection: required_column_indices,
+ projection: LeafProjection { leaf_indices },
filter_schema: projected_schema,
}))
}
@@ -238,7 +267,8 @@ impl FilterCandidateBuilder {
/// prevent the expression from being pushed down to the parquet decoder.
///
/// An expression cannot be pushed down if it references:
-/// - Non-primitive columns (like structs or lists)
+/// - Unsupported nested columns (structs or list fields that are not covered by
+/// the supported predicate set)
/// - Columns that don't exist in the file schema
struct PushdownChecker<'schema> {
/// Does the expression require any non-primitive columns (like structs)?
@@ -247,34 +277,78 @@ struct PushdownChecker<'schema> {
projected_columns: bool,
/// Indices into the file schema of columns required to evaluate the expression.
required_columns: BTreeSet<usize>,
+ /// Tracks the nested column behavior found during traversal.
+ nested_behavior: NestedColumnSupport,
+ /// Whether nested list columns are supported by the predicate semantics.
+ allow_list_columns: bool,
/// The Arrow schema of the parquet file.
file_schema: &'schema Schema,
}
impl<'schema> PushdownChecker<'schema> {
- fn new(file_schema: &'schema Schema) -> Self {
+ fn new(file_schema: &'schema Schema, allow_list_columns: bool) -> Self {
Self {
non_primitive_columns: false,
projected_columns: false,
required_columns: BTreeSet::default(),
+ nested_behavior: NestedColumnSupport::PrimitiveOnly,
+ allow_list_columns,
file_schema,
}
}
fn check_single_column(&mut self, column_name: &str) -> Option<TreeNodeRecursion> {
- if let Ok(idx) = self.file_schema.index_of(column_name) {
- self.required_columns.insert(idx);
- if DataType::is_nested(self.file_schema.field(idx).data_type()) {
- self.non_primitive_columns = true;
+ let idx = match self.file_schema.index_of(column_name) {
+ Ok(idx) => idx,
+ Err(_) => {
+ // Column does not exist in the file schema, so we can't push this down.
+ self.projected_columns = true;
return Some(TreeNodeRecursion::Jump);
}
+ };
+
+ self.required_columns.insert(idx);
+ let data_type = self.file_schema.field(idx).data_type();
+
+ if DataType::is_nested(data_type) {
+ self.handle_nested_type(data_type)
} else {
- // Column does not exist in the file schema, so we can't push this down.
- self.projected_columns = true;
- return Some(TreeNodeRecursion::Jump);
+ None
}
+ }
- None
+ /// Determines whether a nested data type can be pushed down to Parquet decoding.
+ ///
+ /// Returns `Some(TreeNodeRecursion::Jump)` if the nested type prevents pushdown,
+ /// `None` if the type is supported and pushdown can continue.
+ fn handle_nested_type(&mut self, data_type: &DataType) -> Option<TreeNodeRecursion> {
+ if self.is_nested_type_supported(data_type) {
+ // Update to ListsSupported if we haven't encountered unsupported types yet
+ if self.nested_behavior == NestedColumnSupport::PrimitiveOnly {
+ self.nested_behavior = NestedColumnSupport::ListsSupported;
+ }
+ None
+ } else {
+ // Block pushdown for unsupported nested types:
+ // - Structs (regardless of predicate support)
+ // - Lists without supported predicates
+ self.nested_behavior = NestedColumnSupport::Unsupported;
+ self.non_primitive_columns = true;
+ Some(TreeNodeRecursion::Jump)
+ }
+ }
+
+ /// Checks if a nested data type is supported for list column pushdown.
+ ///
+ /// List columns are only supported if:
+ /// 1. The data type is a list variant (List, LargeList, or FixedSizeList)
+ /// 2. The expression contains supported list predicates (e.g., array_has_all)
+ fn is_nested_type_supported(&self, data_type: &DataType) -> bool {
+ let is_list = matches!(
+ data_type,
+ DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _)
+ );
+ self.allow_list_columns && is_list
}
#[inline]
@@ -297,34 +371,138 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {
}
}
+/// Describes the nested column behavior for filter pushdown.
+///
+/// This enum makes explicit the different states a predicate can be in
+/// with respect to nested column handling during Parquet decoding.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum NestedColumnSupport {
+ /// Expression references only primitive (non-nested) columns.
+ /// These can always be pushed down to the Parquet decoder.
+ PrimitiveOnly,
+ /// Expression references list columns with supported predicates
+ /// (e.g., array_has, array_has_all, IS NULL).
+ /// These can be pushed down to the Parquet decoder.
+ ListsSupported,
+ /// Expression references unsupported nested types (e.g., structs)
+ /// or list columns without supported predicates.
+ /// These cannot be pushed down and must be evaluated after decoding.
+ Unsupported,
+}
+
+#[derive(Debug)]
+struct PushdownColumns {
+ required_columns: BTreeSet<usize>,
+ nested: NestedColumnSupport,
+}
+
/// Checks if a given expression can be pushed down to the parquet decoder.
///
-/// Returns `Some(column_indices)` if the expression can be pushed down,
-/// where `column_indices` are the indices into the file schema of all columns
+/// Returns `Some(PushdownColumns)` if the expression can be pushed down,
+/// where the struct contains the indices into the file schema of all columns
/// required to evaluate the expression.
///
/// Returns `None` if the expression cannot be pushed down (e.g., references
-/// non-primitive types or columns not in the file).
+/// unsupported nested types or columns not in the file).
fn pushdown_columns(
expr: &Arc<dyn PhysicalExpr>,
file_schema: &Schema,
-) -> Result<Option<Vec<usize>>> {
- let mut checker = PushdownChecker::new(file_schema);
+) -> Result<Option<PushdownColumns>> {
+ let allow_list_columns = supports_list_predicates(expr);
+ let mut checker = PushdownChecker::new(file_schema, allow_list_columns);
expr.visit(&mut checker)?;
- Ok((!checker.prevents_pushdown())
- .then_some(checker.required_columns.into_iter().collect()))
+ Ok((!checker.prevents_pushdown()).then_some(PushdownColumns {
+ required_columns: checker.required_columns,
+ nested: checker.nested_behavior,
+ }))
+}
+
+fn leaf_indices_for_roots(
+ root_indices: &[usize],
+ schema_descr: &SchemaDescriptor,
+ nested: NestedColumnSupport,
+) -> Vec<usize> {
+ // For primitive-only columns, root indices ARE the leaf indices
+ if nested == NestedColumnSupport::PrimitiveOnly {
+ return root_indices.to_vec();
+ }
+
+ // For List columns, expand to the single leaf column (item field)
+ // For Struct columns (unsupported), this would expand to multiple leaves
+ let root_set: BTreeSet<_> = root_indices.iter().copied().collect();
+
+ (0..schema_descr.num_columns())
+ .filter(|leaf_idx| {
+ root_set.contains(&schema_descr.get_column_root_idx(*leaf_idx))
+ })
+ .collect()
}
/// Checks if a predicate expression can be pushed down to the parquet decoder.
///
/// Returns `true` if all columns referenced by the expression:
/// - Exist in the provided schema
-/// - Are primitive types (not structs, lists, etc.)
+/// - Are primitive types OR list columns with supported predicates
+/// (e.g., `array_has`, `array_has_all`, `array_has_any`, IS NULL, IS NOT NULL)
+/// - Struct columns are not supported and will prevent pushdown
///
/// # Arguments
/// * `expr` - The filter expression to check
/// * `file_schema` - The Arrow schema of the parquet file (or table schema when
/// the file schema is not yet available during planning)
+///
+/// # Examples
+///
+/// Primitive column filters can be pushed down:
+/// ```ignore
+/// use datafusion_expr::{col, Expr};
+/// use datafusion_common::ScalarValue;
+/// use arrow::datatypes::{DataType, Field, Schema};
+/// use std::sync::Arc;
+///
+/// let schema = Arc::new(Schema::new(vec![
+/// Field::new("age", DataType::Int32, false),
+/// ]));
+///
+/// // Primitive filter: can be pushed down
+/// let expr = col("age").gt(Expr::Literal(ScalarValue::Int32(Some(30)), None));
+/// let expr = logical2physical(&expr, &schema);
+/// assert!(can_expr_be_pushed_down_with_schemas(&expr, &schema));
+/// ```
+///
+/// Struct column filters cannot be pushed down:
+/// ```ignore
+/// use arrow::datatypes::Fields;
+///
+/// let schema = Arc::new(Schema::new(vec![
+/// Field::new("person", DataType::Struct(
+/// Fields::from(vec![Field::new("name", DataType::Utf8, true)])
+/// ), true),
+/// ]));
+///
+/// // Struct filter: cannot be pushed down
+/// let expr = col("person").is_not_null();
+/// let expr = logical2physical(&expr, &schema);
+/// assert!(!can_expr_be_pushed_down_with_schemas(&expr, &schema));
+/// ```
+///
+/// List column filters with supported predicates can be pushed down:
+/// ```ignore
+/// use datafusion_functions_nested::expr_fn::{array_has_all, make_array};
+///
+/// let schema = Arc::new(Schema::new(vec![
+/// Field::new("tags", DataType::List(
+/// Arc::new(Field::new("item", DataType::Utf8, true))
+/// ), true),
+/// ]));
+///
+/// // Array filter with supported predicate: can be pushed down
+/// let expr = array_has_all(col("tags"), make_array(vec![
+/// Expr::Literal(ScalarValue::Utf8(Some("rust".to_string())), None)
+/// ]));
+/// let expr = logical2physical(&expr, &schema);
+/// assert!(can_expr_be_pushed_down_with_schemas(&expr, &schema));
+/// ```
pub fn can_expr_be_pushed_down_with_schemas(
expr: &Arc<dyn PhysicalExpr>,
file_schema: &Schema,
@@ -335,7 +513,7 @@ pub fn can_expr_be_pushed_down_with_schemas(
}
}
-/// Calculate the total compressed size of all `Column`'s required for
+/// Calculate the total compressed size of all leaf columns required for
/// predicate `Expr`.
///
/// This value represents the total amount of IO required to evaluate the
@@ -464,21 +642,30 @@ mod test {
use super::*;
use datafusion_common::ScalarValue;
+ use arrow::array::{ListBuilder, StringBuilder};
use arrow::datatypes::{Field, TimeUnit::Nanosecond};
use datafusion_expr::{Expr, col};
+ use datafusion_functions_nested::array_has::{
+ array_has_all_udf, array_has_any_udf, array_has_udf,
+ };
+ use datafusion_functions_nested::expr_fn::{
+ array_has, array_has_all, array_has_any, make_array,
+ };
use datafusion_physical_expr::planner::logical2physical;
use datafusion_physical_expr_adapter::{
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapterFactory,
};
- use datafusion_physical_plan::metrics::{Count, Time};
+ use datafusion_physical_plan::metrics::{Count, ExecutionPlanMetricsSet, Time};
+ use parquet::arrow::ArrowWriter;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::arrow::parquet_to_arrow_schema;
use parquet::file::reader::{FileReader, SerializedFileReader};
+ use tempfile::NamedTempFile;
- // We should ignore predicate that read non-primitive columns
+ // List predicates used by the decoder should be accepted for pushdown
#[test]
- fn test_filter_candidate_builder_ignore_complex_types() {
+ fn test_filter_candidate_builder_supports_list_types() {
let testdata = datafusion_common::test_util::parquet_test_data();
let file = std::fs::File::open(format!("{testdata}/list_columns.parquet"))
.expect("opening file");
@@ -496,11 +683,16 @@ mod test {
let table_schema = Arc::new(table_schema.clone());
+ let list_index = table_schema
+ .index_of("int64_list")
+ .expect("list column should exist");
+
let candidate = FilterCandidateBuilder::new(expr, table_schema)
.build(metadata)
- .expect("building candidate");
+ .expect("building candidate")
+ .expect("list pushdown should be supported");
- assert!(candidate.is_none());
+ assert_eq!(candidate.projection.leaf_indices, vec![list_index]);
}
#[test]
@@ -590,14 +782,233 @@ mod test {
}
#[test]
- fn nested_data_structures_prevent_pushdown() {
+ fn struct_data_structures_prevent_pushdown() {
+ let table_schema = Arc::new(Schema::new(vec![Field::new(
+ "struct_col",
+ DataType::Struct(
+ vec![Arc::new(Field::new("a", DataType::Int32, true))].into(),
+ ),
+ true,
+ )]));
+
+ let expr = col("struct_col").is_not_null();
+ let expr = logical2physical(&expr, &table_schema);
+
+ assert!(!can_expr_be_pushed_down_with_schemas(&expr, &table_schema));
+ }
+
+ #[test]
+ fn mixed_primitive_and_struct_prevents_pushdown() {
+ // Even when a predicate contains both primitive and unsupported nested columns,
+ // the entire predicate should not be pushed down because the struct column
+ // cannot be evaluated during Parquet decoding.
+ let table_schema = Arc::new(Schema::new(vec![
+ Field::new(
+ "struct_col",
+ DataType::Struct(
+ vec![Arc::new(Field::new("a", DataType::Int32, true))].into(),
+ ),
+ true,
+ ),
+ Field::new("int_col", DataType::Int32, false),
+ ]));
+
+ // Expression: (struct_col IS NOT NULL) AND (int_col = 5)
+ // Even though int_col is primitive, the presence of struct_col in the
+ // conjunction should prevent pushdown of the entire expression.
+ let expr = col("struct_col")
+ .is_not_null()
+ .and(col("int_col").eq(Expr::Literal(ScalarValue::Int32(Some(5)), None)));
+ let expr = logical2physical(&expr, &table_schema);
+
+ // The entire expression should not be pushed down
+ assert!(!can_expr_be_pushed_down_with_schemas(&expr, &table_schema));
+
+ // However, just the int_col predicate alone should be pushable
+ let expr_int_only =
+ col("int_col").eq(Expr::Literal(ScalarValue::Int32(Some(5)), None));
+ let expr_int_only = logical2physical(&expr_int_only, &table_schema);
+ assert!(can_expr_be_pushed_down_with_schemas(
+ &expr_int_only,
+ &table_schema
+ ));
+ }
+
+ #[test]
+ fn nested_lists_allow_pushdown_checks() {
let table_schema = Arc::new(get_lists_table_schema());
let expr = col("utf8_list").is_not_null();
let expr = logical2physical(&expr, &table_schema);
check_expression_can_evaluate_against_schema(&expr, &table_schema);
- assert!(!can_expr_be_pushed_down_with_schemas(&expr, &table_schema));
+ assert!(can_expr_be_pushed_down_with_schemas(&expr, &table_schema));
+ }
+
+ #[test]
+ fn array_has_all_pushdown_filters_rows() {
+ // Test array_has_all: checks if array contains all of ["c"]
+ // Rows with "c": row 1 and row 2
+ let expr = array_has_all(
+ col("letters"),
+ make_array(vec![Expr::Literal(
+ ScalarValue::Utf8(Some("c".to_string())),
+ None,
+ )]),
+ );
+ test_array_predicate_pushdown("array_has_all", expr, 1, 2, true);
+ }
+
+ /// Helper function to test array predicate pushdown functionality.
+ ///
+ /// Creates a Parquet file with a list column, applies the given predicate,
+ /// and verifies that rows are correctly filtered during decoding.
+ fn test_array_predicate_pushdown(
+ func_name: &str,
+ predicate_expr: Expr,
+ expected_pruned: usize,
+ expected_matched: usize,
+ expect_list_support: bool,
+ ) {
+ let item_field = Arc::new(Field::new("item", DataType::Utf8, true));
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "letters",
+ DataType::List(item_field),
+ true,
+ )]));
+
+ let mut builder = ListBuilder::new(StringBuilder::new());
+ // Row 0: ["a", "b"]
+ builder.values().append_value("a");
+ builder.values().append_value("b");
+ builder.append(true);
+
+ // Row 1: ["c"]
+ builder.values().append_value("c");
+ builder.append(true);
+
+ // Row 2: ["c", "d"]
+ builder.values().append_value("c");
+ builder.values().append_value("d");
+ builder.append(true);
+
+ let batch =
+ RecordBatch::try_new(schema.clone(), vec![Arc::new(builder.finish())])
+ .expect("record batch");
+
+ let file = NamedTempFile::new().expect("temp file");
+ let mut writer =
+ ArrowWriter::try_new(file.reopen().unwrap(), schema, None).expect("writer");
+ writer.write(&batch).expect("write batch");
+ writer.close().expect("close writer");
+
+ let reader_file = file.reopen().expect("reopen file");
+ let parquet_reader_builder =
+ ParquetRecordBatchReaderBuilder::try_new(reader_file)
+ .expect("reader builder");
+ let metadata = parquet_reader_builder.metadata().clone();
+ let file_schema = parquet_reader_builder.schema().clone();
+
+ let expr = logical2physical(&predicate_expr, &file_schema);
+ if expect_list_support {
+ assert!(supports_list_predicates(&expr));
+ }
+
+ let metrics = ExecutionPlanMetricsSet::new();
+ let file_metrics =
+ ParquetFileMetrics::new(0, &format!("{func_name}.parquet"), &metrics);
+
+ let row_filter =
+ build_row_filter(&expr, &file_schema, &metadata, false, &file_metrics)
+ .expect("building row filter")
+ .expect("row filter should exist");
+
+ let reader = parquet_reader_builder
+ .with_row_filter(row_filter)
+ .build()
+ .expect("build reader");
+
+ let mut total_rows = 0;
+ for batch in reader {
+ let batch = batch.expect("record batch");
+ total_rows += batch.num_rows();
+ }
+
+ assert_eq!(
+ file_metrics.pushdown_rows_pruned.value(),
+ expected_pruned,
+ "{func_name}: expected {expected_pruned} pruned rows"
+ );
+ assert_eq!(
+ file_metrics.pushdown_rows_matched.value(),
+ expected_matched,
+ "{func_name}: expected {expected_matched} matched rows"
+ );
+ assert_eq!(
+ total_rows, expected_matched,
+ "{func_name}: expected {expected_matched} total rows"
+ );
+ }
+
+ #[test]
+ fn array_has_pushdown_filters_rows() {
+ // Test array_has: checks if "c" is in the array
+ // Rows with "c": row 1 and row 2
+ let expr = array_has(
+ col("letters"),
+ Expr::Literal(ScalarValue::Utf8(Some("c".to_string())), None),
+ );
+ test_array_predicate_pushdown("array_has", expr, 1, 2, true);
+ }
+
+ #[test]
+ fn array_has_any_pushdown_filters_rows() {
+ // Test array_has_any: checks if array contains any of ["a", "d"]
+ // Row 0 has "a", row 2 has "d" - both should match
+ let expr = array_has_any(
+ col("letters"),
+ make_array(vec![
+ Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),
+ Expr::Literal(ScalarValue::Utf8(Some("d".to_string())), None),
+ ]),
+ );
+ test_array_predicate_pushdown("array_has_any", expr, 1, 2, true);
+ }
+
+ #[test]
+ fn array_has_udf_pushdown_filters_rows() {
+ let expr = array_has_udf().call(vec![
+ col("letters"),
+ Expr::Literal(ScalarValue::Utf8(Some("c".to_string())), None),
+ ]);
+
+ test_array_predicate_pushdown("array_has_udf", expr, 1, 2, true);
+ }
+
+ #[test]
+ fn array_has_all_udf_pushdown_filters_rows() {
+ let expr = array_has_all_udf().call(vec![
+ col("letters"),
+ make_array(vec![Expr::Literal(
+ ScalarValue::Utf8(Some("c".to_string())),
+ None,
+ )]),
+ ]);
+
+ test_array_predicate_pushdown("array_has_all_udf", expr, 1, 2, true);
+ }
+
+ #[test]
+ fn array_has_any_udf_pushdown_filters_rows() {
+ let expr = array_has_any_udf().call(vec![
+ col("letters"),
+ make_array(vec![
+ Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),
+ Expr::Literal(ScalarValue::Utf8(Some("d".to_string())), None),
+ ]),
+ ]);
+
+ test_array_predicate_pushdown("array_has_any_udf", expr, 1, 2, true);
}
#[test]
diff --git a/datafusion/datasource-parquet/src/supported_predicates.rs b/datafusion/datasource-parquet/src/supported_predicates.rs
new file mode 100644
index 0000000000..a205c12dd0
--- /dev/null
+++ b/datafusion/datasource-parquet/src/supported_predicates.rs
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Registry of physical expressions that support nested list column pushdown
+//! to the Parquet decoder.
+//!
+//! This module provides a trait-based approach for determining which predicates
+//! can be safely evaluated on nested list columns during Parquet decoding.
+
+use std::sync::Arc;
+
+use datafusion_physical_expr::expressions::{IsNotNullExpr, IsNullExpr};
+use datafusion_physical_expr::{PhysicalExpr, ScalarFunctionExpr};
+
+/// Trait for physical expressions that support list column pushdown during
+/// Parquet decoding.
+///
+/// This trait provides a type-safe mechanism for identifying expressions that
+/// can be safely pushed down to the Parquet decoder for evaluation on nested
+/// list columns.
+///
+/// # Implementation Notes
+///
+/// Expression types in external crates cannot directly implement this trait
+/// due to Rust's orphan rules. Instead, we use a blanket implementation that
+/// delegates to a registration mechanism.
+///
+/// # Examples
+///
+/// ```ignore
+/// use datafusion_physical_expr::PhysicalExpr;
+/// use datafusion_datasource_parquet::SupportsListPushdown;
+///
+/// let expr: Arc<dyn PhysicalExpr> = ...;
+/// if expr.supports_list_pushdown() {
+/// // Can safely push down to Parquet decoder
+/// }
+/// ```
+pub trait SupportsListPushdown {
+ /// Returns `true` if this expression supports list column pushdown.
+ fn supports_list_pushdown(&self) -> bool;
+}
+
+/// Blanket implementation for all physical expressions.
+///
+/// This delegates to specialized predicates that check whether the concrete
+/// expression type is registered as supporting list pushdown. This design
+/// allows the trait to work with expression types defined in external crates.
+impl SupportsListPushdown for dyn PhysicalExpr {
+ fn supports_list_pushdown(&self) -> bool {
+ is_null_check(self) || is_supported_scalar_function(self)
+ }
+}
+
+/// Checks if an expression is a NULL or NOT NULL check.
+///
+/// These checks are universally supported for all column types.
+fn is_null_check(expr: &dyn PhysicalExpr) -> bool {
+ expr.as_any().downcast_ref::<IsNullExpr>().is_some()
+ || expr.as_any().downcast_ref::<IsNotNullExpr>().is_some()
+}
+
+/// Checks if an expression is a scalar function registered for list pushdown.
+///
+/// Returns `true` if the expression is a `ScalarFunctionExpr` whose function
+/// is in the registry of supported operations.
+fn is_supported_scalar_function(expr: &dyn PhysicalExpr) -> bool {
+ scalar_function_name(expr).is_some_and(|name| {
+ // Registry of verified array functions
+ matches!(name, "array_has" | "array_has_all" | "array_has_any")
+ })
+}
+
+fn scalar_function_name(expr: &dyn PhysicalExpr) -> Option<&str> {
+ expr.as_any()
+ .downcast_ref::<ScalarFunctionExpr>()
+ .map(ScalarFunctionExpr::name)
+}
+
+/// Checks whether the given physical expression contains a supported nested
+/// predicate (for example, `array_has_all`).
+///
+/// This function recursively traverses the expression tree to determine if
+/// any node contains predicates that support list column pushdown to the
+/// Parquet decoder.
+///
+/// # Supported predicates
+///
+/// - `IS NULL` and `IS NOT NULL` checks on any column type
+/// - Array functions: `array_has`, `array_has_all`, `array_has_any`
+///
+/// # Returns
+///
+/// `true` if the expression or any of its children contain supported predicates.
+pub fn supports_list_predicates(expr: &Arc<dyn PhysicalExpr>) -> bool {
+ expr.supports_list_pushdown()
+ || expr
+ .children()
+ .iter()
+ .any(|child| supports_list_predicates(child))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_null_check_detection() {
+ use datafusion_physical_expr::expressions::Column;
+
+ let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("test", 0));
+ assert!(!is_null_check(col_expr.as_ref()));
+
+ // IsNullExpr and IsNotNullExpr detection requires actual instances
+ // which need schema setup - tested in integration tests
+ }
+
+ #[test]
+ fn test_supported_scalar_functions() {
+ use datafusion_physical_expr::expressions::Column;
+
+ let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("test", 0));
+
+ // Non-function expressions should return false
+ assert!(!is_supported_scalar_function(col_expr.as_ref()));
+
+ // Testing with actual ScalarFunctionExpr requires function setup
+ // and is better suited for integration tests
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
index 8bb79d5769..aa94e2e2f2 100644
--- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
@@ -563,3 +563,114 @@ ORDER BY start_timestamp, trace_id
LIMIT 1;
----
2024-10-01T00:00:00
+
+###
+# Array function predicate pushdown tests
+# These tests verify that array_has, array_has_all, and array_has_any predicates
+# are correctly pushed down to the DataSourceExec node
+###
+
+# Create test data with array columns
+statement ok
+COPY (
+ SELECT 1 as id, ['rust', 'performance'] as tags
+ UNION ALL
+ SELECT 2 as id, ['python', 'javascript'] as tags
+ UNION ALL
+ SELECT 3 as id, ['rust', 'webassembly'] as tags
+)
+TO 'test_files/scratch/parquet_filter_pushdown/array_data/data.parquet';
+
+statement ok
+CREATE EXTERNAL TABLE array_test STORED AS PARQUET LOCATION 'test_files/scratch/parquet_filter_pushdown/array_data/';
+
+statement ok
+SET datafusion.execution.parquet.pushdown_filters = true;
+
+# Test array_has predicate pushdown
+query I?
+SELECT id, tags FROM array_test WHERE array_has(tags, 'rust') ORDER BY id;
+----
+1 [rust, performance]
+3 [rust, webassembly]
+
+query TT
+EXPLAIN SELECT id, tags FROM array_test WHERE array_has(tags, 'rust') ORDER BY id;
+----
+logical_plan
+01)Sort: array_test.id ASC NULLS LAST
+02)--Filter: array_has(array_test.tags, Utf8("rust"))
+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has(array_test.tags, Utf8("rust"))]
+physical_plan
+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has(tags@1, rust)
+
+# Test array_has_all predicate pushdown
+query I?
+SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust', 'performance']) ORDER BY id;
+----
+1 [rust, performance]
+
+query TT
+EXPLAIN SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust', 'performance']) ORDER BY id;
+----
+logical_plan
+01)Sort: array_test.id ASC NULLS LAST
+02)--Filter: array_has_all(array_test.tags, List([rust, performance]))
+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust, performance]))]
+physical_plan
+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust, performance])
+
+# Test array_has_any predicate pushdown
+query I?
+SELECT id, tags FROM array_test WHERE array_has_any(tags, ['python', 'go']) ORDER BY id;
+----
+2 [python, javascript]
+
+query TT
+EXPLAIN SELECT id, tags FROM array_test WHERE array_has_any(tags, ['python', 'go']) ORDER BY id;
+----
+logical_plan
+01)Sort: array_test.id ASC NULLS LAST
+02)--Filter: array_has_any(array_test.tags, List([python, go]))
+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_any(array_test.tags, List([python, go]))]
+physical_plan
+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_any(tags@1, [python, go])
+
+# Test complex predicate with OR
+query I?
+SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust']) OR array_has_any(tags, ['python', 'go']) ORDER BY id;
+----
+1 [rust, performance]
+2 [python, javascript]
+3 [rust, webassembly]
+
+query TT
+EXPLAIN SELECT id, tags FROM array_test WHERE array_has_all(tags, ['rust']) OR array_has_any(tags, ['python', 'go']) ORDER BY id;
+----
+logical_plan
+01)Sort: array_test.id ASC NULLS LAST
+02)--Filter: array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))
+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))]
+physical_plan
+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust]) OR array_has_any(tags@1, [python, go])
+
+# Test array function with other predicates
+query I?
+SELECT id, tags FROM array_test WHERE id > 1 AND array_has(tags, 'rust') ORDER BY id;
+----
+3 [rust, webassembly]
+
+query TT
+EXPLAIN SELECT id, tags FROM array_test WHERE id > 1 AND array_has(tags, 'rust') ORDER BY id;
+----
+logical_plan
+01)Sort: array_test.id ASC NULLS LAST
+02)--Filter: array_test.id > Int64(1) AND array_has(array_test.tags, Utf8("rust"))
+03)----TableScan: array_test projection=[id, tags], partial_filters=[array_test.id > Int64(1), array_has(array_test.tags, Utf8("rust"))]
+physical_plan
+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=id@0 > 1 AND array_has(tags@1, rust), pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]