Skip to content

Apache DataFusion SQL Query Engine

Support distinct-from predicates in Parquet pruning (#22084)

Daniël Heresfe8dbfa94a89
diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs
index 3ec3541af9..441d1af3e9 100644
--- a/datafusion/core/tests/parquet/row_group_pruning.rs
+++ b/datafusion/core/tests/parquet/row_group_pruning.rs
@@ -1777,6 +1777,15 @@ fn make_i32_batch(
RecordBatch::try_new(schema, vec![array]).map_err(DataFusionError::from)
}
+fn make_nullable_i32_batch(
+ name: &str,
+ values: Vec<Option<i32>>,
+) -> datafusion_common::error::Result<RecordBatch> {
+ let schema = Arc::new(Schema::new(vec![Field::new(name, DataType::Int32, true)]));
+ let array: ArrayRef = Arc::new(Int32Array::from(values));
+ RecordBatch::try_new(schema, vec![array]).map_err(DataFusionError::from)
+}
+
// Helper function to create a batch with two Int32 columns
fn make_two_col_i32_batch(
name_a: &str,
@@ -1793,6 +1802,72 @@ fn make_two_col_i32_batch(
RecordBatch::try_new(schema, vec![array_a, array_b]).map_err(DataFusionError::from)
}
+#[tokio::test]
+async fn prune_is_not_distinct_from_i32() -> datafusion_common::error::Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
+ let batches = vec![
+ make_nullable_i32_batch("a", vec![None, None])?,
+ make_nullable_i32_batch("a", vec![Some(0), Some(0)])?,
+ make_nullable_i32_batch("a", vec![Some(0), Some(1)])?,
+ make_nullable_i32_batch("a", vec![Some(2), Some(3)])?,
+ make_nullable_i32_batch("a", vec![None, Some(5)])?,
+ ];
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query("SELECT a FROM t WHERE a IS NOT DISTINCT FROM 0")
+ .with_expected_errors(Some(0))
+ .with_expected_rows(3)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(2))
+ .with_fully_matched_by_stats(Some(1))
+ .with_pruned_by_stats(Some(3))
+ .with_limit_pruned_row_groups(Some(0))
+ .test_row_group_prune_with_custom_data(schema.clone(), batches.clone(), 2)
+ .await;
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query("SELECT a FROM t WHERE a IS NOT DISTINCT FROM NULL")
+ .with_expected_errors(Some(0))
+ .with_expected_rows(3)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(2))
+ .with_fully_matched_by_stats(Some(0))
+ .with_pruned_by_stats(Some(3))
+ .with_limit_pruned_row_groups(Some(0))
+ .test_row_group_prune_with_custom_data(schema.clone(), batches.clone(), 2)
+ .await;
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query("SELECT a FROM t WHERE a IS DISTINCT FROM 0")
+ .with_expected_errors(Some(0))
+ .with_expected_rows(7)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(4))
+ .with_fully_matched_by_stats(Some(1))
+ .with_pruned_by_stats(Some(1))
+ .with_limit_pruned_row_groups(Some(0))
+ .test_row_group_prune_with_custom_data(schema.clone(), batches.clone(), 2)
+ .await;
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query("SELECT a FROM t WHERE a IS DISTINCT FROM NULL")
+ .with_expected_errors(Some(0))
+ .with_expected_rows(7)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(4))
+ .with_fully_matched_by_stats(Some(3))
+ .with_pruned_by_stats(Some(1))
+ .with_limit_pruned_row_groups(Some(0))
+ .test_row_group_prune_with_custom_data(schema, batches, 2)
+ .await;
+
+ Ok(())
+}
+
#[tokio::test]
async fn test_limit_pruning_basic() -> datafusion_common::error::Result<()> {
// Scenario: Simple integer column, multiple row groups
diff --git a/datafusion/expr-common/src/operator.rs b/datafusion/expr-common/src/operator.rs
index 427069b326..a078a27f2a 100644
--- a/datafusion/expr-common/src/operator.rs
+++ b/datafusion/expr-common/src/operator.rs
@@ -255,9 +255,9 @@ impl Operator {
Operator::GtEq => Some(Operator::LtEq),
Operator::AtArrow => Some(Operator::ArrowAt),
Operator::ArrowAt => Some(Operator::AtArrow),
- Operator::IsDistinctFrom
- | Operator::IsNotDistinctFrom
- | Operator::Plus
+ Operator::IsDistinctFrom => Some(Operator::IsDistinctFrom),
+ Operator::IsNotDistinctFrom => Some(Operator::IsNotDistinctFrom),
+ Operator::Plus
| Operator::Minus
| Operator::Multiply
| Operator::Divide
diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs
index 76cf14be88..013a06812a 100644
--- a/datafusion/pruning/src/pruning_predicate.rs
+++ b/datafusion/pruning/src/pruning_predicate.rs
@@ -1155,8 +1155,16 @@ fn rewrite_expr_to_prunable(
Ok((left, reverse_operator(op)?, right))
} else if let Some(not) = column_expr.downcast_ref::<phys_expr::NotExpr>() {
// `!col = true` --> `col = !true`
- if op != Operator::Eq && op != Operator::NotEq {
- return plan_err!("Not with operator other than Eq / NotEq is not supported");
+ if !matches!(
+ op,
+ Operator::Eq
+ | Operator::NotEq
+ | Operator::IsDistinctFrom
+ | Operator::IsNotDistinctFrom
+ ) {
+ return plan_err!(
+ "Not with operator other than Eq / NotEq / IsDistinctFrom / IsNotDistinctFrom is not supported"
+ );
}
if not.arg().downcast_ref::<phys_expr::Column>().is_some() {
let left = Arc::clone(not.arg());
@@ -1193,6 +1201,8 @@ fn is_compare_op(op: Operator) -> bool {
| Operator::LtEq
| Operator::Gt
| Operator::GtEq
+ | Operator::IsDistinctFrom
+ | Operator::IsNotDistinctFrom
| Operator::LikeMatch
| Operator::NotLikeMatch
)
@@ -1616,45 +1626,14 @@ fn build_statistics_expr(
expr_builder: &mut PruningExpressionBuilder,
) -> Result<Arc<dyn PhysicalExpr>> {
let statistics_expr: Arc<dyn PhysicalExpr> = match expr_builder.op() {
- Operator::NotEq => {
- // column != literal => (min, max) = literal =>
- // !(min != literal && max != literal) ==>
- // min != literal || literal != max
- let min_column_expr = expr_builder.min_column_expr()?;
- let max_column_expr = expr_builder.max_column_expr()?;
- Arc::new(phys_expr::BinaryExpr::new(
- Arc::new(phys_expr::BinaryExpr::new(
- min_column_expr,
- Operator::NotEq,
- Arc::clone(expr_builder.scalar_expr()),
- )),
- Operator::Or,
- Arc::new(phys_expr::BinaryExpr::new(
- Arc::clone(expr_builder.scalar_expr()),
- Operator::NotEq,
- max_column_expr,
- )),
- ))
- }
+ Operator::NotEq => build_ne_statistics_expr(expr_builder)?,
Operator::Eq => {
// column = literal => (min, max) = literal => min <= literal && literal <= max
// (column / 2) = 4 => (column_min / 2) <= 4 && 4 <= (column_max / 2)
- let min_column_expr = expr_builder.min_column_expr()?;
- let max_column_expr = expr_builder.max_column_expr()?;
- Arc::new(phys_expr::BinaryExpr::new(
- Arc::new(phys_expr::BinaryExpr::new(
- min_column_expr,
- Operator::LtEq,
- Arc::clone(expr_builder.scalar_expr()),
- )),
- Operator::And,
- Arc::new(phys_expr::BinaryExpr::new(
- Arc::clone(expr_builder.scalar_expr()),
- Operator::LtEq,
- max_column_expr,
- )),
- ))
+ build_eq_statistics_expr(expr_builder)?
}
+ Operator::IsDistinctFrom => return build_is_distinct_from(expr_builder),
+ Operator::IsNotDistinctFrom => return build_is_not_distinct_from(expr_builder),
Operator::NotLikeMatch => build_not_like_match(expr_builder)?,
Operator::LikeMatch => build_like_match(expr_builder).ok_or_else(|| {
plan_datafusion_err!(
@@ -1704,6 +1683,126 @@ fn build_statistics_expr(
Ok(statistics_expr)
}
+fn binary_expr(
+ left: Arc<dyn PhysicalExpr>,
+ op: Operator,
+ right: Arc<dyn PhysicalExpr>,
+) -> Arc<dyn PhysicalExpr> {
+ Arc::new(phys_expr::BinaryExpr::new(left, op, right))
+}
+
+fn and_expr(
+ left: Arc<dyn PhysicalExpr>,
+ right: Arc<dyn PhysicalExpr>,
+) -> Arc<dyn PhysicalExpr> {
+ binary_expr(left, Operator::And, right)
+}
+
+fn or_expr(
+ left: Arc<dyn PhysicalExpr>,
+ right: Arc<dyn PhysicalExpr>,
+) -> Arc<dyn PhysicalExpr> {
+ binary_expr(left, Operator::Or, right)
+}
+
+fn build_eq_statistics_expr(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let min_column_expr = expr_builder.min_column_expr()?;
+ let max_column_expr = expr_builder.max_column_expr()?;
+ Ok(and_expr(
+ binary_expr(
+ min_column_expr,
+ Operator::LtEq,
+ Arc::clone(expr_builder.scalar_expr()),
+ ),
+ binary_expr(
+ Arc::clone(expr_builder.scalar_expr()),
+ Operator::LtEq,
+ max_column_expr,
+ ),
+ ))
+}
+
+fn build_ne_statistics_expr(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let min_column_expr = expr_builder.min_column_expr()?;
+ let max_column_expr = expr_builder.max_column_expr()?;
+ Ok(or_expr(
+ binary_expr(
+ min_column_expr,
+ Operator::NotEq,
+ Arc::clone(expr_builder.scalar_expr()),
+ ),
+ binary_expr(
+ Arc::clone(expr_builder.scalar_expr()),
+ Operator::NotEq,
+ max_column_expr,
+ ),
+ ))
+}
+
+fn column_has_nulls_expr(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(binary_expr(
+ expr_builder.null_count_column_expr()?,
+ Operator::Gt,
+ Arc::new(phys_expr::Literal::new(ScalarValue::UInt64(Some(0)))),
+ ))
+}
+
+fn column_has_non_nulls_expr(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(binary_expr(
+ expr_builder.null_count_column_expr()?,
+ Operator::NotEq,
+ expr_builder.row_count_column_expr()?,
+ ))
+}
+
+fn build_is_distinct_from(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let scalar_expr = Arc::clone(expr_builder.scalar_expr());
+
+ Ok(or_expr(
+ and_expr(
+ Arc::new(phys_expr::IsNullExpr::new(Arc::clone(&scalar_expr))),
+ column_has_non_nulls_expr(expr_builder)?,
+ ),
+ and_expr(
+ Arc::new(phys_expr::IsNotNullExpr::new(scalar_expr)),
+ or_expr(
+ column_has_nulls_expr(expr_builder)?,
+ build_ne_statistics_expr(expr_builder)?,
+ ),
+ ),
+ ))
+}
+
+fn build_is_not_distinct_from(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let scalar_expr = Arc::clone(expr_builder.scalar_expr());
+
+ Ok(or_expr(
+ and_expr(
+ Arc::new(phys_expr::IsNullExpr::new(Arc::clone(&scalar_expr))),
+ column_has_nulls_expr(expr_builder)?,
+ ),
+ and_expr(
+ Arc::new(phys_expr::IsNotNullExpr::new(scalar_expr)),
+ and_expr(
+ column_has_non_nulls_expr(expr_builder)?,
+ build_eq_statistics_expr(expr_builder)?,
+ ),
+ ),
+ ))
+}
+
/// returns the string literal of the scalar value if it is a string
fn unpack_string(s: &ScalarValue) -> Option<&str> {
s.try_as_str().flatten()
@@ -1920,19 +2019,11 @@ fn wrap_null_count_check_expr(
statistics_expr: Arc<dyn PhysicalExpr>,
expr_builder: &mut PruningExpressionBuilder,
) -> Result<Arc<dyn PhysicalExpr>> {
- // x_null_count != x_row_count
- let not_when_null_count_eq_row_count = Arc::new(phys_expr::BinaryExpr::new(
- expr_builder.null_count_column_expr()?,
- Operator::NotEq,
- expr_builder.row_count_column_expr()?,
- ));
-
// (x_null_count != x_row_count) AND (<statistics_expr>)
- Ok(Arc::new(phys_expr::BinaryExpr::new(
- not_when_null_count_eq_row_count,
- Operator::And,
+ Ok(and_expr(
+ column_has_non_nulls_expr(expr_builder)?,
statistics_expr,
- )))
+ ))
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -1960,7 +2051,7 @@ mod tests {
datatypes::TimeUnit,
};
use datafusion_expr::expr::InList;
- use datafusion_expr::{Expr, cast, is_null, try_cast};
+ use datafusion_expr::{BinaryExpr, Expr, cast, is_null, try_cast};
use datafusion_functions_nested::expr_fn::{array_has, make_array};
use datafusion_physical_expr::expressions::{
self as phys_expr, DynamicFilterPhysicalExpr,
@@ -3992,6 +4083,90 @@ mod tests {
);
}
+ #[test]
+ fn prune_int32_col_is_not_distinct_from() {
+ let (schema, statistics) = int32_setup();
+
+ // Without null counts, IS NOT DISTINCT FROM a non-null literal can
+ // still use min/max ranges, but unknown all-null containers must be kept.
+ let expected_ret = &[true, false, false, true, false];
+
+ prune_with_expr(
+ is_not_distinct_from(col("i"), lit(0)),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ // The operator is symmetric, so the scalar-left form should prune the
+ // same row groups.
+ prune_with_expr(
+ is_not_distinct_from(lit(0), col("i")),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ let statistics = statistics
+ .with_row_counts("i", vec![Some(10), Some(9), None, Some(4), Some(10)])
+ .with_null_counts("i", vec![Some(0), Some(1), None, Some(4), Some(0)]);
+
+ let expected_ret = &[true, false, false, false, false];
+ prune_with_expr(
+ is_not_distinct_from(col("i"), lit(0)),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ let expected_ret = &[false, true, true, true, false];
+ prune_with_expr(
+ is_not_distinct_from(col("i"), lit(ScalarValue::Int32(None))),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+ }
+
+ #[test]
+ fn prune_int32_col_is_distinct_from() {
+ let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, true)]));
+ let statistics = TestStatistics::new().with(
+ "i",
+ ContainerStats::new_i32(
+ vec![Some(0), Some(0), Some(5), None],
+ vec![Some(0), Some(2), Some(5), None],
+ )
+ .with_row_counts(vec![Some(2), Some(2), Some(2), Some(2)])
+ .with_null_counts(vec![Some(0), Some(0), Some(0), Some(2)]),
+ );
+
+ let expected_ret = &[false, true, true, true];
+ prune_with_expr(
+ is_distinct_from(col("i"), lit(0)),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ // The operator is symmetric, so the scalar-left form should prune the
+ // same row groups.
+ prune_with_expr(
+ is_distinct_from(lit(0), col("i")),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ let expected_ret = &[true, true, true, false];
+ prune_with_expr(
+ is_distinct_from(col("i"), lit(ScalarValue::Int32(None))),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+ }
+
#[test]
fn prune_int32_col_eq_zero_cast() {
let (schema, statistics) = int32_setup();
@@ -5381,6 +5556,22 @@ mod tests {
assert_eq!(result, expected);
}
+ fn is_not_distinct_from(left: Expr, right: Expr) -> Expr {
+ Expr::BinaryExpr(BinaryExpr::new(
+ Box::new(left),
+ Operator::IsNotDistinctFrom,
+ Box::new(right),
+ ))
+ }
+
+ fn is_distinct_from(left: Expr, right: Expr) -> Expr {
+ Expr::BinaryExpr(BinaryExpr::new(
+ Box::new(left),
+ Operator::IsDistinctFrom,
+ Box::new(right),
+ ))
+ }
+
fn test_build_predicate_expression(
expr: &Expr,
schema: &Schema,