Skip to content

Apache DataFusion SQL Query Engine

Support simplify not for physical expr (#18970)

xudong.w0490aec7e1c6
diff --git a/datafusion/physical-expr/src/simplifier/mod.rs b/datafusion/physical-expr/src/simplifier/mod.rs
index 80d6ee0a7b..d06629d03c 100644
--- a/datafusion/physical-expr/src/simplifier/mod.rs
+++ b/datafusion/physical-expr/src/simplifier/mod.rs
@@ -24,10 +24,13 @@ use datafusion_common::{
};
use std::sync::Arc;
-use crate::PhysicalExpr;
+use crate::{simplifier::not::simplify_not_expr, PhysicalExpr};
+pub mod not;
pub mod unwrap_cast;
+const MAX_LOOP_COUNT: usize = 5;
+
/// Simplifies physical expressions by applying various optimizations
///
/// This can be useful after adapting expressions from a table schema
@@ -48,7 +51,17 @@ impl<'a> PhysicalExprSimplifier<'a> {
&mut self,
expr: Arc<dyn PhysicalExpr>,
) -> Result<Arc<dyn PhysicalExpr>> {
- Ok(expr.rewrite(self)?.data)
+ let mut current_expr = expr;
+ let mut count = 0;
+ while count < MAX_LOOP_COUNT {
+ count += 1;
+ let result = current_expr.rewrite(self)?;
+ if !result.transformed {
+ return Ok(result.data);
+ }
+ current_expr = result.data;
+ }
+ Ok(current_expr)
}
}
@@ -56,24 +69,32 @@ impl<'a> TreeNodeRewriter for PhysicalExprSimplifier<'a> {
type Node = Arc<dyn PhysicalExpr>;
fn f_up(&mut self, node: Self::Node) -> Result<Transformed<Self::Node>> {
- // Apply unwrap cast optimization
#[cfg(test)]
let original_type = node.data_type(self.schema).unwrap();
- let unwrapped = unwrap_cast::unwrap_cast_in_comparison(node, self.schema)?;
+
+ // Apply NOT expression simplification first, then unwrap cast optimization
+ let rewritten =
+ simplify_not_expr(&node, self.schema)?.transform_data(|node| {
+ unwrap_cast::unwrap_cast_in_comparison(node, self.schema)
+ })?;
+
#[cfg(test)]
assert_eq!(
- unwrapped.data.data_type(self.schema).unwrap(),
+ rewritten.data.data_type(self.schema).unwrap(),
original_type,
"Simplified expression should have the same data type as the original"
);
- Ok(unwrapped)
+
+ Ok(rewritten)
}
}
#[cfg(test)]
mod tests {
use super::*;
- use crate::expressions::{col, lit, BinaryExpr, CastExpr, Literal, TryCastExpr};
+ use crate::expressions::{
+ col, in_list, lit, BinaryExpr, CastExpr, Literal, NotExpr, TryCastExpr,
+ };
use arrow::datatypes::{DataType, Field, Schema};
use datafusion_common::ScalarValue;
use datafusion_expr::Operator;
@@ -86,6 +107,42 @@ mod tests {
])
}
+ fn not_test_schema() -> Schema {
+ Schema::new(vec![
+ Field::new("a", DataType::Boolean, false),
+ Field::new("b", DataType::Boolean, false),
+ Field::new("c", DataType::Int32, false),
+ ])
+ }
+
+ /// Helper function to extract a Literal from a PhysicalExpr
+ fn as_literal(expr: &Arc<dyn PhysicalExpr>) -> &Literal {
+ expr.as_any()
+ .downcast_ref::<Literal>()
+ .unwrap_or_else(|| panic!("Expected Literal, got: {expr}"))
+ }
+
+ /// Helper function to extract a BinaryExpr from a PhysicalExpr
+ fn as_binary(expr: &Arc<dyn PhysicalExpr>) -> &BinaryExpr {
+ expr.as_any()
+ .downcast_ref::<BinaryExpr>()
+ .unwrap_or_else(|| panic!("Expected BinaryExpr, got: {expr}"))
+ }
+
+ /// Assert that simplifying `input` produces `expected`
+ fn assert_not_simplify(
+ simplifier: &mut PhysicalExprSimplifier,
+ input: Arc<dyn PhysicalExpr>,
+ expected: Arc<dyn PhysicalExpr>,
+ ) {
+ let result = simplifier.simplify(Arc::clone(&input)).unwrap();
+ assert_eq!(
+ &result,
+ &expected,
+ "Simplification should transform:\n input: {input}\n to: {expected}\n got: {result}"
+ );
+ }
+
#[test]
fn test_simplify() {
let schema = test_schema();
@@ -101,7 +158,7 @@ mod tests {
// Apply full simplification (uses TreeNodeRewriter)
let optimized = simplifier.simplify(binary_expr).unwrap();
- let optimized_binary = optimized.as_any().downcast_ref::<BinaryExpr>().unwrap();
+ let optimized_binary = as_binary(&optimized);
// Should be optimized to: c2 != INT64(99) (c2 is INT64, literal cast to match)
let left_expr = optimized_binary.left();
@@ -109,11 +166,7 @@ mod tests {
left_expr.as_any().downcast_ref::<CastExpr>().is_none()
&& left_expr.as_any().downcast_ref::<TryCastExpr>().is_none()
);
- let right_literal = optimized_binary
- .right()
- .as_any()
- .downcast_ref::<Literal>()
- .unwrap();
+ let right_literal = as_literal(optimized_binary.right());
assert_eq!(right_literal.value(), &ScalarValue::Int64(Some(99)));
}
@@ -138,14 +191,10 @@ mod tests {
// Apply simplification
let optimized = simplifier.simplify(or_expr).unwrap();
- let or_binary = optimized.as_any().downcast_ref::<BinaryExpr>().unwrap();
+ let or_binary = as_binary(&optimized);
// Verify left side: c1 > INT32(5)
- let left_binary = or_binary
- .left()
- .as_any()
- .downcast_ref::<BinaryExpr>()
- .unwrap();
+ let left_binary = as_binary(or_binary.left());
let left_left_expr = left_binary.left();
assert!(
left_left_expr.as_any().downcast_ref::<CastExpr>().is_none()
@@ -154,19 +203,11 @@ mod tests {
.downcast_ref::<TryCastExpr>()
.is_none()
);
- let left_literal = left_binary
- .right()
- .as_any()
- .downcast_ref::<Literal>()
- .unwrap();
+ let left_literal = as_literal(left_binary.right());
assert_eq!(left_literal.value(), &ScalarValue::Int32(Some(5)));
// Verify right side: c2 <= INT64(10)
- let right_binary = or_binary
- .right()
- .as_any()
- .downcast_ref::<BinaryExpr>()
- .unwrap();
+ let right_binary = as_binary(or_binary.right());
let right_left_expr = right_binary.left();
assert!(
right_left_expr
@@ -178,11 +219,276 @@ mod tests {
.downcast_ref::<TryCastExpr>()
.is_none()
);
- let right_literal = right_binary
- .right()
- .as_any()
- .downcast_ref::<Literal>()
- .unwrap();
+ let right_literal = as_literal(right_binary.right());
assert_eq!(right_literal.value(), &ScalarValue::Int64(Some(10)));
}
+
+ #[test]
+ fn test_double_negation_elimination() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(NOT(c > 5)) -> c > 5
+ let inner_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ col("c", &schema)?,
+ Operator::Gt,
+ lit(ScalarValue::Int32(Some(5))),
+ ));
+ let inner_not = Arc::new(NotExpr::new(Arc::clone(&inner_expr)));
+ let double_not: Arc<dyn PhysicalExpr> = Arc::new(NotExpr::new(inner_not));
+
+ let expected = inner_expr;
+ assert_not_simplify(&mut simplifier, double_not, expected);
+ Ok(())
+ }
+
+ #[test]
+ fn test_not_literal() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(TRUE) -> FALSE
+ let not_true = Arc::new(NotExpr::new(lit(ScalarValue::Boolean(Some(true)))));
+ let expected = lit(ScalarValue::Boolean(Some(false)));
+ assert_not_simplify(&mut simplifier, not_true, expected);
+
+ // NOT(FALSE) -> TRUE
+ let not_false = Arc::new(NotExpr::new(lit(ScalarValue::Boolean(Some(false)))));
+ let expected = lit(ScalarValue::Boolean(Some(true)));
+ assert_not_simplify(&mut simplifier, not_false, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_negate_comparison() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(c = 5) -> c != 5
+ let not_eq = Arc::new(NotExpr::new(Arc::new(BinaryExpr::new(
+ col("c", &schema)?,
+ Operator::Eq,
+ lit(ScalarValue::Int32(Some(5))),
+ ))));
+ let expected = Arc::new(BinaryExpr::new(
+ col("c", &schema)?,
+ Operator::NotEq,
+ lit(ScalarValue::Int32(Some(5))),
+ ));
+ assert_not_simplify(&mut simplifier, not_eq, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_demorgans_law_and() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(a AND b) -> NOT a OR NOT b
+ let and_expr = Arc::new(BinaryExpr::new(
+ col("a", &schema)?,
+ Operator::And,
+ col("b", &schema)?,
+ ));
+ let not_and: Arc<dyn PhysicalExpr> = Arc::new(NotExpr::new(and_expr));
+
+ let expected: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ Arc::new(NotExpr::new(col("a", &schema)?)),
+ Operator::Or,
+ Arc::new(NotExpr::new(col("b", &schema)?)),
+ ));
+ assert_not_simplify(&mut simplifier, not_and, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_demorgans_law_or() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(a OR b) -> NOT a AND NOT b
+ let or_expr = Arc::new(BinaryExpr::new(
+ col("a", &schema)?,
+ Operator::Or,
+ col("b", &schema)?,
+ ));
+ let not_or: Arc<dyn PhysicalExpr> = Arc::new(NotExpr::new(or_expr));
+
+ let expected: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ Arc::new(NotExpr::new(col("a", &schema)?)),
+ Operator::And,
+ Arc::new(NotExpr::new(col("b", &schema)?)),
+ ));
+ assert_not_simplify(&mut simplifier, not_or, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_demorgans_with_comparison_simplification() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(c = 1 AND c = 2) -> c != 1 OR c != 2
+ let eq1 = Arc::new(BinaryExpr::new(
+ col("c", &schema)?,
+ Operator::Eq,
+ lit(ScalarValue::Int32(Some(1))),
+ ));
+ let eq2 = Arc::new(BinaryExpr::new(
+ col("c", &schema)?,
+ Operator::Eq,
+ lit(ScalarValue::Int32(Some(2))),
+ ));
+ let and_expr = Arc::new(BinaryExpr::new(eq1, Operator::And, eq2));
+ let not_and: Arc<dyn PhysicalExpr> = Arc::new(NotExpr::new(and_expr));
+
+ let expected: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ Arc::new(BinaryExpr::new(
+ col("c", &schema)?,
+ Operator::NotEq,
+ lit(ScalarValue::Int32(Some(1))),
+ )),
+ Operator::Or,
+ Arc::new(BinaryExpr::new(
+ col("c", &schema)?,
+ Operator::NotEq,
+ lit(ScalarValue::Int32(Some(2))),
+ )),
+ ));
+ assert_not_simplify(&mut simplifier, not_and, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_not_of_not_and_not() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(NOT(a) AND NOT(b)) -> a OR b
+ let not_a = Arc::new(NotExpr::new(col("a", &schema)?));
+ let not_b = Arc::new(NotExpr::new(col("b", &schema)?));
+ let and_expr = Arc::new(BinaryExpr::new(not_a, Operator::And, not_b));
+ let not_and: Arc<dyn PhysicalExpr> = Arc::new(NotExpr::new(and_expr));
+
+ let expected: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ col("a", &schema)?,
+ Operator::Or,
+ col("b", &schema)?,
+ ));
+ assert_not_simplify(&mut simplifier, not_and, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_not_in_list() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(c IN (1, 2, 3)) -> c NOT IN (1, 2, 3)
+ let list = vec![
+ lit(ScalarValue::Int32(Some(1))),
+ lit(ScalarValue::Int32(Some(2))),
+ lit(ScalarValue::Int32(Some(3))),
+ ];
+ let in_list_expr = in_list(col("c", &schema)?, list.clone(), &false, &schema)?;
+ let not_in: Arc<dyn PhysicalExpr> = Arc::new(NotExpr::new(in_list_expr));
+
+ let expected = in_list(col("c", &schema)?, list, &true, &schema)?;
+ assert_not_simplify(&mut simplifier, not_in, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_not_not_in_list() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(c NOT IN (1, 2, 3)) -> c IN (1, 2, 3)
+ let list = vec![
+ lit(ScalarValue::Int32(Some(1))),
+ lit(ScalarValue::Int32(Some(2))),
+ lit(ScalarValue::Int32(Some(3))),
+ ];
+ let not_in_list_expr = in_list(col("c", &schema)?, list.clone(), &true, &schema)?;
+ let not_not_in: Arc<dyn PhysicalExpr> = Arc::new(NotExpr::new(not_in_list_expr));
+
+ let expected = in_list(col("c", &schema)?, list, &false, &schema)?;
+ assert_not_simplify(&mut simplifier, not_not_in, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_double_not_in_list() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // NOT(NOT(c IN (1, 2, 3))) -> c IN (1, 2, 3)
+ let list = vec![
+ lit(ScalarValue::Int32(Some(1))),
+ lit(ScalarValue::Int32(Some(2))),
+ lit(ScalarValue::Int32(Some(3))),
+ ];
+ let in_list_expr = in_list(col("c", &schema)?, list.clone(), &false, &schema)?;
+ let not_in = Arc::new(NotExpr::new(in_list_expr));
+ let double_not: Arc<dyn PhysicalExpr> = Arc::new(NotExpr::new(not_in));
+
+ let expected = in_list(col("c", &schema)?, list, &false, &schema)?;
+ assert_not_simplify(&mut simplifier, double_not, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_deeply_nested_not() -> Result<()> {
+ let schema = not_test_schema();
+ let mut simplifier = PhysicalExprSimplifier::new(&schema);
+
+ // Create a deeply nested NOT expression: NOT(NOT(NOT(...NOT(c > 5)...)))
+ // This tests that we don't get stack overflow with many nested NOTs.
+ // With recursive_protection enabled (default), this should work by
+ // automatically growing the stack as needed.
+ let inner_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ col("c", &schema)?,
+ Operator::Gt,
+ lit(ScalarValue::Int32(Some(5))),
+ ));
+
+ let mut expr = Arc::clone(&inner_expr);
+ // Create 200 layers of NOT to test deep recursion handling
+ for _ in 0..200 {
+ expr = Arc::new(NotExpr::new(expr));
+ }
+
+ // With 200 NOTs (even number), should simplify back to the original expression
+ let expected = inner_expr;
+ assert_not_simplify(&mut simplifier, Arc::clone(&expr), expected);
+
+ // Manually dismantle the deep input expression to avoid Stack Overflow on Drop
+ // If we just let `expr` go out of scope, Rust's recursive Drop will blow the stack
+ // even with recursive_protection, because Drop doesn't use the #[recursive] attribute.
+ // We peel off layers one by one to avoid deep recursion in Drop.
+ while let Some(not_expr) = expr.as_any().downcast_ref::<NotExpr>() {
+ // Clone the child (Arc increment).
+ // Now child has 2 refs: one in parent, one in `child`.
+ let child = Arc::clone(not_expr.arg());
+
+ // Reassign `expr` to `child`.
+ // This drops the old `expr` (Parent).
+ // Parent refcount -> 0, Parent is dropped.
+ // Parent drops its reference to Child.
+ // Child refcount decrements 2 -> 1.
+ // Child is NOT dropped recursively because we still hold it in `expr`
+ expr = child;
+ }
+
+ Ok(())
+ }
}
diff --git a/datafusion/physical-expr/src/simplifier/not.rs b/datafusion/physical-expr/src/simplifier/not.rs
new file mode 100644
index 0000000000..1ea969f58f
--- /dev/null
+++ b/datafusion/physical-expr/src/simplifier/not.rs
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Simplify NOT expressions in physical expressions
+//!
+//! This module provides optimizations for NOT expressions such as:
+//! - Double negation elimination: NOT(NOT(expr)) -> expr
+//! - NOT with binary comparisons: NOT(a = b) -> a != b
+//! - NOT with IN expressions: NOT(a IN (list)) -> a NOT IN (list)
+//! - De Morgan's laws: NOT(A AND B) -> NOT A OR NOT B
+//! - Constant folding: NOT(TRUE) -> FALSE, NOT(FALSE) -> TRUE
+//!
+//! This function is designed to work with TreeNodeRewriter's f_up traversal,
+//! which means children are already simplified when this function is called.
+//! The TreeNodeRewriter will automatically call this function repeatedly until
+//! no more transformations are possible.
+
+use std::sync::Arc;
+
+use arrow::datatypes::Schema;
+use datafusion_common::{tree_node::Transformed, Result, ScalarValue};
+use datafusion_expr::Operator;
+
+use crate::expressions::{in_list, lit, BinaryExpr, InListExpr, Literal, NotExpr};
+use crate::PhysicalExpr;
+
+/// Attempts to simplify NOT expressions by applying one level of transformation
+///
+/// This function applies a single simplification rule and returns. When used with
+/// TreeNodeRewriter, multiple passes will automatically be applied until no more
+/// transformations are possible.
+pub fn simplify_not_expr(
+ expr: &Arc<dyn PhysicalExpr>,
+ schema: &Schema,
+) -> Result<Transformed<Arc<dyn PhysicalExpr>>> {
+ // Check if this is a NOT expression
+ let not_expr = match expr.as_any().downcast_ref::<NotExpr>() {
+ Some(not_expr) => not_expr,
+ None => return Ok(Transformed::no(Arc::clone(expr))),
+ };
+
+ let inner_expr = not_expr.arg();
+
+ // Handle NOT(NOT(expr)) -> expr (double negation elimination)
+ if let Some(inner_not) = inner_expr.as_any().downcast_ref::<NotExpr>() {
+ return Ok(Transformed::yes(Arc::clone(inner_not.arg())));
+ }
+
+ // Handle NOT(literal) -> !literal
+ if let Some(literal) = inner_expr.as_any().downcast_ref::<Literal>() {
+ if let ScalarValue::Boolean(Some(val)) = literal.value() {
+ return Ok(Transformed::yes(lit(ScalarValue::Boolean(Some(!val)))));
+ }
+ if let ScalarValue::Boolean(None) = literal.value() {
+ return Ok(Transformed::yes(lit(ScalarValue::Boolean(None))));
+ }
+ }
+
+ // Handle NOT(IN list) -> NOT IN list
+ if let Some(in_list_expr) = inner_expr.as_any().downcast_ref::<InListExpr>() {
+ let negated = !in_list_expr.negated();
+ let new_in_list = in_list(
+ Arc::clone(in_list_expr.expr()),
+ in_list_expr.list().to_vec(),
+ &negated,
+ schema,
+ )?;
+ return Ok(Transformed::yes(new_in_list));
+ }
+
+ // Handle NOT(binary_expr)
+ if let Some(binary_expr) = inner_expr.as_any().downcast_ref::<BinaryExpr>() {
+ if let Some(negated_op) = binary_expr.op().negate() {
+ let new_binary = Arc::new(BinaryExpr::new(
+ Arc::clone(binary_expr.left()),
+ negated_op,
+ Arc::clone(binary_expr.right()),
+ ));
+ return Ok(Transformed::yes(new_binary));
+ }
+
+ // Handle De Morgan's laws for AND/OR
+ match binary_expr.op() {
+ Operator::And => {
+ // NOT(A AND B) -> NOT A OR NOT B
+ let not_left: Arc<dyn PhysicalExpr> =
+ Arc::new(NotExpr::new(Arc::clone(binary_expr.left())));
+ let not_right: Arc<dyn PhysicalExpr> =
+ Arc::new(NotExpr::new(Arc::clone(binary_expr.right())));
+ let new_binary =
+ Arc::new(BinaryExpr::new(not_left, Operator::Or, not_right));
+ return Ok(Transformed::yes(new_binary));
+ }
+ Operator::Or => {
+ // NOT(A OR B) -> NOT A AND NOT B
+ let not_left: Arc<dyn PhysicalExpr> =
+ Arc::new(NotExpr::new(Arc::clone(binary_expr.left())));
+ let not_right: Arc<dyn PhysicalExpr> =
+ Arc::new(NotExpr::new(Arc::clone(binary_expr.right())));
+ let new_binary =
+ Arc::new(BinaryExpr::new(not_left, Operator::And, not_right));
+ return Ok(Transformed::yes(new_binary));
+ }
+ _ => {}
+ }
+ }
+
+ // If no simplification possible, return the original expression
+ Ok(Transformed::no(Arc::clone(expr)))
+}