Apache DataFusion SQL Query Engine
feat: simplify contradictory ranges across AND/OR predicates (#26132)
xudong.w44e5403be20f
diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rsindex 4ec9747058..8eff81d935 100644--- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs+++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs@@ -35,7 +35,7 @@ use datafusion::scalar::ScalarValue;use datafusion_catalog::Session;use datafusion_common::cast::as_primitive_array;use datafusion_common::tree_node::TreeNodeRecursion;-use datafusion_common::{DataFusionError, internal_err, not_impl_err};+use datafusion_common::{DataFusionError, assert_batches_eq, internal_err, not_impl_err};use datafusion_expr::expr::{BinaryExpr, Cast};use datafusion_functions_aggregate::expr_fn::count;use datafusion_physical_expr::EquivalenceProperties;@@ -290,3 +290,78 @@ async fn test_filter_pushdown_results() -> Result<()> {assert_provider_row_count(2, 0).await?;Ok(())}++/// Fail at physical planning time if an impossible branch reaches its provider.+#[derive(Debug)]+struct RejectScanProvider {+ pushdown: TableProviderFilterPushDown,+}++#[async_trait]+impl TableProvider for RejectScanProvider {+ fn schema(&self) -> SchemaRef {+ Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)]))+ }++ fn table_type(&self) -> TableType {+ TableType::Base+ }++ async fn scan(+ &self,+ _state: &dyn Session,+ _projection: Option<&[usize]>,+ _filters: &[Expr],+ _limit: Option<usize>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ internal_err!("An unsatisfiable branch must not call scan()")+ }++ fn supports_filters_pushdown(+ &self,+ filters: &[&Expr],+ ) -> Result<Vec<TableProviderFilterPushDown>> {+ Ok(vec![self.pushdown.clone(); filters.len()])+ }+}++#[tokio::test]+async fn contradictory_disjunction_eliminates_provider_scan() -> Result<()> {+ for pushdown in [+ TableProviderFilterPushDown::Exact,+ TableProviderFilterPushDown::Inexact,+ TableProviderFilterPushDown::Unsupported,+ ] {+ let ctx = SessionContext::new();+ ctx.register_table("backfill", Arc::new(RejectScanProvider { pushdown }))?;+ let df = ctx+ .sql(+ "SELECT value FROM (+ SELECT CAST(20 AS BIGINT) AS value+ UNION ALL+ SELECT x AS value FROM backfill+ WHERE (x >= 1 AND x < 3) OR (x >= 5 AND x < 7)+ ) u WHERE value >= 10 ORDER BY value LIMIT 1",+ )+ .await?;+ let plan = df+ .clone()+ .into_optimized_plan()?+ .display_indent()+ .to_string();+ assert!(!plan.contains("TableScan: backfill"), "{plan}");+ assert!(!plan.contains("Union"), "{plan}");+ let batches = df.collect().await?;+ assert_batches_eq!(+ [+ "+-------+",+ "| value |",+ "+-------+",+ "| 20 |",+ "+-------+"+ ],+ &batches+ );+ }+ Ok(())+}diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex 999cddfb62..2fb2c953fe 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -52,8 +52,8 @@ use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion,};use datafusion_common::{- Column, DFSchema, Result, assert_eq_or_internal_err, internal_err, plan_err,- qualified_name,+ Column, DFSchema, Result, ScalarValue, assert_eq_or_internal_err, internal_err,+ plan_err, qualified_name,};use datafusion_expr::expr::WindowFunction;use datafusion_expr::expr_rewriter::replace_col;@@ -62,7 +62,7 @@ use datafusion_expr::utils::{conjunction, expr_to_columns, split_conjunction, split_conjunction_owned,};use datafusion_expr::{- BinaryExpr, Distinct, Expr, Filter, Operator, Projection,+ BinaryExpr, Distinct, EmptyRelation, Expr, Filter, Operator, Projection,TableProviderFilterPushDown, and, or,};@@ -889,8 +889,25 @@ impl OptimizerRule for PushDownFilter {// predicates on rows that have already been filtered out.let (new_predicates, reorder_changed) = reorder_predicates(new_predicates);- let count_changed = old_predicate_len != new_predicates.len();- if count_changed || reorder_changed {+ // An OR branch can change without changing the number of conjuncts.+ // Ignore permutations from grouping comparisons by column: the cost+ // reordering above decides when to change predicate evaluation order.+ let predicates_changed = if old_predicate_len != new_predicates.len() {+ true+ } else {+ let old_predicates = split_conjunction(&filter.predicate);+ new_predicates+ .iter()+ .zip(&old_predicates)+ .any(|(new, old)| new != *old)+ && {+ let new_predicates: IndexSet<_> = new_predicates.iter().collect();+ old_predicates+ .iter()+ .any(|old| !new_predicates.contains(old))+ }+ };+ if predicates_changed || reorder_changed {let Some(new_predicate) = conjunction(new_predicates) else {// new_predicates is empty - remove the filter entirely// Return the child plan without the filter@@ -899,14 +916,32 @@ impl OptimizerRule for PushDownFilter {filter.predicate = new_predicate;}+ // A provider accepting Exact pushdown would absorb FALSE into its scan,+ // hiding the contradiction from EliminateFilter and still invoking scan().+ if matches!(+ filter.predicate,+ Expr::Literal(ScalarValue::Boolean(Some(false)), _)+ ) {+ return Ok(Transformed::yes(LogicalPlan::EmptyRelation(+ EmptyRelation {+ produce_one_row: false,+ schema: Arc::clone(filter.input.schema()),+ },+ )));+ }+// If the child has a fetch (limit) or skip (offset), pushing a filter// below it would change semantics: the limit/offset should apply before// the filter, not after.if filter.input.fetch()?.is_some() || filter.input.skip()?.is_some() {- return Ok(Transformed::no(LogicalPlan::Filter(filter)));+ return Ok(Transformed::new(+ LogicalPlan::Filter(filter),+ predicates_changed || reorder_changed,+ TreeNodeRecursion::Continue,+ ));}- match Arc::unwrap_or_clone(filter.input) {+ let result = match Arc::unwrap_or_clone(filter.input) {LogicalPlan::Filter(mut child_filter) => {// Child filters first to preserve execution order.// Use IndexSet to remove duplicates while preserving predicate order.@@ -1025,7 +1060,11 @@ impl OptimizerRule for PushDownFilter {// If no non-unnest predicates exist, early returnif non_unnest_predicates.is_empty() {filter.input = Arc::new(LogicalPlan::Unnest(unnest));- return Ok(Transformed::no(LogicalPlan::Filter(filter)));+ return Ok(Transformed::new(+ LogicalPlan::Filter(filter),+ predicates_changed || reorder_changed,+ TreeNodeRecursion::Continue,+ ));}// Push down non-unnest filter predicate@@ -1305,7 +1344,11 @@ impl OptimizerRule for PushDownFilter {.all(|res| res == &TableProviderFilterPushDown::Unsupported){filter.input = Arc::new(LogicalPlan::TableScan(scan));- return Ok(Transformed::no(LogicalPlan::Filter(filter)));+ return Ok(Transformed::new(+ LogicalPlan::Filter(filter),+ predicates_changed || reorder_changed,+ TreeNodeRecursion::Continue,+ ));}// Compose scan filters from non-volatile filters of `Exact` or `Inexact` pushdown type@@ -1331,7 +1374,11 @@ impl OptimizerRule for PushDownFilter {&& scan.filters == new_scan_filters{filter.input = Arc::new(LogicalPlan::TableScan(scan));- return Ok(Transformed::no(LogicalPlan::Filter(filter)));+ return Ok(Transformed::new(+ LogicalPlan::Filter(filter),+ predicates_changed || reorder_changed,+ TreeNodeRecursion::Continue,+ ));} else {scan.filters = new_scan_filters;}@@ -1356,7 +1403,11 @@ impl OptimizerRule for PushDownFilter {// so we return the original Filter unchanged.if extension_plan.node.inputs().is_empty() {filter.input = Arc::new(LogicalPlan::Extension(extension_plan));- return Ok(Transformed::no(LogicalPlan::Filter(filter)));+ return Ok(Transformed::new(+ LogicalPlan::Filter(filter),+ predicates_changed || reorder_changed,+ TreeNodeRecursion::Continue,+ ));}let prevent_cols =extension_plan.node.prevent_predicate_push_down_columns();@@ -1373,7 +1424,11 @@ impl OptimizerRule for PushDownFilter {// all predicates are kept, no changes neededif predicate_push_or_keep.iter().all(|&x| !x) {filter.input = Arc::new(LogicalPlan::Extension(extension_plan));- return Ok(Transformed::no(LogicalPlan::Filter(filter)));+ return Ok(Transformed::new(+ LogicalPlan::Filter(filter),+ predicates_changed || reorder_changed,+ TreeNodeRecursion::Continue,+ ));}// going to push some predicates down, so split the predicates@@ -1414,7 +1469,11 @@ impl OptimizerRule for PushDownFilter {filter.input = Arc::new(child);Ok(Transformed::no(LogicalPlan::Filter(filter)))}- }+ };+ result.map(|mut result| {+ result.transformed |= predicates_changed || reorder_changed;+ result+ })}}@@ -4653,14 +4712,8 @@ mod tests {TestUserNode",);- // Check that the filter is pushed down to the user-defined node- assert_optimized_plan_equal!(- plan,- @r"- Filter: Boolean(false)- TestUserNode- "- )+ // An impossible filter eliminates even a leaf extension before pushdown.+ assert_optimized_plan_equal!(plan, @"EmptyRelation: rows=0")}/// Test that filters are NOT pushed through MoveTowardsLeafNodes projections.diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rsindex 35f0cc513d..5309deb34a 100644--- a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs+++ b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs@@ -29,11 +29,18 @@use super::utils::{is_false, is_null};use datafusion_common::utils::normalize_float_zero_scalar;-use datafusion_common::{Column, Result, ScalarValue, internal_err};+use datafusion_common::{Column, DataFusionError, Result, ScalarValue, internal_err};+use datafusion_expr::utils::{+ conjunction, disjunction, split_binary_owned, split_conjunction_owned,+};use datafusion_expr::{BinaryExpr, Expr, Operator, lit};+use indexmap::IndexSet;use std::cmp::Ordering;use std::collections::BTreeMap;+const MAX_DISJUNCTION_WORK: usize = 4096;+const MAX_DISJUNCTION_DEPTH: usize = 32;+/// Simplifies a list of predicates by removing redundancies.////// This function takes a vector of predicate expressions and groups them by the column they reference.@@ -41,6 +48,9 @@ use std::collections::BTreeMap;/// are analyzed to remove redundant conditions. For instance, `x > 5 AND x > 6` is simplified to/// `x > 6`. Predicates that contradict each other, such as `x > 6 AND x < 5`, reduce the whole/// conjunction to `false`. Other predicates that do not fit this pattern are retained as-is.+/// Comparisons in an enclosing conjunction also constrain each branch of an `OR`:+/// `x >= 10 AND (x < 3 OR x < 7)` cannot be true. This simplification preserves+/// which rows pass a filter, rather than Boolean values in projection expressions.////// # Arguments/// * `predicates` - A vector of `Expr` representing the predicates to simplify.@@ -48,6 +58,117 @@ use std::collections::BTreeMap;/// # Returns/// A `Result` containing a vector of simplified `Expr` predicates.pub fn simplify_predicates(predicates: Vec<Expr>) -> Result<Vec<Expr>> {+ let mut remaining = MAX_DISJUNCTION_WORK;+ simplify_with_context(predicates, &[], 0, &mut remaining)+}++/// Only descend through positive AND/OR expressions. In particular, replacing a+/// nullable, unsatisfiable predicate with false below NOT would change its truth set.+/// Keep assumptions outside the expressions they simplify, and never distribute+/// conjunctions over disjunctions. The budget bounds repeated comparison work.+fn simplify_with_context(+ predicates: Vec<Expr>,+ assumptions: &[Expr],+ depth: usize,+ remaining: &mut usize,+) -> Result<Vec<Expr>> {+ let predicates = simplify_conjunction(predicates)?;+ if predicates.iter().any(is_false) {+ return Ok(always_false());+ }+ let has_disjunction = predicates.iter().any(|expr| {+ matches!(+ expr,+ Expr::BinaryExpr(BinaryExpr {+ op: Operator::Or,+ ..+ })+ )+ });+ if assumptions.is_empty() && !has_disjunction {+ return Ok(predicates);+ }+ let cost = assumptions.len().saturating_add(predicates.len());+ if depth >= MAX_DISJUNCTION_DEPTH || cost > *remaining {+ return Ok(predicates);+ }+ *remaining -= cost;++ let context = assumptions+ .iter()+ .chain(predicates.iter().filter(|expr| is_column_comparison(expr)))+ .cloned()+ .collect();+ let context = simplify_conjunction(context)?;+ if context.iter().any(is_false) {+ return Ok(always_false());+ }++ let mut result = Vec::with_capacity(predicates.len());+ for predicate in predicates {+ let predicate = simplify_disjunction(predicate, &context, depth, remaining)?;+ if is_false(&predicate) {+ return Ok(always_false());+ }+ result.push(predicate);+ }+ Ok(result)+}++fn simplify_disjunction(+ predicate: Expr,+ assumptions: &[Expr],+ depth: usize,+ remaining: &mut usize,+) -> Result<Expr> {+ if !matches!(+ predicate,+ Expr::BinaryExpr(BinaryExpr {+ op: Operator::Or,+ ..+ })+ ) {+ return Ok(predicate);+ }+ let mut simplify_branch = |branch| {+ let original = split_conjunction_owned(branch);+ let predicates =+ simplify_with_context(original.clone(), assumptions, depth + 1, remaining)?;+ // Grouping comparisons by column can merely permute an AND branch.+ // Preserve its evaluation order when no predicates were simplified.+ let predicates = if original.len() == predicates.len()+ && original != predicates+ && original.iter().collect::<IndexSet<_>>()+ == predicates.iter().collect::<IndexSet<_>>()+ {+ original+ } else {+ predicates+ };+ Ok::<_, DataFusionError>(conjunction(predicates).unwrap_or_else(|| lit(true)))+ };+ // Flatten OR chains so the depth limit counts AND/OR nesting, not disjuncts.+ let mut branches = Vec::new();+ for branch in split_binary_owned(predicate, Operator::Or) {+ let branch = simplify_branch(branch)?;+ if !is_false(&branch) {+ branches.push(branch);+ }+ }+ Ok(disjunction(branches).unwrap_or_else(|| lit(false)))+}++fn is_column_comparison(expr: &Expr) -> bool {+ matches!(expr,+ Expr::BinaryExpr(BinaryExpr { left, op, right })+ if matches!(op, Operator::Eq | Operator::NotEq | Operator::Lt+ | Operator::LtEq | Operator::Gt | Operator::GtEq)+ && ((matches!(left.as_ref(), Expr::Column(_)) && right.as_literal().is_some())+ || (matches!(right.as_ref(), Expr::Column(_)) && left.as_literal().is_some()))+ )+}++fn simplify_conjunction(predicates: Vec<Expr>) -> Result<Vec<Expr>> {// Early return for simple casesif predicates.len() <= 1 {return Ok(predicates);@@ -392,6 +513,33 @@ mod tests {use arrow::datatypes::DataType;use datafusion_expr::{cast, col, lit};+ #[test]+ fn test_context_budget_exhaustion_is_conservative() -> Result<()> {+ let bound = col("x").gt_eq(lit(10));+ let disjunction = col("x").lt(lit(3)).or(col("x").lt(lit(7)));+ let predicates = vec![bound.clone(), disjunction.clone()];+ let result = simplify_with_context(predicates.clone(), &[], 0, &mut 0)?;+ assert_eq!(result.len(), 2);+ assert!(result.contains(&bound));+ assert!(result.contains(&disjunction));+ assert_eq!(simplify_predicates(predicates)?, vec![lit(false)]);+ Ok(())+ }++ #[test]+ fn test_context_depth_limit_is_conservative() -> Result<()> {+ let disjunction = col("x").lt(lit(3)).or(col("x").lt(lit(7)));+ let mut remaining = MAX_DISJUNCTION_WORK;+ let result = simplify_with_context(+ vec![disjunction.clone()],+ &[col("x").gt_eq(lit(10))],+ MAX_DISJUNCTION_DEPTH,+ &mut remaining,+ )?;+ assert_eq!(result, vec![disjunction]);+ Ok(())+ }+#[test]fn test_simplify_predicates_with_cast() {// Test that predicates on cast expressions are not grouped with predicates on the raw columndiff --git a/datafusion/sqllogictest/test_files/simplify_predicate_disjunctions.slt b/datafusion/sqllogictest/test_files/simplify_predicate_disjunctions.sltnew file mode 100644index 0000000000..0593d90641--- /dev/null+++ b/datafusion/sqllogictest/test_files/simplify_predicate_disjunctions.slt@@ -0,0 +1,344 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Contextual range reasoning must preserve filter results, including nullable inputs.+statement ok+SET datafusion.explain.logical_plan_only = true;++statement ok+CREATE TABLE disjoint_ranges (x BIGINT, y BIGINT) AS VALUES+ (NULL, 1), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5),+ (6, 6), (7, 7), (10, 10), (11, 11);++# Neither OR branch can be true under the outer lower bound.+query TT+EXPLAIN SELECT x FROM disjoint_ranges+WHERE x >= 10 AND ((x >= 1 AND x < 3) OR (x >= 5 AND x < 7));+----+logical_plan EmptyRelation: rows=0++# A flat chain of 120 disjoint ranges must not consume the AND/OR nesting limit.+# Every range is below the outer bound, so all branches can be removed.+statement ok+CREATE VIEW many_disjoint_ranges AS+SELECT x FROM disjoint_ranges WHERE+ (x >= 0 AND x < 2)+ OR (x >= 10 AND x < 12)+ OR (x >= 20 AND x < 22)+ OR (x >= 30 AND x < 32)+ OR (x >= 40 AND x < 42)+ OR (x >= 50 AND x < 52)+ OR (x >= 60 AND x < 62)+ OR (x >= 70 AND x < 72)+ OR (x >= 80 AND x < 82)+ OR (x >= 90 AND x < 92)+ OR (x >= 100 AND x < 102)+ OR (x >= 110 AND x < 112)+ OR (x >= 120 AND x < 122)+ OR (x >= 130 AND x < 132)+ OR (x >= 140 AND x < 142)+ OR (x >= 150 AND x < 152)+ OR (x >= 160 AND x < 162)+ OR (x >= 170 AND x < 172)+ OR (x >= 180 AND x < 182)+ OR (x >= 190 AND x < 192)+ OR (x >= 200 AND x < 202)+ OR (x >= 210 AND x < 212)+ OR (x >= 220 AND x < 222)+ OR (x >= 230 AND x < 232)+ OR (x >= 240 AND x < 242)+ OR (x >= 250 AND x < 252)+ OR (x >= 260 AND x < 262)+ OR (x >= 270 AND x < 272)+ OR (x >= 280 AND x < 282)+ OR (x >= 290 AND x < 292)+ OR (x >= 300 AND x < 302)+ OR (x >= 310 AND x < 312)+ OR (x >= 320 AND x < 322)+ OR (x >= 330 AND x < 332)+ OR (x >= 340 AND x < 342)+ OR (x >= 350 AND x < 352)+ OR (x >= 360 AND x < 362)+ OR (x >= 370 AND x < 372)+ OR (x >= 380 AND x < 382)+ OR (x >= 390 AND x < 392)+ OR (x >= 400 AND x < 402)+ OR (x >= 410 AND x < 412)+ OR (x >= 420 AND x < 422)+ OR (x >= 430 AND x < 432)+ OR (x >= 440 AND x < 442)+ OR (x >= 450 AND x < 452)+ OR (x >= 460 AND x < 462)+ OR (x >= 470 AND x < 472)+ OR (x >= 480 AND x < 482)+ OR (x >= 490 AND x < 492)+ OR (x >= 500 AND x < 502)+ OR (x >= 510 AND x < 512)+ OR (x >= 520 AND x < 522)+ OR (x >= 530 AND x < 532)+ OR (x >= 540 AND x < 542)+ OR (x >= 550 AND x < 552)+ OR (x >= 560 AND x < 562)+ OR (x >= 570 AND x < 572)+ OR (x >= 580 AND x < 582)+ OR (x >= 590 AND x < 592)+ OR (x >= 600 AND x < 602)+ OR (x >= 610 AND x < 612)+ OR (x >= 620 AND x < 622)+ OR (x >= 630 AND x < 632)+ OR (x >= 640 AND x < 642)+ OR (x >= 650 AND x < 652)+ OR (x >= 660 AND x < 662)+ OR (x >= 670 AND x < 672)+ OR (x >= 680 AND x < 682)+ OR (x >= 690 AND x < 692)+ OR (x >= 700 AND x < 702)+ OR (x >= 710 AND x < 712)+ OR (x >= 720 AND x < 722)+ OR (x >= 730 AND x < 732)+ OR (x >= 740 AND x < 742)+ OR (x >= 750 AND x < 752)+ OR (x >= 760 AND x < 762)+ OR (x >= 770 AND x < 772)+ OR (x >= 780 AND x < 782)+ OR (x >= 790 AND x < 792)+ OR (x >= 800 AND x < 802)+ OR (x >= 810 AND x < 812)+ OR (x >= 820 AND x < 822)+ OR (x >= 830 AND x < 832)+ OR (x >= 840 AND x < 842)+ OR (x >= 850 AND x < 852)+ OR (x >= 860 AND x < 862)+ OR (x >= 870 AND x < 872)+ OR (x >= 880 AND x < 882)+ OR (x >= 890 AND x < 892)+ OR (x >= 900 AND x < 902)+ OR (x >= 910 AND x < 912)+ OR (x >= 920 AND x < 922)+ OR (x >= 930 AND x < 932)+ OR (x >= 940 AND x < 942)+ OR (x >= 950 AND x < 952)+ OR (x >= 960 AND x < 962)+ OR (x >= 970 AND x < 972)+ OR (x >= 980 AND x < 982)+ OR (x >= 990 AND x < 992)+ OR (x >= 1000 AND x < 1002)+ OR (x >= 1010 AND x < 1012)+ OR (x >= 1020 AND x < 1022)+ OR (x >= 1030 AND x < 1032)+ OR (x >= 1040 AND x < 1042)+ OR (x >= 1050 AND x < 1052)+ OR (x >= 1060 AND x < 1062)+ OR (x >= 1070 AND x < 1072)+ OR (x >= 1080 AND x < 1082)+ OR (x >= 1090 AND x < 1092)+ OR (x >= 1100 AND x < 1102)+ OR (x >= 1110 AND x < 1112)+ OR (x >= 1120 AND x < 1122)+ OR (x >= 1130 AND x < 1132)+ OR (x >= 1140 AND x < 1142)+ OR (x >= 1150 AND x < 1152)+ OR (x >= 1160 AND x < 1162)+ OR (x >= 1170 AND x < 1172)+ OR (x >= 1180 AND x < 1182)+ OR (x >= 1190 AND x < 1192);++query TT+EXPLAIN SELECT x FROM many_disjoint_ranges WHERE x >= 5000;+----+logical_plan EmptyRelation: rows=0++# Pruning the entire chain must not depend on repeated optimizer passes.+statement ok+SET datafusion.optimizer.max_passes = 1;++query TT+EXPLAIN SELECT x FROM many_disjoint_ranges WHERE x >= 5000;+----+logical_plan+01)SubqueryAlias: many_disjoint_ranges+02)--Projection: disjoint_ranges.x+03)----EmptyRelation: rows=0++# A surviving range still returns its matching rows and excludes NULL.+query I+SELECT x FROM many_disjoint_ranges WHERE x >= 10 ORDER BY x;+----+10+11++statement ok+SET datafusion.optimizer.max_passes = 3;++# The request lies in the gap between intervals.+query TT+EXPLAIN SELECT x FROM disjoint_ranges+WHERE x >= 3 AND x < 5 AND ((x >= 1 AND x < 3) OR (x >= 5 AND x < 7));+----+logical_plan EmptyRelation: rows=0++# Removing one impossible disjunct must retain the outer lower bound (x=5 fails).+query I+SELECT x FROM disjoint_ranges+WHERE x >= 6 AND ((x >= 1 AND x < 3) OR (x >= 5 AND x < 11)) ORDER BY x;+----+6+7+10++query TT+EXPLAIN SELECT x FROM disjoint_ranges+WHERE x >= 6 AND ((x >= 1 AND x < 3) OR (x >= 5 AND x < 11));+----+logical_plan+01)Filter: disjoint_ranges.x >= Int64(6) AND disjoint_ranges.x < Int64(11)+02)--TableScan: disjoint_ranges projection=[x]++# The upper endpoint remains possible when both bounds include it.+query I+SELECT x FROM disjoint_ranges+WHERE 7 <= x AND ((x >= 1 AND x < 3) OR (x >= 5 AND x <= 7)) ORDER BY x;+----+7++# All branches can contradict the context, including equalities and inequalities.+query TT+EXPLAIN SELECT x FROM disjoint_ranges WHERE x = 6 AND (x < 3 OR x != 6);+----+logical_plan EmptyRelation: rows=0++# Inner conjunctions contribute assumptions only to their descendants.+query I+SELECT x FROM disjoint_ranges+WHERE x >= 3 AND ((x < 7 AND (x < 3 OR x >= 6)) OR x >= 10) ORDER BY x;+----+6+10+11++# An unrelated column can satisfy the disjunction.+query I+SELECT x FROM disjoint_ranges WHERE x >= 10 AND (x < 3 OR y = 10) ORDER BY x;+----+10++# Do not infer bounds through unsupported arithmetic/casts or leak them across OR.+query I+SELECT x FROM disjoint_ranges+WHERE x >= 10 AND (x < 3 OR x - 10 = 1) ORDER BY x;+----+11++query I+SELECT x FROM disjoint_ranges WHERE (x >= 10 AND x < 3) OR x = 2 ORDER BY x;+----+2++# NULL results under NOT must remain NULL; only positive AND/OR contexts are visited.+query I+SELECT x FROM disjoint_ranges+WHERE NOT (x >= 10 AND (x < 3 OR x < 7)) ORDER BY x;+----+1+2+3+4+5+6+7+10+11++# A nullable Boolean projection must not be rewritten using filter-only semantics.+query B+SELECT x >= 10 AND (x < 3 OR x < 7) FROM disjoint_ranges WHERE x IS NULL;+----+NULL++# Precise timestamp bounds: two disjoint months and a request in a later year.+statement ok+CREATE TABLE disjoint_times (ts TIMESTAMP);++query TT+EXPLAIN SELECT ts FROM disjoint_times+WHERE ts >= TIMESTAMP '2026-01-01'+AND ((ts >= TIMESTAMP '2022-05-01' AND ts < TIMESTAMP '2022-06-01')+ OR (ts >= TIMESTAMP '2022-08-01' AND ts < TIMESTAMP '2022-09-01'));+----+logical_plan EmptyRelation: rows=0++# Nanosecond precision at the boundary must distinguish < from <=.+query TT+EXPLAIN SELECT ts FROM disjoint_times+WHERE ts >= TIMESTAMP '2022-09-01 00:00:00.000000001'+AND (ts < TIMESTAMP '2022-09-01 00:00:00.000000001'+ OR ts <= TIMESTAMP '2022-09-01 00:00:00.000000000');+----+logical_plan EmptyRelation: rows=0++# Time zone offsets denote instants; equal instants at an exclusive bound conflict.+statement ok+CREATE TABLE zoned_times AS+SELECT arrow_cast('2022-09-01T00:00:00-04:00',+ 'Timestamp(Nanosecond, Some("America/New_York"))') AS ts;++query TT+EXPLAIN SELECT ts FROM zoned_times+WHERE ts >= arrow_cast('2022-09-01T04:00:00Z',+ 'Timestamp(Nanosecond, Some("America/New_York"))')+AND (ts < arrow_cast('2022-09-01T00:00:00-04:00',+ 'Timestamp(Nanosecond, Some("America/New_York"))')+ OR ts < arrow_cast('2022-06-01T00:00:00-04:00',+ 'Timestamp(Nanosecond, Some("America/New_York"))'));+----+logical_plan EmptyRelation: rows=0++# A type conversion can change values, so its result must not be treated as x itself.+query I+SELECT x FROM disjoint_ranges+WHERE x >= 10 AND (x < 3 OR CAST(x AS VARCHAR) < '2') ORDER BY x;+----+10+11++# Outer aliases, ordering and limit must not prevent removal of the empty union arm.+query TT+EXPLAIN SELECT * FROM (+ SELECT CAST(20 AS BIGINT) AS value+ UNION ALL+ SELECT x AS value FROM disjoint_ranges+ WHERE (x >= 1 AND x < 3) OR (x >= 5 AND x < 7)+) u WHERE value >= 10 ORDER BY value LIMIT 1;+----+logical_plan+01)Sort: u.value ASC NULLS LAST, fetch=1+02)--SubqueryAlias: u+03)----Projection: Int64(20) AS value+04)------EmptyRelation: rows=1++query I+SELECT * FROM (+ SELECT CAST(20 AS BIGINT) AS value+ UNION ALL+ SELECT x AS value FROM disjoint_ranges+ WHERE (x >= 1 AND x < 3) OR (x >= 5 AND x < 7)+) u WHERE value >= 10 ORDER BY value LIMIT 1;+----+20++statement ok+SET datafusion.explain.logical_plan_only = false;diff --git a/datafusion/sqllogictest/test_files/simplify_predicates.slt b/datafusion/sqllogictest/test_files/simplify_predicates.sltindex 3ec4df3f42..7345b56fc4 100644--- a/datafusion/sqllogictest/test_files/simplify_predicates.slt+++ b/datafusion/sqllogictest/test_files/simplify_predicates.slt@@ -284,7 +284,7 @@ WHERE (int_col > 5 AND int_col > 6)OR (float_col < 10 AND float_col < 8);----logical_plan-01)Filter: test_data.int_col > Int32(5) AND test_data.int_col > Int32(6) OR test_data.float_col < Float32(10) AND test_data.float_col < Float32(8)+01)Filter: test_data.int_col > Int32(6) OR test_data.float_col < Float32(8)02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col]