Apache DataFusion SQL Query Engine
Correct implementations of `NullableInterval::and` and `NullableInterval::or`. (#18625)
Pepijn Van Eeckhoudte661b33ee3b9
diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rsindex bbf14da1eb..942f3d48c6 100644--- a/datafusion/expr-common/src/interval_arithmetic.rs+++ b/datafusion/expr-common/src/interval_arithmetic.rs@@ -1738,6 +1738,44 @@ impl From<ScalarValue> for NullableInterval {}impl NullableInterval {+ /// An interval that only contains 'false'.+ pub const FALSE: Self = NullableInterval::NotNull {+ values: Interval::CERTAINLY_FALSE,+ };++ /// An interval that only contains 'true'.+ pub const TRUE: Self = NullableInterval::NotNull {+ values: Interval::CERTAINLY_TRUE,+ };++ /// An interval that only contains 'unknown' (boolean null).+ pub const UNKNOWN: Self = NullableInterval::Null {+ datatype: DataType::Boolean,+ };++ /// An interval that only contains 'true' and 'false'.+ /// This interval is equivalent to [Interval::UNCERTAIN].+ pub const TRUE_OR_FALSE: Self = NullableInterval::NotNull {+ values: Interval::UNCERTAIN,+ };++ /// An interval that only contains 'true' and 'unknown'.+ pub const TRUE_OR_UNKNOWN: Self = NullableInterval::MaybeNull {+ values: Interval::CERTAINLY_TRUE,+ };++ /// An interval that only contains 'false' and 'unknown'.+ pub const FALSE_OR_UNKNOWN: Self = NullableInterval::MaybeNull {+ values: Interval::CERTAINLY_FALSE,+ };++ /// An interval that contains all possible boolean values: 'true', 'false' and 'unknown'.+ ///+ /// Note that this is different from [Interval::UNCERTAIN] which only contains 'true' and 'false'.+ pub const UNCERTAIN: Self = NullableInterval::MaybeNull {+ values: Interval::UNCERTAIN,+ };+/// Get the values interval, or None if this interval is definitely null.pub fn values(&self) -> Option<&Interval> {match self {@@ -1756,27 +1794,31 @@ impl NullableInterval {/// Return true if the value is definitely true (and not null).pub fn is_certainly_true(&self) -> bool {- match self {- Self::Null { .. } | Self::MaybeNull { .. } => false,- Self::NotNull { values } => values == &Interval::CERTAINLY_TRUE,- }+ self == &Self::TRUE}/// Return true if the value is definitely false (and not null).pub fn is_certainly_false(&self) -> bool {- match self {- Self::Null { .. } => false,- Self::MaybeNull { .. } => false,- Self::NotNull { values } => values == &Interval::CERTAINLY_FALSE,- }+ self == &Self::FALSE+ }++ /// Return true if the value is definitely null (and not true or false).+ pub fn is_certainly_unknown(&self) -> bool {+ self == &Self::UNKNOWN}/// Perform logical negation on a boolean nullable interval.fn not(&self) -> Result<Self> {match self {- Self::Null { datatype } => Ok(Self::Null {- datatype: datatype.clone(),- }),+ Self::Null { datatype } => {+ if datatype == &DataType::Boolean {+ Ok(Self::UNKNOWN)+ } else {+ internal_err!(+ "Cannot apply logical negation to a non-boolean interval"+ )+ }+ }Self::MaybeNull { values } => Ok(Self::MaybeNull {values: values.not()?,}),@@ -1786,6 +1828,62 @@ impl NullableInterval {}}+ /// Compute the logical conjunction of this (boolean) interval with the+ /// given boolean interval.+ pub fn and<T: Borrow<Self>>(&self, rhs: T) -> Result<Self> {+ if self == &Self::FALSE || rhs.borrow() == &Self::FALSE {+ return Ok(Self::FALSE);+ }++ match (self.values(), rhs.borrow().values()) {+ (Some(l), Some(r)) => {+ let values = l.and(r)?;+ match (self, rhs.borrow()) {+ (Self::NotNull { .. }, Self::NotNull { .. }) => {+ Ok(Self::NotNull { values })+ }+ _ => Ok(Self::MaybeNull { values }),+ }+ }+ (Some(v), None) | (None, Some(v)) => {+ if v.contains_value(ScalarValue::Boolean(Some(false)))? {+ Ok(Self::FALSE_OR_UNKNOWN)+ } else {+ Ok(Self::UNKNOWN)+ }+ }+ _ => Ok(Self::UNKNOWN),+ }+ }++ /// Compute the logical disjunction of this (boolean) interval with the+ /// given boolean interval.+ pub fn or<T: Borrow<Self>>(&self, rhs: T) -> Result<Self> {+ if self == &Self::TRUE || rhs.borrow() == &Self::TRUE {+ return Ok(Self::TRUE);+ }++ match (self.values(), rhs.borrow().values()) {+ (Some(l), Some(r)) => {+ let values = l.or(r)?;+ match (self, rhs.borrow()) {+ (Self::NotNull { .. }, Self::NotNull { .. }) => {+ Ok(Self::NotNull { values })+ }+ _ => Ok(Self::MaybeNull { values }),+ }+ }+ (Some(v), None) | (None, Some(v)) => {+ if v.contains_value(ScalarValue::Boolean(Some(true)))? {+ Ok(Self::TRUE_OR_UNKNOWN)+ } else {+ Ok(Self::UNKNOWN)+ }+ }+ _ => Ok(Self::UNKNOWN),+ }+ }+/// Apply the given operator to this interval and the given interval.////// # Examples@@ -1869,6 +1967,8 @@ impl NullableInterval {Operator::IsNotDistinctFrom => self.apply_operator(&Operator::IsDistinctFrom, rhs).map(|i| i.not())?,+ Operator::And => self.and(rhs),+ Operator::Or => self.or(rhs),_ => {if let (Some(left_values), Some(right_values)) =(self.values(), rhs.values())@@ -1967,6 +2067,7 @@ mod tests {operator::Operator,};+ use crate::interval_arithmetic::NullableInterval;use arrow::datatypes::DataType;use datafusion_common::rounding::{next_down, next_up};use datafusion_common::{Result, ScalarValue};@@ -4226,4 +4327,555 @@ mod tests {Ok(())}++ #[test]+ fn nullable_and_test() -> Result<()> {+ let cases = vec![+ (+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::UNKNOWN,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_FALSE,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::TRUE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::UNKNOWN,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::UNCERTAIN,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::UNKNOWN,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::UNCERTAIN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::TRUE,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE,+ NullableInterval::TRUE_OR_FALSE,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_FALSE,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::UNKNOWN,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::TRUE,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::UNCERTAIN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ ];++ for case in cases {+ assert_eq!(+ case.0.apply_operator(&Operator::And, &case.1).unwrap(),+ case.2,+ "Failed for {} AND {}",+ case.0,+ case.1+ );+ }+ Ok(())+ }++ #[test]+ fn nullable_or_test() -> Result<()> {+ let cases = vec![+ (+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::FALSE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::TRUE,+ NullableInterval::UNCERTAIN,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ NullableInterval::FALSE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::UNKNOWN,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_FALSE,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE,+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::FALSE,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::UNKNOWN,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::UNKNOWN,+ NullableInterval::UNCERTAIN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::FALSE,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::FALSE,+ NullableInterval::TRUE_OR_FALSE,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_FALSE,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::FALSE,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::UNCERTAIN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::TRUE,+ NullableInterval::TRUE,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::UNKNOWN,+ NullableInterval::UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::UNCERTAIN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::UNCERTAIN,+ NullableInterval::UNCERTAIN,+ ),+ ];++ for case in cases {+ assert_eq!(+ case.0.apply_operator(&Operator::Or, &case.1).unwrap(),+ case.2,+ "Failed for {} OR {}",+ case.0,+ case.1+ );+ }+ Ok(())+ }++ #[test]+ fn nullable_not_test() -> Result<()> {+ let cases = vec![+ (NullableInterval::TRUE, NullableInterval::FALSE),+ (NullableInterval::FALSE, NullableInterval::TRUE),+ (NullableInterval::UNKNOWN, NullableInterval::UNKNOWN),+ (+ NullableInterval::TRUE_OR_FALSE,+ NullableInterval::TRUE_OR_FALSE,+ ),+ (+ NullableInterval::TRUE_OR_UNKNOWN,+ NullableInterval::FALSE_OR_UNKNOWN,+ ),+ (+ NullableInterval::FALSE_OR_UNKNOWN,+ NullableInterval::TRUE_OR_UNKNOWN,+ ),+ (NullableInterval::UNCERTAIN, NullableInterval::UNCERTAIN),+ ];++ for case in cases {+ assert_eq!(case.0.not().unwrap(), case.1, "Failed for NOT {}", case.0,);+ }+ Ok(())+ }}