Skip to content

Apache DataFusion SQL Query Engine

feat: warn on NULL equality predicates (#22948)

Alex Metellicab75e034d5b
diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs
index 01e5ec4f14..c00dcb82ff 100644
--- a/datafusion/sql/src/expr/mod.rs
+++ b/datafusion/sql/src/expr/mod.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+use std::ops::ControlFlow;
+
use arrow::datatypes::{DataType, TimeUnit};
use datafusion_expr::planner::{
PlannerResult, RawBinaryExpr, RawDictionaryExpr, RawFieldAccessExpr,
@@ -22,13 +24,14 @@ use datafusion_expr::planner::{
use sqlparser::ast::{
AccessExpr, BinaryOperator, CastFormat, CastKind, CeilFloorKind,
DataType as SQLDataType, DateTimeField, DictionaryField, Expr as SQLExpr,
- ExprWithAlias as SQLExprWithAlias, JsonPath, MapEntry, StructField, Subscript,
- TrimWhereField, TypedString, Value, ValueWithSpan,
+ ExprWithAlias as SQLExprWithAlias, JsonPath, MapEntry, Spanned, StructField,
+ Subscript, TrimWhereField, TypedString, Value, ValueWithSpan,
};
+use sqlparser::ast::{Query, Visit, Visitor};
use datafusion_common::{
- DFSchema, Result, ScalarValue, internal_datafusion_err, internal_err, not_impl_err,
- plan_err,
+ DFSchema, Diagnostic, Result, ScalarValue, Span, internal_datafusion_err,
+ internal_err, not_impl_err, plan_err,
};
use datafusion_expr::expr::ScalarFunction;
@@ -54,7 +57,86 @@ mod substring;
mod unary_op;
mod value;
+fn null_value_span(expr: &SQLExpr) -> Option<Option<Span>> {
+ if let SQLExpr::Value(ValueWithSpan {
+ value: Value::Null,
+ span,
+ }) = expr
+ {
+ Some(Span::try_from_sqlparser_span(*span))
+ } else {
+ None
+ }
+}
+
+fn null_equality_warning(expr: &SQLExpr) -> Option<Diagnostic> {
+ let SQLExpr::BinaryOp { left, op, right } = expr else {
+ return None;
+ };
+
+ let null_span = null_value_span(left).or_else(|| null_value_span(right))?;
+
+ let (message, help) = match op {
+ BinaryOperator::Eq => (
+ "comparison with NULL using `=` always evaluates to NULL",
+ "use `IS NULL` to check for NULL values",
+ ),
+ BinaryOperator::NotEq => (
+ "comparison with NULL using `<>` always evaluates to NULL",
+ "use `IS NOT NULL` to check for non-NULL values",
+ ),
+ _ => return None,
+ };
+
+ Some(
+ Diagnostic::new_warning(message, Span::try_from_sqlparser_span(expr.span()))
+ .with_help(help, null_span),
+ )
+}
+
+struct NullEqualityPredicateVisitor<'a, 'b, S: ContextProvider> {
+ sql_to_rel: &'a SqlToRel<'b, S>,
+ subquery_depth: usize,
+}
+
+impl<'a, 'b, S: ContextProvider> NullEqualityPredicateVisitor<'a, 'b, S> {
+ fn new(sql_to_rel: &'a SqlToRel<'b, S>) -> Self {
+ Self {
+ sql_to_rel,
+ subquery_depth: 0,
+ }
+ }
+}
+
+impl<S: ContextProvider> Visitor for NullEqualityPredicateVisitor<'_, '_, S> {
+ type Break = ();
+
+ fn pre_visit_query(&mut self, _query: &Query) -> ControlFlow<Self::Break> {
+ self.subquery_depth += 1;
+ ControlFlow::Continue(())
+ }
+
+ fn post_visit_query(&mut self, _query: &Query) -> ControlFlow<Self::Break> {
+ self.subquery_depth -= 1;
+ ControlFlow::Continue(())
+ }
+
+ fn pre_visit_expr(&mut self, expr: &SQLExpr) -> ControlFlow<Self::Break> {
+ if self.subquery_depth == 0
+ && let Some(warning) = null_equality_warning(expr)
+ {
+ self.sql_to_rel.add_warning(warning);
+ }
+ ControlFlow::Continue(())
+ }
+}
+
impl<S: ContextProvider> SqlToRel<'_, S> {
+ pub(crate) fn warn_on_null_equality_predicate(&self, predicate: &SQLExpr) {
+ let mut visitor = NullEqualityPredicateVisitor::new(self);
+ let _ = predicate.visit(&mut visitor);
+ }
+
pub(crate) fn sql_expr_to_logical_expr_with_alias(
&self,
sql: SQLExprWithAlias,
diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs
index 01215ae343..20a80e4f8a 100644
--- a/datafusion/sql/src/planner.rs
+++ b/datafusion/sql/src/planner.rs
@@ -18,7 +18,7 @@
//! [`SqlToRel`]: SQL Query Planner (produces [`LogicalPlan`] from SQL AST)
use std::collections::HashMap;
use std::str::FromStr;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
use std::vec;
use crate::utils::make_decimal_type;
@@ -455,6 +455,7 @@ pub struct SqlToRel<'a, S: ContextProvider> {
pub(crate) context_provider: &'a S,
pub(crate) options: ParserOptions,
pub(crate) ident_normalizer: IdentNormalizer,
+ warnings: Mutex<Vec<Diagnostic>>,
}
impl<'a, S: ContextProvider> SqlToRel<'a, S> {
@@ -477,9 +478,27 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
context_provider,
options,
ident_normalizer: IdentNormalizer::new(ident_normalize),
+ warnings: Mutex::new(vec![]),
}
}
+ pub(crate) fn add_warning(&self, warning: Diagnostic) {
+ self.warnings
+ .lock()
+ .expect("warning diagnostic lock poisoned")
+ .push(warning);
+ }
+
+ /// Drain and return non-fatal warnings collected during SQL planning.
+ pub fn take_warnings(&self) -> Vec<Diagnostic> {
+ std::mem::take(
+ &mut self
+ .warnings
+ .lock()
+ .expect("warning diagnostic lock poisoned"),
+ )
+ }
+
pub fn build_schema(&self, columns: Vec<SQLColumnDef>) -> Result<Schema> {
let mut fields = Vec::with_capacity(columns.len());
diff --git a/datafusion/sql/src/relation/join.rs b/datafusion/sql/src/relation/join.rs
index 3343890c6d..475d9a5b38 100644
--- a/datafusion/sql/src/relation/join.rs
+++ b/datafusion/sql/src/relation/join.rs
@@ -122,6 +122,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
JoinConstraint::On(sql_expr) => {
let join_schema = left.schema().join(right.schema())?;
// parse ON expression
+ self.warn_on_null_equality_predicate(&sql_expr);
let expr = self.sql_to_expr(sql_expr, &join_schema, planner_context)?;
LogicalPlanBuilder::from(left)
.join_on(right, join_type, Some(expr))?
diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs
index b0099b8a1d..ba7353c424 100644
--- a/datafusion/sql/src/select.rs
+++ b/datafusion/sql/src/select.rs
@@ -198,6 +198,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
let having_expr_opt = select
.having
.map::<Result<Expr>, _>(|having_expr| {
+ self.warn_on_null_equality_predicate(&having_expr);
let having_expr = self.sql_expr_to_logical_expr(
having_expr,
&combined_schema,
@@ -865,6 +866,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
Some(predicate_expr) => {
let fallback_schemas = plan.fallback_normalize_schemas();
+ self.warn_on_null_equality_predicate(&predicate_expr);
let filter_expr =
self.sql_to_expr(predicate_expr, plan.schema(), planner_context)?;
diff --git a/datafusion/sql/tests/cases/diagnostic.rs b/datafusion/sql/tests/cases/diagnostic.rs
index 226d84df25..df46a48d88 100644
--- a/datafusion/sql/tests/cases/diagnostic.rs
+++ b/datafusion/sql/tests/cases/diagnostic.rs
@@ -17,12 +17,17 @@
use datafusion_functions::string;
use insta::assert_snapshot;
-use std::{collections::HashMap, sync::Arc};
+use std::{collections::HashMap, ops::ControlFlow, sync::Arc};
+use datafusion_common::diagnostic::DiagnosticKind;
use datafusion_common::{Diagnostic, Location, Result, Span};
use datafusion_sql::{
- parser::{DFParser, DFParserBuilder},
+ parser::{DFParser, DFParserBuilder, Statement as DFStatement},
planner::{ParserOptions, SqlToRel},
+ sqlparser::{
+ ast::{Expr as SQLExpr, visit_expressions_mut},
+ tokenizer::Span as SQLParserSpan,
+ },
};
use regex::Regex;
@@ -51,6 +56,41 @@ fn do_query(sql: &'static str) -> Diagnostic {
}
}
+fn do_query_warnings(sql: &'static str) -> Vec<Diagnostic> {
+ let statement = DFParserBuilder::new(sql)
+ .build()
+ .expect("unable to create parser")
+ .parse_statement()
+ .expect("unable to parse query");
+ do_statement_warnings(statement)
+}
+
+fn do_statement_warnings(statement: DFStatement) -> Vec<Diagnostic> {
+ let options = ParserOptions {
+ collect_spans: true,
+ ..ParserOptions::default()
+ };
+ let state = MockSessionState::default();
+ let context = MockContextProvider { state };
+ let sql_to_rel = SqlToRel::new_with_options(&context, options);
+ sql_to_rel
+ .statement_to_plan(statement)
+ .expect("expected planning to succeed");
+ sql_to_rel.take_warnings()
+}
+
+fn clear_value_spans(statement: &mut DFStatement) {
+ let DFStatement::Statement(statement) = statement else {
+ panic!("expected sqlparser statement");
+ };
+ let _ = visit_expressions_mut(statement.as_mut(), |expr| {
+ if let SQLExpr::Value(value) = expr {
+ value.span = SQLParserSpan::empty();
+ }
+ ControlFlow::<()>::Continue(())
+ });
+}
+
/// Given a query that contains tag delimited spans, returns a mapping from the
/// span name to the [`Span`]. Tags are comments of the form `/*tag*/`. In case
/// you want the same location to open two spans, or close open and open
@@ -440,3 +480,194 @@ fn test_syntax_error() -> Result<()> {
},
}
}
+
+#[test]
+fn test_eq_null_warning_in_where() -> Result<()> {
+ let query = "SELECT * FROM person WHERE /*cmp*/first_name = /*null*/NULL/*null+cmp*/";
+ let spans = get_spans(query);
+ let warnings = do_query_warnings(query);
+ assert_eq!(warnings.len(), 1);
+
+ let warning = &warnings[0];
+ assert_eq!(warning.kind, DiagnosticKind::Warning);
+ assert_snapshot!(
+ warning.message,
+ @"comparison with NULL using `=` always evaluates to NULL"
+ );
+ assert_eq!(warning.span, Some(spans["cmp"]));
+ assert_snapshot!(
+ warning.helps[0].message,
+ @"use `IS NULL` to check for NULL values"
+ );
+ assert_eq!(warning.helps[0].span, Some(spans["null"]));
+ Ok(())
+}
+
+#[test]
+fn test_null_eq_warning_in_where() -> Result<()> {
+ let query = "SELECT * FROM person WHERE /*cmp+null*/NULL/*null*/ = first_name/*cmp*/";
+ let spans = get_spans(query);
+ let warnings = do_query_warnings(query);
+ assert_eq!(warnings.len(), 1);
+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);
+ assert_snapshot!(
+ warnings[0].message,
+ @"comparison with NULL using `=` always evaluates to NULL"
+ );
+ assert_eq!(warnings[0].span, Some(spans["cmp"]));
+ assert_eq!(warnings[0].helps[0].span, Some(spans["null"]));
+ Ok(())
+}
+
+#[test]
+fn test_not_eq_null_warning_in_where() -> Result<()> {
+ let query =
+ "SELECT * FROM person WHERE /*cmp*/first_name <> /*null*/NULL/*null+cmp*/";
+ let spans = get_spans(query);
+ let warnings = do_query_warnings(query);
+ assert_eq!(warnings.len(), 1);
+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);
+ assert_snapshot!(
+ warnings[0].message,
+ @"comparison with NULL using `<>` always evaluates to NULL"
+ );
+ assert_eq!(warnings[0].span, Some(spans["cmp"]));
+ assert_snapshot!(
+ warnings[0].helps[0].message,
+ @"use `IS NOT NULL` to check for non-NULL values"
+ );
+ assert_eq!(warnings[0].helps[0].span, Some(spans["null"]));
+ Ok(())
+}
+
+#[test]
+fn test_eq_null_warning_in_join_on() -> Result<()> {
+ let query =
+ "SELECT * FROM person a JOIN person b ON /*cmp*/a.id = /*null*/NULL/*null+cmp*/";
+ let spans = get_spans(query);
+ let warnings = do_query_warnings(query);
+ assert_eq!(warnings.len(), 1);
+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);
+ assert_snapshot!(
+ warnings[0].message,
+ @"comparison with NULL using `=` always evaluates to NULL"
+ );
+ assert_eq!(warnings[0].span, Some(spans["cmp"]));
+ Ok(())
+}
+
+#[test]
+fn test_eq_null_warning_in_having() -> Result<()> {
+ let query = "SELECT first_name FROM person GROUP BY first_name HAVING /*cmp*/1 = /*null*/NULL/*null+cmp*/";
+ let spans = get_spans(query);
+ let warnings = do_query_warnings(query);
+ assert_eq!(warnings.len(), 1);
+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);
+ assert_snapshot!(
+ warnings[0].message,
+ @"comparison with NULL using `=` always evaluates to NULL"
+ );
+ assert_eq!(warnings[0].span, Some(spans["cmp"]));
+ Ok(())
+}
+
+#[test]
+fn test_eq_null_warning_nested_in_case_predicate() -> Result<()> {
+ let query = "SELECT * FROM person WHERE CASE WHEN /*cmp*/first_name = /*null*/NULL/*null+cmp*/ THEN true ELSE false END";
+ let spans = get_spans(query);
+ let warnings = do_query_warnings(query);
+ assert_eq!(warnings.len(), 1);
+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);
+ assert_eq!(warnings[0].span, Some(spans["cmp"]));
+ Ok(())
+}
+
+#[test]
+fn test_eq_null_warning_under_is_null_predicate() -> Result<()> {
+ let query = "SELECT * FROM person WHERE (/*cmp*/first_name = /*null*/NULL/*null+cmp*/) IS NULL";
+ let spans = get_spans(query);
+ let warnings = do_query_warnings(query);
+ assert_eq!(warnings.len(), 1);
+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);
+ assert_eq!(warnings[0].span, Some(spans["cmp"]));
+ assert_eq!(warnings[0].helps[0].span, Some(spans["null"]));
+ Ok(())
+}
+
+#[test]
+fn test_eq_null_warning_without_null_span() -> Result<()> {
+ let query = "SELECT * FROM person WHERE first_name = NULL";
+ let mut statement = DFParserBuilder::new(query)
+ .build()
+ .expect("unable to create parser")
+ .parse_statement()
+ .expect("unable to parse query");
+ clear_value_spans(&mut statement);
+
+ let warnings = do_statement_warnings(statement);
+ assert_eq!(warnings.len(), 1);
+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);
+ assert_snapshot!(
+ warnings[0].message,
+ @"comparison with NULL using `=` always evaluates to NULL"
+ );
+ assert_eq!(warnings[0].helps[0].span, None);
+ Ok(())
+}
+
+#[test]
+fn test_is_null_has_no_warning() -> Result<()> {
+ let warnings = do_query_warnings("SELECT * FROM person WHERE first_name IS NULL");
+ assert!(warnings.is_empty(), "unexpected warnings: {warnings:?}");
+ Ok(())
+}
+
+#[test]
+fn test_eq_null_projection_has_no_warning() -> Result<()> {
+ let warnings = do_query_warnings("SELECT first_name = NULL FROM person");
+ assert!(warnings.is_empty(), "unexpected warnings: {warnings:?}");
+ Ok(())
+}
+
+#[test]
+fn test_eq_null_projection_in_exists_has_no_warning() -> Result<()> {
+ let warnings = do_query_warnings(
+ "SELECT * FROM person WHERE EXISTS (SELECT first_name = NULL FROM person)",
+ );
+ assert!(warnings.is_empty(), "unexpected warnings: {warnings:?}");
+ Ok(())
+}
+
+#[test]
+fn test_eq_null_warning_in_exists_subquery_where() -> Result<()> {
+ let query = "SELECT * FROM person WHERE EXISTS (SELECT 1 FROM person WHERE /*cmp*/first_name = /*null*/NULL/*null+cmp*/)";
+ let spans = get_spans(query);
+ let warnings = do_query_warnings(query);
+ assert_eq!(warnings.len(), 1);
+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);
+ assert_snapshot!(
+ warnings[0].message,
+ @"comparison with NULL using `=` always evaluates to NULL"
+ );
+ assert_eq!(warnings[0].span, Some(spans["cmp"]));
+ assert_eq!(warnings[0].helps[0].span, Some(spans["null"]));
+ Ok(())
+}
+
+#[test]
+fn test_multiple_null_comparison_warnings() -> Result<()> {
+ let warnings = do_query_warnings(
+ "SELECT * FROM person WHERE first_name = NULL OR last_name <> NULL",
+ );
+ assert_eq!(warnings.len(), 2);
+ assert!(warnings.iter().all(|w| w.kind == DiagnosticKind::Warning));
+ assert_snapshot!(
+ warnings[0].message,
+ @"comparison with NULL using `=` always evaluates to NULL"
+ );
+ assert_snapshot!(
+ warnings[1].message,
+ @"comparison with NULL using `<>` always evaluates to NULL"
+ );
+ Ok(())
+}