Apache DataFusion SQL Query Engine
feat: warn on NULL equality predicates (#22948)
Alex Metellicab75e034d5b
diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rsindex 01e5ec4f14..c00dcb82ff 100644--- a/datafusion/sql/src/expr/mod.rs+++ b/datafusion/sql/src/expr/mod.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+use std::ops::ControlFlow;+use arrow::datatypes::{DataType, TimeUnit};use datafusion_expr::planner::{PlannerResult, RawBinaryExpr, RawDictionaryExpr, RawFieldAccessExpr,@@ -22,13 +24,14 @@ use datafusion_expr::planner::{use sqlparser::ast::{AccessExpr, BinaryOperator, CastFormat, CastKind, CeilFloorKind,DataType as SQLDataType, DateTimeField, DictionaryField, Expr as SQLExpr,- ExprWithAlias as SQLExprWithAlias, JsonPath, MapEntry, StructField, Subscript,- TrimWhereField, TypedString, Value, ValueWithSpan,+ ExprWithAlias as SQLExprWithAlias, JsonPath, MapEntry, Spanned, StructField,+ Subscript, TrimWhereField, TypedString, Value, ValueWithSpan,};+use sqlparser::ast::{Query, Visit, Visitor};use datafusion_common::{- DFSchema, Result, ScalarValue, internal_datafusion_err, internal_err, not_impl_err,- plan_err,+ DFSchema, Diagnostic, Result, ScalarValue, Span, internal_datafusion_err,+ internal_err, not_impl_err, plan_err,};use datafusion_expr::expr::ScalarFunction;@@ -54,7 +57,86 @@ mod substring;mod unary_op;mod value;+fn null_value_span(expr: &SQLExpr) -> Option<Option<Span>> {+ if let SQLExpr::Value(ValueWithSpan {+ value: Value::Null,+ span,+ }) = expr+ {+ Some(Span::try_from_sqlparser_span(*span))+ } else {+ None+ }+}++fn null_equality_warning(expr: &SQLExpr) -> Option<Diagnostic> {+ let SQLExpr::BinaryOp { left, op, right } = expr else {+ return None;+ };++ let null_span = null_value_span(left).or_else(|| null_value_span(right))?;++ let (message, help) = match op {+ BinaryOperator::Eq => (+ "comparison with NULL using `=` always evaluates to NULL",+ "use `IS NULL` to check for NULL values",+ ),+ BinaryOperator::NotEq => (+ "comparison with NULL using `<>` always evaluates to NULL",+ "use `IS NOT NULL` to check for non-NULL values",+ ),+ _ => return None,+ };++ Some(+ Diagnostic::new_warning(message, Span::try_from_sqlparser_span(expr.span()))+ .with_help(help, null_span),+ )+}++struct NullEqualityPredicateVisitor<'a, 'b, S: ContextProvider> {+ sql_to_rel: &'a SqlToRel<'b, S>,+ subquery_depth: usize,+}++impl<'a, 'b, S: ContextProvider> NullEqualityPredicateVisitor<'a, 'b, S> {+ fn new(sql_to_rel: &'a SqlToRel<'b, S>) -> Self {+ Self {+ sql_to_rel,+ subquery_depth: 0,+ }+ }+}++impl<S: ContextProvider> Visitor for NullEqualityPredicateVisitor<'_, '_, S> {+ type Break = ();++ fn pre_visit_query(&mut self, _query: &Query) -> ControlFlow<Self::Break> {+ self.subquery_depth += 1;+ ControlFlow::Continue(())+ }++ fn post_visit_query(&mut self, _query: &Query) -> ControlFlow<Self::Break> {+ self.subquery_depth -= 1;+ ControlFlow::Continue(())+ }++ fn pre_visit_expr(&mut self, expr: &SQLExpr) -> ControlFlow<Self::Break> {+ if self.subquery_depth == 0+ && let Some(warning) = null_equality_warning(expr)+ {+ self.sql_to_rel.add_warning(warning);+ }+ ControlFlow::Continue(())+ }+}+impl<S: ContextProvider> SqlToRel<'_, S> {+ pub(crate) fn warn_on_null_equality_predicate(&self, predicate: &SQLExpr) {+ let mut visitor = NullEqualityPredicateVisitor::new(self);+ let _ = predicate.visit(&mut visitor);+ }+pub(crate) fn sql_expr_to_logical_expr_with_alias(&self,sql: SQLExprWithAlias,diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rsindex 01215ae343..20a80e4f8a 100644--- a/datafusion/sql/src/planner.rs+++ b/datafusion/sql/src/planner.rs@@ -18,7 +18,7 @@//! [`SqlToRel`]: SQL Query Planner (produces [`LogicalPlan`] from SQL AST)use std::collections::HashMap;use std::str::FromStr;-use std::sync::Arc;+use std::sync::{Arc, Mutex};use std::vec;use crate::utils::make_decimal_type;@@ -455,6 +455,7 @@ pub struct SqlToRel<'a, S: ContextProvider> {pub(crate) context_provider: &'a S,pub(crate) options: ParserOptions,pub(crate) ident_normalizer: IdentNormalizer,+ warnings: Mutex<Vec<Diagnostic>>,}impl<'a, S: ContextProvider> SqlToRel<'a, S> {@@ -477,9 +478,27 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {context_provider,options,ident_normalizer: IdentNormalizer::new(ident_normalize),+ warnings: Mutex::new(vec![]),}}+ pub(crate) fn add_warning(&self, warning: Diagnostic) {+ self.warnings+ .lock()+ .expect("warning diagnostic lock poisoned")+ .push(warning);+ }++ /// Drain and return non-fatal warnings collected during SQL planning.+ pub fn take_warnings(&self) -> Vec<Diagnostic> {+ std::mem::take(+ &mut self+ .warnings+ .lock()+ .expect("warning diagnostic lock poisoned"),+ )+ }+pub fn build_schema(&self, columns: Vec<SQLColumnDef>) -> Result<Schema> {let mut fields = Vec::with_capacity(columns.len());diff --git a/datafusion/sql/src/relation/join.rs b/datafusion/sql/src/relation/join.rsindex 3343890c6d..475d9a5b38 100644--- a/datafusion/sql/src/relation/join.rs+++ b/datafusion/sql/src/relation/join.rs@@ -122,6 +122,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {JoinConstraint::On(sql_expr) => {let join_schema = left.schema().join(right.schema())?;// parse ON expression+ self.warn_on_null_equality_predicate(&sql_expr);let expr = self.sql_to_expr(sql_expr, &join_schema, planner_context)?;LogicalPlanBuilder::from(left).join_on(right, join_type, Some(expr))?diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rsindex b0099b8a1d..ba7353c424 100644--- a/datafusion/sql/src/select.rs+++ b/datafusion/sql/src/select.rs@@ -198,6 +198,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let having_expr_opt = select.having.map::<Result<Expr>, _>(|having_expr| {+ self.warn_on_null_equality_predicate(&having_expr);let having_expr = self.sql_expr_to_logical_expr(having_expr,&combined_schema,@@ -865,6 +866,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {Some(predicate_expr) => {let fallback_schemas = plan.fallback_normalize_schemas();+ self.warn_on_null_equality_predicate(&predicate_expr);let filter_expr =self.sql_to_expr(predicate_expr, plan.schema(), planner_context)?;diff --git a/datafusion/sql/tests/cases/diagnostic.rs b/datafusion/sql/tests/cases/diagnostic.rsindex 226d84df25..df46a48d88 100644--- a/datafusion/sql/tests/cases/diagnostic.rs+++ b/datafusion/sql/tests/cases/diagnostic.rs@@ -17,12 +17,17 @@use datafusion_functions::string;use insta::assert_snapshot;-use std::{collections::HashMap, sync::Arc};+use std::{collections::HashMap, ops::ControlFlow, sync::Arc};+use datafusion_common::diagnostic::DiagnosticKind;use datafusion_common::{Diagnostic, Location, Result, Span};use datafusion_sql::{- parser::{DFParser, DFParserBuilder},+ parser::{DFParser, DFParserBuilder, Statement as DFStatement},planner::{ParserOptions, SqlToRel},+ sqlparser::{+ ast::{Expr as SQLExpr, visit_expressions_mut},+ tokenizer::Span as SQLParserSpan,+ },};use regex::Regex;@@ -51,6 +56,41 @@ fn do_query(sql: &'static str) -> Diagnostic {}}+fn do_query_warnings(sql: &'static str) -> Vec<Diagnostic> {+ let statement = DFParserBuilder::new(sql)+ .build()+ .expect("unable to create parser")+ .parse_statement()+ .expect("unable to parse query");+ do_statement_warnings(statement)+}++fn do_statement_warnings(statement: DFStatement) -> Vec<Diagnostic> {+ let options = ParserOptions {+ collect_spans: true,+ ..ParserOptions::default()+ };+ let state = MockSessionState::default();+ let context = MockContextProvider { state };+ let sql_to_rel = SqlToRel::new_with_options(&context, options);+ sql_to_rel+ .statement_to_plan(statement)+ .expect("expected planning to succeed");+ sql_to_rel.take_warnings()+}++fn clear_value_spans(statement: &mut DFStatement) {+ let DFStatement::Statement(statement) = statement else {+ panic!("expected sqlparser statement");+ };+ let _ = visit_expressions_mut(statement.as_mut(), |expr| {+ if let SQLExpr::Value(value) = expr {+ value.span = SQLParserSpan::empty();+ }+ ControlFlow::<()>::Continue(())+ });+}+/// Given a query that contains tag delimited spans, returns a mapping from the/// span name to the [`Span`]. Tags are comments of the form `/*tag*/`. In case/// you want the same location to open two spans, or close open and open@@ -440,3 +480,194 @@ fn test_syntax_error() -> Result<()> {},}}++#[test]+fn test_eq_null_warning_in_where() -> Result<()> {+ let query = "SELECT * FROM person WHERE /*cmp*/first_name = /*null*/NULL/*null+cmp*/";+ let spans = get_spans(query);+ let warnings = do_query_warnings(query);+ assert_eq!(warnings.len(), 1);++ let warning = &warnings[0];+ assert_eq!(warning.kind, DiagnosticKind::Warning);+ assert_snapshot!(+ warning.message,+ @"comparison with NULL using `=` always evaluates to NULL"+ );+ assert_eq!(warning.span, Some(spans["cmp"]));+ assert_snapshot!(+ warning.helps[0].message,+ @"use `IS NULL` to check for NULL values"+ );+ assert_eq!(warning.helps[0].span, Some(spans["null"]));+ Ok(())+}++#[test]+fn test_null_eq_warning_in_where() -> Result<()> {+ let query = "SELECT * FROM person WHERE /*cmp+null*/NULL/*null*/ = first_name/*cmp*/";+ let spans = get_spans(query);+ let warnings = do_query_warnings(query);+ assert_eq!(warnings.len(), 1);+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);+ assert_snapshot!(+ warnings[0].message,+ @"comparison with NULL using `=` always evaluates to NULL"+ );+ assert_eq!(warnings[0].span, Some(spans["cmp"]));+ assert_eq!(warnings[0].helps[0].span, Some(spans["null"]));+ Ok(())+}++#[test]+fn test_not_eq_null_warning_in_where() -> Result<()> {+ let query =+ "SELECT * FROM person WHERE /*cmp*/first_name <> /*null*/NULL/*null+cmp*/";+ let spans = get_spans(query);+ let warnings = do_query_warnings(query);+ assert_eq!(warnings.len(), 1);+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);+ assert_snapshot!(+ warnings[0].message,+ @"comparison with NULL using `<>` always evaluates to NULL"+ );+ assert_eq!(warnings[0].span, Some(spans["cmp"]));+ assert_snapshot!(+ warnings[0].helps[0].message,+ @"use `IS NOT NULL` to check for non-NULL values"+ );+ assert_eq!(warnings[0].helps[0].span, Some(spans["null"]));+ Ok(())+}++#[test]+fn test_eq_null_warning_in_join_on() -> Result<()> {+ let query =+ "SELECT * FROM person a JOIN person b ON /*cmp*/a.id = /*null*/NULL/*null+cmp*/";+ let spans = get_spans(query);+ let warnings = do_query_warnings(query);+ assert_eq!(warnings.len(), 1);+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);+ assert_snapshot!(+ warnings[0].message,+ @"comparison with NULL using `=` always evaluates to NULL"+ );+ assert_eq!(warnings[0].span, Some(spans["cmp"]));+ Ok(())+}++#[test]+fn test_eq_null_warning_in_having() -> Result<()> {+ let query = "SELECT first_name FROM person GROUP BY first_name HAVING /*cmp*/1 = /*null*/NULL/*null+cmp*/";+ let spans = get_spans(query);+ let warnings = do_query_warnings(query);+ assert_eq!(warnings.len(), 1);+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);+ assert_snapshot!(+ warnings[0].message,+ @"comparison with NULL using `=` always evaluates to NULL"+ );+ assert_eq!(warnings[0].span, Some(spans["cmp"]));+ Ok(())+}++#[test]+fn test_eq_null_warning_nested_in_case_predicate() -> Result<()> {+ let query = "SELECT * FROM person WHERE CASE WHEN /*cmp*/first_name = /*null*/NULL/*null+cmp*/ THEN true ELSE false END";+ let spans = get_spans(query);+ let warnings = do_query_warnings(query);+ assert_eq!(warnings.len(), 1);+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);+ assert_eq!(warnings[0].span, Some(spans["cmp"]));+ Ok(())+}++#[test]+fn test_eq_null_warning_under_is_null_predicate() -> Result<()> {+ let query = "SELECT * FROM person WHERE (/*cmp*/first_name = /*null*/NULL/*null+cmp*/) IS NULL";+ let spans = get_spans(query);+ let warnings = do_query_warnings(query);+ assert_eq!(warnings.len(), 1);+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);+ assert_eq!(warnings[0].span, Some(spans["cmp"]));+ assert_eq!(warnings[0].helps[0].span, Some(spans["null"]));+ Ok(())+}++#[test]+fn test_eq_null_warning_without_null_span() -> Result<()> {+ let query = "SELECT * FROM person WHERE first_name = NULL";+ let mut statement = DFParserBuilder::new(query)+ .build()+ .expect("unable to create parser")+ .parse_statement()+ .expect("unable to parse query");+ clear_value_spans(&mut statement);++ let warnings = do_statement_warnings(statement);+ assert_eq!(warnings.len(), 1);+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);+ assert_snapshot!(+ warnings[0].message,+ @"comparison with NULL using `=` always evaluates to NULL"+ );+ assert_eq!(warnings[0].helps[0].span, None);+ Ok(())+}++#[test]+fn test_is_null_has_no_warning() -> Result<()> {+ let warnings = do_query_warnings("SELECT * FROM person WHERE first_name IS NULL");+ assert!(warnings.is_empty(), "unexpected warnings: {warnings:?}");+ Ok(())+}++#[test]+fn test_eq_null_projection_has_no_warning() -> Result<()> {+ let warnings = do_query_warnings("SELECT first_name = NULL FROM person");+ assert!(warnings.is_empty(), "unexpected warnings: {warnings:?}");+ Ok(())+}++#[test]+fn test_eq_null_projection_in_exists_has_no_warning() -> Result<()> {+ let warnings = do_query_warnings(+ "SELECT * FROM person WHERE EXISTS (SELECT first_name = NULL FROM person)",+ );+ assert!(warnings.is_empty(), "unexpected warnings: {warnings:?}");+ Ok(())+}++#[test]+fn test_eq_null_warning_in_exists_subquery_where() -> Result<()> {+ let query = "SELECT * FROM person WHERE EXISTS (SELECT 1 FROM person WHERE /*cmp*/first_name = /*null*/NULL/*null+cmp*/)";+ let spans = get_spans(query);+ let warnings = do_query_warnings(query);+ assert_eq!(warnings.len(), 1);+ assert_eq!(warnings[0].kind, DiagnosticKind::Warning);+ assert_snapshot!(+ warnings[0].message,+ @"comparison with NULL using `=` always evaluates to NULL"+ );+ assert_eq!(warnings[0].span, Some(spans["cmp"]));+ assert_eq!(warnings[0].helps[0].span, Some(spans["null"]));+ Ok(())+}++#[test]+fn test_multiple_null_comparison_warnings() -> Result<()> {+ let warnings = do_query_warnings(+ "SELECT * FROM person WHERE first_name = NULL OR last_name <> NULL",+ );+ assert_eq!(warnings.len(), 2);+ assert!(warnings.iter().all(|w| w.kind == DiagnosticKind::Warning));+ assert_snapshot!(+ warnings[0].message,+ @"comparison with NULL using `=` always evaluates to NULL"+ );+ assert_snapshot!(+ warnings[1].message,+ @"comparison with NULL using `<>` always evaluates to NULL"+ );+ Ok(())+}