Skip to content

Apache DataFusion SQL Query Engine

fix: account for empty scalar subqueries in nullability (#24516)

Fredrik Fornwall4798476d5e63
diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs
index ee70e90153..8a9d0b8937 100644
--- a/datafusion/expr/src/expr_schema.rs
+++ b/datafusion/expr/src/expr_schema.rs
@@ -369,9 +369,9 @@ impl ExprSchemable for Expr {
Ok(expr_nullable | subquery_nullable)
}
- Expr::ScalarSubquery(subquery) => {
- Ok(subquery.subquery.schema().field(0).is_nullable())
- }
+ // A scalar subquery may return no rows, in which case it evaluates to NULL
+ // regardless of the nullability of its projected field.
+ Expr::ScalarSubquery(_) => Ok(true),
Expr::BinaryExpr(BinaryExpr { left, right, .. }) => {
Ok(left.nullable(input_schema)? || right.nullable(input_schema)?)
}
@@ -517,9 +517,15 @@ impl ExprSchemable for Expr {
| Expr::Exists { .. } => {
Ok(Arc::new(Field::new(&schema_name, DataType::Boolean, false)))
}
- Expr::ScalarSubquery(subquery) => {
- Ok(Arc::clone(&subquery.subquery.schema().fields()[0]))
- }
+ Expr::ScalarSubquery(subquery) => Ok(Arc::new(
+ subquery
+ .subquery
+ .schema()
+ .field(0)
+ .as_ref()
+ .clone()
+ .with_nullable(true),
+ )),
Expr::BinaryExpr(BinaryExpr { left, right, op }) => {
let (left_field, right_field) =
(left.to_field(schema)?.1, right.to_field(schema)?.1);
@@ -800,7 +806,7 @@ mod tests {
use crate::logical_plan::builder::LogicalTableSource;
use crate::{
LogicalPlanBuilder, and, col, in_subquery, lit, not, or,
- out_ref_col_with_metadata, when,
+ out_ref_col_with_metadata, scalar_subquery, when,
};
use arrow::datatypes::Schema;
@@ -1268,6 +1274,23 @@ mod tests {
);
}
+ #[test]
+ fn scalar_subquery_is_nullable_with_non_nullable_output() {
+ let subquery = LogicalPlanBuilder::empty(false)
+ .project(vec![lit(1)])
+ .unwrap()
+ .build()
+ .unwrap();
+ assert!(!subquery.schema().field(0).is_nullable());
+
+ let expr = scalar_subquery(Arc::new(subquery));
+ assert!(expr.nullable(&MockExprSchema::new()).unwrap());
+
+ let field = expr.to_field(&MockExprSchema::new()).unwrap().1;
+ assert_eq!(field.data_type(), &DataType::Int32);
+ assert!(field.is_nullable());
+ }
+
#[test]
fn test_scalar_variable() {
let mut meta = HashMap::new();
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index b7e9624946..5beb54b560 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -3788,6 +3788,21 @@ mod tests {
);
}
+ #[test]
+ fn simplify_scalar_subquery_is_null() {
+ let subquery = LogicalPlanBuilder::empty(false)
+ .project(vec![lit(1)])
+ .unwrap()
+ .build()
+ .unwrap();
+ let scalar_subquery = scalar_subquery(Arc::new(subquery));
+
+ assert_eq!(
+ simplify(scalar_subquery.clone().is_null()),
+ scalar_subquery.is_null()
+ );
+ }
+
#[test]
fn simplify_expr_is_unknown() {
assert_eq!(simplify(col("c2").is_unknown()), col("c2").is_unknown(),);
diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs
index f80d1b15bd..09b2e5ce04 100644
--- a/datafusion/physical-expr/src/planner.rs
+++ b/datafusion/physical-expr/src/planner.rs
@@ -536,10 +536,10 @@ pub fn create_physical_expr(
);
}
let dt = schema.field(0).data_type().clone();
- let nullable = schema.field(0).is_nullable();
Ok(Arc::new(ScalarSubqueryExpr::new(
dt,
- nullable,
+ // A scalar subquery may return no rows and evaluate to NULL.
+ true,
index,
planning_ctx.results().clone(),
)))
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index c38fb4c7ee..65760886d1 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -2010,6 +2010,24 @@ SELECT (SELECT v FROM sq_empty);
----
NULL
+# A zero-row scalar subquery is nullable even when its projected expression is not.
+query I
+SELECT (SELECT 1 WHERE FALSE);
+----
+NULL
+
+# Its nullability must prevent SimplifyExpressions from folding IS NULL to false.
+query B
+SELECT (SELECT 1 WHERE FALSE) IS NULL;
+----
+true
+
+# The same holds for the opposite fold direction.
+query B
+SELECT (SELECT 1 WHERE FALSE) IS NOT NULL;
+----
+false
+
# Scalar subquery returning zero rows in arithmetic → NULL propagation
query I
SELECT x + (SELECT v FROM sq_empty) FROM sq_main;
@@ -2350,6 +2368,21 @@ SELECT (SELECT v FROM sq_empty);
----
NULL
+# A zero-row scalar subquery is nullable even when its projected expression is
+# not. The rewrite to a left join already produces a nullable column here, so
+# this pins both paths to the same result.
+query I
+SELECT (SELECT 1 WHERE FALSE);
+----
+NULL
+
+# SimplifyExpressions runs before ScalarSubqueryToJoin, so this fold is still
+# governed by Expr::ScalarSubquery nullability on this path.
+query B
+SELECT (SELECT 1 WHERE FALSE) IS NULL;
+----
+true
+
# Scalar subquery returning zero rows in arithmetic → NULL propagation
query I
SELECT x + (SELECT v FROM sq_empty) FROM sq_main;