Skip to content

Apache DataFusion SQL Query Engine

fix: Correctly compute nullability in recursive CTE schemas (#22552)

Neil Conway08bd332dd74a
diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rs
index dd313ebb4c..5ec688526c 100644
--- a/datafusion/catalog/src/cte_worktable.rs
+++ b/datafusion/catalog/src/cte_worktable.rs
@@ -36,14 +36,16 @@ use crate::{ScanArgs, ScanResult, Session, TableProvider};
pub struct CteWorkTable {
/// The name of the CTE work table
name: String,
- /// This schema must be shared across both the static and recursive terms of a recursive query
+ /// Schema exposed by recursive self-references while planning the recursive term.
+ ///
+ /// This is a conservative work-table schema, not the final recursive query output
+ /// schema. For example, the SQL planner may mark fields nullable here so recursive
+ /// references do not inherit unsound anchor-term nullability assumptions.
table_schema: SchemaRef,
}
impl CteWorkTable {
- /// construct a new CteWorkTable with the given name and schema
- /// This schema must match the schema of the recursive term of the query
- /// Since the scan method will contain an physical plan that assumes this schema
+ /// Construct a new CteWorkTable with the given name and self-reference schema.
pub fn new(name: &str, table_schema: SchemaRef) -> Self {
Self {
name: name.to_owned(),
@@ -56,7 +58,7 @@ impl CteWorkTable {
&self.name
}
- /// The schema of the recursive term of the query
+ /// The schema exposed by scans of the recursive self-reference.
pub fn schema(&self) -> SchemaRef {
Arc::clone(&self.table_schema)
}
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index 5496db3a8d..dd741ee6ff 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -1798,11 +1798,15 @@ impl DefaultPhysicalPlanner {
}
}
LogicalPlan::RecursiveQuery(RecursiveQuery {
- name, is_distinct, ..
+ name,
+ is_distinct,
+ schema,
+ ..
}) => {
let [static_term, recursive_term] = children.two()?;
Arc::new(RecursiveQueryExec::try_new(
name.clone(),
+ Arc::clone(schema.inner()),
static_term,
recursive_term,
*is_distinct,
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rs
index 17e3dba14b..a7cec182f7 100644
--- a/datafusion/core/tests/sql/explain_analyze.rs
+++ b/datafusion/core/tests/sql/explain_analyze.rs
@@ -1014,7 +1014,7 @@ async fn parquet_recursive_projection_pushdown() -> Result<()> {
SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
RecursiveQueryExec: name=number_series, is_distinct=false
CoalescePartitionsExec
- ProjectionExec: expr=[id@0 as id, 1 as level]
+ ProjectionExec: expr=[CAST(id@0 AS Int64) as id, CAST(1 AS Int64) as level]
FilterExec: id@0 = 1
RepartitionExec: partitioning=RoundRobinBatch(NUM_CORES), input_partitions=1
DataSourceExec: file_groups={1 group: [[TMP_DIR/hierarchy.parquet]]}, projection=[id], file_type=parquet, predicate=id@0 = 1, pruning_predicate=id_null_count@2 != row_count@3 AND id_min@0 <= 1 AND 1 <= id_max@1, required_guarantees=[id in (1)]
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
index 29bc448c8f..2ecb12c30a 100644
--- a/datafusion/expr/src/logical_plan/builder.rs
+++ b/datafusion/expr/src/logical_plan/builder.rs
@@ -192,12 +192,13 @@ impl LogicalPlanBuilder {
// Ensure that the recursive term has the same field types as the static term
let coerced_recursive_term =
coerce_plan_expr_for_schema(recursive_term, self.plan.schema())?;
- Ok(Self::from(LogicalPlan::RecursiveQuery(RecursiveQuery {
+ let recursive_query = RecursiveQuery::try_new(
name,
- static_term: self.plan,
- recursive_term: Arc::new(coerced_recursive_term),
+ self.plan,
+ Arc::new(coerced_recursive_term),
is_distinct,
- })))
+ )?;
+ Ok(Self::from(LogicalPlan::RecursiveQuery(recursive_query)))
}
/// Create a values list based relation, and the schema is inferred from data, consuming
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index cef20dcd5a..1bfecd06c2 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -354,10 +354,7 @@ impl LogicalPlan {
LogicalPlan::Copy(CopyTo { output_schema, .. }) => output_schema,
LogicalPlan::Ddl(ddl) => ddl.schema(),
LogicalPlan::Unnest(Unnest { schema, .. }) => schema,
- LogicalPlan::RecursiveQuery(RecursiveQuery { static_term, .. }) => {
- // we take the schema of the static term as the schema of the entire recursive query
- static_term.schema()
- }
+ LogicalPlan::RecursiveQuery(RecursiveQuery { schema, .. }) => schema,
}
}
@@ -741,7 +738,14 @@ impl LogicalPlan {
};
Ok(LogicalPlan::Distinct(distinct))
}
- LogicalPlan::RecursiveQuery(_) => Ok(self),
+ LogicalPlan::RecursiveQuery(RecursiveQuery {
+ name,
+ static_term,
+ recursive_term,
+ is_distinct,
+ schema: _,
+ }) => RecursiveQuery::try_new(name, static_term, recursive_term, is_distinct)
+ .map(LogicalPlan::RecursiveQuery),
LogicalPlan::Analyze(_) => Ok(self),
LogicalPlan::Explain(_) => Ok(self),
LogicalPlan::TableScan(_) => Ok(self),
@@ -1081,12 +1085,13 @@ impl LogicalPlan {
}) => {
self.assert_no_expressions(expr)?;
let (static_term, recursive_term) = self.only_two_inputs(inputs)?;
- Ok(LogicalPlan::RecursiveQuery(RecursiveQuery {
- name: name.clone(),
- static_term: Arc::new(static_term),
- recursive_term: Arc::new(recursive_term),
- is_distinct: *is_distinct,
- }))
+ RecursiveQuery::try_new(
+ name.clone(),
+ Arc::new(static_term),
+ Arc::new(recursive_term),
+ *is_distinct,
+ )
+ .map(LogicalPlan::RecursiveQuery)
}
LogicalPlan::Analyze(a) => {
self.assert_no_expressions(expr)?;
@@ -2262,7 +2267,7 @@ impl PartialOrd for EmptyRelation {
/// intermediate table, then empty the intermediate table.
///
/// [Postgres Docs]: https://www.postgresql.org/docs/current/queries-with.html#QUERIES-WITH-RECURSIVE
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct RecursiveQuery {
/// Name of the query
pub name: String,
@@ -2274,6 +2279,90 @@ pub struct RecursiveQuery {
/// Should the output of the recursive term be deduplicated (`UNION`) or
/// not (`UNION ALL`).
pub is_distinct: bool,
+ /// Schema exposed to parent plans after reconciling the static and recursive terms.
+ pub schema: DFSchemaRef,
+}
+
+impl PartialOrd for RecursiveQuery {
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ match self.name.partial_cmp(&other.name) {
+ Some(Ordering::Equal) => {
+ match self.static_term.partial_cmp(&other.static_term) {
+ Some(Ordering::Equal) => {
+ match self.recursive_term.partial_cmp(&other.recursive_term) {
+ Some(Ordering::Equal) => {
+ self.is_distinct.partial_cmp(&other.is_distinct)
+ }
+ cmp => cmp,
+ }
+ }
+ cmp => cmp,
+ }
+ }
+ cmp => cmp,
+ }
+ // If the query definition compares equal but the derived schema differs,
+ // return `None` instead of contradicting `PartialEq` with `Some(Equal)`.
+ // TODO (https://github.com/apache/datafusion/issues/17477) avoid recomparing all fields
+ .filter(|cmp| *cmp != Ordering::Equal || self == other)
+ }
+}
+
+impl RecursiveQuery {
+ pub fn try_new(
+ name: String,
+ static_term: Arc<LogicalPlan>,
+ recursive_term: Arc<LogicalPlan>,
+ is_distinct: bool,
+ ) -> Result<Self> {
+ let schema =
+ recursive_query_output_schema(static_term.schema(), recursive_term.schema())?;
+ Ok(Self {
+ name,
+ static_term,
+ recursive_term,
+ is_distinct,
+ schema,
+ })
+ }
+}
+
+/// Compute a recursive query's output schema by considering both its static and
+/// recursive terms.
+///
+/// Field names, types, and metadata come from the static term. A field is
+/// nullable if either the static or the recursive term produces a nullable
+/// value in that position, matching how `UNION` reconciles branch nullability.
+///
+/// Functional dependencies are intentionally dropped: the recursive term
+/// appends rows that can duplicate values the static term guarantees unique, so
+/// any FDs carried by the static term may not hold over the combined output.
+fn recursive_query_output_schema(
+ static_schema: &DFSchemaRef,
+ recursive_schema: &DFSchemaRef,
+) -> Result<DFSchemaRef> {
+ if static_schema.fields().len() != recursive_schema.fields().len() {
+ return Err(DataFusionError::Plan(format!(
+ "Non-recursive term and recursive term must have the same number of columns ({} != {})",
+ static_schema.fields().len(),
+ recursive_schema.fields().len()
+ )));
+ }
+
+ let fields = static_schema
+ .iter()
+ .zip(recursive_schema.fields())
+ .map(|((qualifier, static_field), recursive_field)| {
+ let nullable = static_field.is_nullable() || recursive_field.is_nullable();
+ (
+ qualifier.cloned(),
+ static_field.as_ref().clone().with_nullable(nullable).into(),
+ )
+ })
+ .collect::<Vec<_>>();
+
+ DFSchema::new_with_metadata(fields, static_schema.metadata().clone())
+ .map(DFSchemaRef::new)
}
/// Values expression. See
@@ -4671,6 +4760,74 @@ mod tests {
.build()
}
+ fn recursive_term_scan(name: &str, fields: Vec<Field>) -> Result<Arc<LogicalPlan>> {
+ Ok(Arc::new(
+ table_scan(Some(name), &Schema::new(fields), None)?.build()?,
+ ))
+ }
+
+ #[test]
+ fn recursive_query_widens_nullability_per_column() -> Result<()> {
+ // Column `a` is non-nullable in both terms and must stay non-nullable;
+ // column `b` is non-nullable in the static term but nullable in the
+ // recursive term, so the output must widen it to nullable.
+ let static_term = recursive_term_scan(
+ "static",
+ vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ ],
+ )?;
+ let recursive_term = recursive_term_scan(
+ "rec",
+ vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, true),
+ ],
+ )?;
+
+ let query =
+ RecursiveQuery::try_new("t".to_string(), static_term, recursive_term, false)?;
+
+ // Names and types are taken from the static term.
+ assert_eq!(query.schema.field(0).name(), "a");
+ assert_eq!(query.schema.field(1).name(), "b");
+ assert_eq!(query.schema.field(0).data_type(), &DataType::Int32);
+ assert_eq!(query.schema.field(1).data_type(), &DataType::Int32);
+ // Nullability is widened independently per column.
+ assert!(!query.schema.field(0).is_nullable());
+ assert!(query.schema.field(1).is_nullable());
+ // `schema()` returns the widened recursive-query schema.
+ assert_eq!(
+ LogicalPlan::RecursiveQuery(query.clone()).schema(),
+ &query.schema
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn recursive_query_rejects_column_count_mismatch() -> Result<()> {
+ let static_term =
+ recursive_term_scan("static", vec![Field::new("a", DataType::Int32, false)])?;
+ let recursive_term = recursive_term_scan(
+ "rec",
+ vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ ],
+ )?;
+
+ let err =
+ RecursiveQuery::try_new("t".to_string(), static_term, recursive_term, false)
+ .unwrap_err();
+ assert!(
+ err.strip_backtrace()
+ .contains("must have the same number of columns"),
+ "unexpected error: {err}"
+ );
+ Ok(())
+ }
+
#[test]
fn test_display_indent() -> Result<()> {
let plan = display_plan()?;
diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rs
index 2c6be54705..e0cdec9e2c 100644
--- a/datafusion/expr/src/logical_plan/tree_node.rs
+++ b/datafusion/expr/src/logical_plan/tree_node.rs
@@ -337,13 +337,18 @@ impl TreeNode for LogicalPlan {
static_term,
recursive_term,
is_distinct,
+ schema,
}) => (static_term, recursive_term).map_elements(f)?.update_data(
|(static_term, recursive_term)| {
+ // Ordinary child rewrites preserve derived schemas. Call
+ // `LogicalPlan::recompute_schema` when child schemas should
+ // be reconciled again.
LogicalPlan::RecursiveQuery(RecursiveQuery {
name,
static_term,
recursive_term,
is_distinct,
+ schema,
})
},
),
diff --git a/datafusion/expr/src/planner.rs b/datafusion/expr/src/planner.rs
index 00f1973572..7aaf3a98cb 100644
--- a/datafusion/expr/src/planner.rs
+++ b/datafusion/expr/src/planner.rs
@@ -61,7 +61,8 @@ pub trait ContextProvider {
not_impl_err!("Table Functions are not supported")
}
- /// Provides an intermediate table that is used to store the results of a CTE during execution
+ /// Provides an intermediate table that is used to expose a recursive CTE
+ /// self-reference during planning and execution.
///
/// CTE stands for "Common Table Expression"
///
@@ -72,6 +73,9 @@ pub trait ContextProvider {
/// of the sql crate (for example [`CteWorkTable`]).
///
/// The [`ContextProvider`] provides a way to "hide" this dependency.
+ /// The schema argument is the schema to expose for scans of the recursive
+ /// self-reference, which may be more conservative than the final recursive
+ /// query output schema.
///
/// [`SqlToRel`]: https://docs.rs/datafusion/latest/datafusion/sql/planner/struct.SqlToRel.html
/// [`CteWorkTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/cte_worktable/struct.CteWorkTable.html
diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rs
index b1dc820cfb..7289ac43e5 100644
--- a/datafusion/physical-plan/src/recursive_query.rs
+++ b/datafusion/physical-plan/src/recursive_query.rs
@@ -35,7 +35,7 @@ use crate::{
};
use arrow::array::{BooleanArray, BooleanBuilder};
use arrow::compute::filter_record_batch;
-use arrow::datatypes::{Field, Schema, SchemaRef};
+use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::{
@@ -84,6 +84,7 @@ impl RecursiveQueryExec {
/// Create a new RecursiveQueryExec
pub fn try_new(
name: String,
+ output_schema: SchemaRef,
static_term: Arc<dyn ExecutionPlan>,
recursive_term: Arc<dyn ExecutionPlan>,
is_distinct: bool,
@@ -91,8 +92,6 @@ impl RecursiveQueryExec {
// Each recursive query needs its own work table
let work_table = Arc::new(WorkTable::new(name.clone()));
// Use the same work table for both the WorkTableExec and the recursive term
- let output_schema =
- recursive_output_schema(&static_term.schema(), &recursive_term.schema());
let static_term = project_plan_to_schema(static_term, &output_schema)?;
let recursive_term = assign_work_table(recursive_term, &work_table)?;
let recursive_term = project_plan_to_schema(recursive_term, &output_schema)?;
@@ -177,6 +176,7 @@ impl ExecutionPlan for RecursiveQueryExec {
) -> Result<Arc<dyn ExecutionPlan>> {
RecursiveQueryExec::try_new(
self.name.clone(),
+ self.schema(),
Arc::clone(&children[0]),
Arc::clone(&children[1]),
self.is_distinct,
@@ -363,30 +363,6 @@ impl RecursiveQueryStream {
}
}
-fn recursive_output_schema(
- static_schema: &SchemaRef,
- recursive_schema: &SchemaRef,
-) -> SchemaRef {
- let fields = static_schema
- .fields()
- .iter()
- .zip(recursive_schema.fields())
- .map(|(static_field, recursive_field)| {
- Field::new(
- static_field.name(),
- static_field.data_type().clone(),
- static_field.is_nullable() || recursive_field.is_nullable(),
- )
- .with_metadata(static_field.metadata().clone())
- })
- .collect::<Vec<_>>();
-
- Arc::new(Schema::new_with_metadata(
- fields,
- static_schema.metadata().clone(),
- ))
-}
-
fn assign_work_table(
plan: Arc<dyn ExecutionPlan>,
work_table: &Arc<WorkTable>,
@@ -537,6 +513,7 @@ mod tests {
let exec = RecursiveQueryExec::try_new(
"numbers".to_string(),
+ static_term.schema(),
Arc::clone(&static_term),
Arc::clone(&recursive_term),
false,
@@ -558,9 +535,15 @@ mod tests {
let static_term = empty_exec(vec![Field::new("value", DataType::Int32, false)]);
let recursive_term =
empty_exec(vec![Field::new("value + Int32(1)", DataType::Int32, true)]);
+ let output_schema = Arc::new(Schema::new(vec![Field::new(
+ "value",
+ DataType::Int32,
+ true,
+ )]));
let exec = RecursiveQueryExec::try_new(
"numbers".to_string(),
+ Arc::clone(&output_schema),
static_term,
recursive_term,
false,
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index 9bb6e74329..49593a6c6a 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -1193,12 +1193,15 @@ impl AsLogicalPlan for LogicalPlanNode {
))?
.try_into_logical_plan(ctx, extension_codec)?;
- Ok(LogicalPlan::RecursiveQuery(RecursiveQuery {
- name: recursive_query_node.name.clone(),
- static_term: Arc::new(static_term),
- recursive_term: Arc::new(recursive_term),
- is_distinct: recursive_query_node.is_distinct,
- }))
+ // The output schema is derived state, so decoding goes through
+ // the constructor after restoring the child terms.
+ RecursiveQuery::try_new(
+ recursive_query_node.name.clone(),
+ Arc::new(static_term),
+ Arc::new(recursive_term),
+ recursive_query_node.is_distinct,
+ )
+ .map(LogicalPlan::RecursiveQuery)
}
LogicalPlanType::CteWorkTableScan(cte_work_table_scan_node) => {
let CteWorkTableScanNode { name, schema } = cte_work_table_scan_node;
diff --git a/datafusion/sql/src/cte.rs b/datafusion/sql/src/cte.rs
index 18766d7056..31cb22f4ef 100644
--- a/datafusion/sql/src/cte.rs
+++ b/datafusion/sql/src/cte.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
use crate::planner::{ContextProvider, PlannerContext, SqlToRel};
+use arrow::datatypes::{Schema, SchemaRef};
use datafusion_common::{
Result, not_impl_err, plan_err,
tree_node::{TreeNode, TreeNodeRecursion},
@@ -127,15 +128,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// in the case of DataFusion).
//
// Since we can't simply register a table during planning stage (it is
- // an execution problem), we'll use a relation object that preserves the
- // schema of the input perfectly and also knows which recursive CTE it is
- // bound to.
+ // an execution problem), we'll use a relation object that knows which
+ // recursive CTE it is bound to.
// ---------- Step 2: Create a temporary relation ------------------
// Step 2.1: Create a table source for the temporary relation
- let work_table_source = self
- .context_provider
- .create_cte_work_table(cte_name, Arc::clone(static_plan.schema().inner()))?;
+ // Recursive self-references must expose conservative (nullable)
+ // columns. Deriving them from the static term's possibly non-nullable
+ // schema would let the recursive term treat values from previous
+ // iterations as non-nullable.
+ let work_table_source = self.context_provider.create_cte_work_table(
+ cte_name,
+ nullable_schema(static_plan.schema().inner()),
+ )?;
// Step 2.2: Create a temporary relation logical plan that will be used
// as the input to the recursive term
@@ -184,6 +189,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
}
}
+/// Return a copy of `schema` with every field marked nullable, preserving field
+/// and schema metadata.
+fn nullable_schema(schema: &Schema) -> SchemaRef {
+ Arc::new(Schema::new_with_metadata(
+ schema
+ .fields()
+ .iter()
+ .map(|field| field.as_ref().clone().with_nullable(true))
+ .collect::<Vec<_>>(),
+ schema.metadata().clone(),
+ ))
+}
+
fn has_work_table_reference(
plan: &LogicalPlan,
work_table_source: &Arc<dyn TableSource>,
diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.slt
index d13e0d4f08..8d85139766 100644
--- a/datafusion/sqllogictest/test_files/cte.slt
+++ b/datafusion/sqllogictest/test_files/cte.slt
@@ -171,7 +171,7 @@ logical_plan
07)--------TableScan: nodes projection=[id]
physical_plan
01)RecursiveQueryExec: name=nodes, is_distinct=false
-02)--ProjectionExec: expr=[1 as id]
+02)--ProjectionExec: expr=[CAST(1 AS Int64) as id]
03)----PlaceholderRowExec
04)--CoalescePartitionsExec
05)----ProjectionExec: expr=[id@0 + 1 as id]
@@ -699,7 +699,7 @@ WITH RECURSIVE region_sales AS (
SELECT
s.salesperson_id AS salesperson_id,
SUM(s.sale_amount) AS amount,
- SUM(0) as level
+ 0 as level
FROM
sales s
GROUP BY
@@ -1079,7 +1079,7 @@ logical_plan
07)--------TableScan: numbers projection=[n]
physical_plan
01)RecursiveQueryExec: name=numbers, is_distinct=false
-02)--ProjectionExec: expr=[1 as n]
+02)--ProjectionExec: expr=[CAST(1 AS Int64) as n]
03)----PlaceholderRowExec
04)--CoalescePartitionsExec
05)----ProjectionExec: expr=[n@0 + 1 as n]
@@ -1104,7 +1104,7 @@ logical_plan
07)--------TableScan: numbers projection=[n]
physical_plan
01)RecursiveQueryExec: name=numbers, is_distinct=false
-02)--ProjectionExec: expr=[1 as n]
+02)--ProjectionExec: expr=[CAST(1 AS Int64) as n]
03)----PlaceholderRowExec
04)--CoalescePartitionsExec
05)----ProjectionExec: expr=[n@0 + 1 as n]
@@ -1161,7 +1161,7 @@ logical_plan
physical_plan
01)GlobalLimitExec: skip=0, fetch=5
02)--RecursiveQueryExec: name=r, is_distinct=false
-03)----ProjectionExec: expr=[0 as k, 0 as v]
+03)----ProjectionExec: expr=[CAST(0 AS Int64) as k, CAST(0 AS Int64) as v]
04)------PlaceholderRowExec
05)----SortExec: TopK(fetch=1), expr=[v@1 ASC NULLS LAST], preserve_partitioning=[false]
06)------WorkTableExec: name=r
@@ -1300,6 +1300,186 @@ DROP TABLE cte_schema_reread;
statement ok
DROP TABLE cte_schema_records;
+##########
+## Recursive CTE nullability widening
+##
+## A recursive term can introduce NULLs that the static (anchor) term never
+## produces. The recursive CTE output schema must therefore widen nullability
+## across both terms, otherwise nullability-based optimizer simplifications
+## (e.g. removing IS NULL / IS NOT NULL predicates) produce wrong results.
+##########
+
+# recursive self-reference must use conservative nullability even when the
+# anchor term uses non-null literals. Otherwise optimizer nullability-based
+# simplification can remove this semantically required IS NOT NULL guard.
+query II rowsort
+WITH RECURSIVE t(a, b) AS (
+ SELECT 0 AS a, 0 AS b
+ UNION ALL
+ SELECT b AS a, CAST(NULL AS INT) AS b FROM t WHERE a IS NOT NULL
+)
+SELECT * FROM t
+----
+0 0
+0 NULL
+NULL NULL
+
+# outer IS NOT NULL filters must see recursive output as nullable, not just the
+# non-null anchor literal.
+query II rowsort
+WITH RECURSIVE t(a, b) AS (
+ SELECT 0 AS a, 0 AS b
+ UNION ALL
+ SELECT b AS a, CAST(NULL AS INT) AS b FROM t WHERE a IS NOT NULL
+)
+SELECT * FROM t WHERE a IS NOT NULL
+----
+0 0
+0 NULL
+
+# outer IS NULL filters must see recursive output as nullable, not just the
+# non-null anchor literal.
+query I
+WITH RECURSIVE t(a) AS (
+ SELECT 1 AS a
+ UNION ALL
+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL
+)
+SELECT * FROM t WHERE a IS NULL
+----
+NULL
+
+# deduplicating recursive CTE must preserve widened nullability for outer filters.
+query I
+WITH RECURSIVE t(a) AS (
+ SELECT 1 AS a
+ UNION
+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL
+)
+SELECT * FROM t WHERE a IS NULL
+----
+NULL
+
+# recursive output nullability must be tracked per column, not just for the
+# first column.
+query II rowsort
+WITH RECURSIVE t(a, b) AS (
+ SELECT 0 AS a, 0 AS b
+ UNION ALL
+ SELECT b AS a, CAST(NULL AS INT) AS b FROM t WHERE a IS NOT NULL
+)
+SELECT * FROM t WHERE b IS NULL
+----
+0 NULL
+NULL NULL
+
+# recursive output nullability must survive recursive term type coercion.
+query I
+WITH RECURSIVE t(a) AS (
+ SELECT 1::INT AS a
+ UNION ALL
+ SELECT CAST(NULL AS BIGINT) AS a FROM t WHERE a IS NOT NULL
+)
+SELECT * FROM t WHERE a IS NULL
+----
+NULL
+
+# DESCRIBE should expose the widened recursive output nullability.
+query TTT
+DESCRIBE WITH RECURSIVE t(a) AS (
+ SELECT 1 AS a
+ UNION ALL
+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL
+)
+SELECT * FROM t
+----
+a Int64 YES
+
+# recursive self-reference must not simplify away IS NULL guards when the
+# anchor term is nullable and the recursive term is non-null.
+query I rowsort
+WITH RECURSIVE t(a) AS (
+ SELECT CAST(NULL AS INT) AS a
+ UNION ALL
+ SELECT 1 AS a FROM t WHERE a IS NULL
+)
+SELECT * FROM t
+----
+1
+NULL
+
+# widened recursive nullability must survive aggregate physical planning.
+query III
+WITH RECURSIVE t(a) AS (
+ SELECT 1 AS a
+ UNION ALL
+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL
+)
+SELECT COUNT(*), COUNT(a), SUM(a) FROM t
+----
+2 1 1
+
+# outer filters must still see widened nullability through a derived projection.
+query I
+WITH RECURSIVE t(a) AS (
+ SELECT 1 AS a
+ UNION ALL
+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL
+)
+SELECT x FROM (SELECT a AS x FROM t) WHERE x IS NULL
+----
+NULL
+
+# per-column nullability widening must survive type coercion in multi-column
+# recursive terms.
+query II
+WITH RECURSIVE t(a, b) AS (
+ SELECT 1::INT AS a, 2::INT AS b
+ UNION ALL
+ SELECT a + 1 AS a, CAST(NULL AS BIGINT) AS b FROM t WHERE a < 2
+)
+SELECT * FROM t WHERE b IS NULL
+----
+2 NULL
+
+# join planning must preserve recursive output nullability for null-sensitive
+# predicates above the recursive query.
+query II
+WITH RECURSIVE t(a) AS (
+ SELECT 1 AS a
+ UNION ALL
+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL
+)
+SELECT t.a, u.b FROM t LEFT JOIN (SELECT 1 AS b) u ON t.a = u.b WHERE u.b IS NULL
+----
+NULL NULL
+
+# A recursive CTE must not inherit the static term's uniqueness / primary-key
+# functional dependencies: the recursive term can append rows that duplicate
+# the static term's "unique" keys, so an outer DISTINCT must not be optimized
+# away based on a stale dependency.
+statement ok
+CREATE TABLE recursive_cte_pk(id INT NOT NULL PRIMARY KEY);
+
+statement ok
+INSERT INTO recursive_cte_pk VALUES (2), (1);
+
+query I rowsort
+SELECT DISTINCT id FROM (
+ WITH RECURSIVE t(id) AS (
+ SELECT id FROM recursive_cte_pk
+ UNION ALL
+ SELECT id - 1 FROM t WHERE id > 1
+ )
+ SELECT id FROM t
+)
+----
+1
+2
+
+statement ok
+DROP TABLE recursive_cte_pk;
+
statement count 0
set datafusion.execution.enable_recursive_ctes = false;
diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt
index 5bb4817be9..d8e90e294f 100644
--- a/datafusion/sqllogictest/test_files/explain_tree.slt
+++ b/datafusion/sqllogictest/test_files/explain_tree.slt
@@ -1577,7 +1577,7 @@ physical_plan
04)┌─────────────┴─────────────┐┌─────────────┴─────────────┐
05)│ ProjectionExec ││ CoalescePartitionsExec │
06)│ -------------------- ││ │
-07)│ id: 1 ││ │
+07)│ id: CAST(1 AS Int64) ││ │
08)└─────────────┬─────────────┘└─────────────┬─────────────┘
09)┌─────────────┴─────────────┐┌─────────────┴─────────────┐
10)│ PlaceholderRowExec ││ ProjectionExec │