Apache DataFusion SQL Query Engine
fix: Correctly compute nullability in recursive CTE schemas (#22552)
Neil Conway08bd332dd74a
diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rsindex dd313ebb4c..5ec688526c 100644--- a/datafusion/catalog/src/cte_worktable.rs+++ b/datafusion/catalog/src/cte_worktable.rs@@ -36,14 +36,16 @@ use crate::{ScanArgs, ScanResult, Session, TableProvider};pub struct CteWorkTable {/// The name of the CTE work tablename: String,- /// This schema must be shared across both the static and recursive terms of a recursive query+ /// Schema exposed by recursive self-references while planning the recursive term.+ ///+ /// This is a conservative work-table schema, not the final recursive query output+ /// schema. For example, the SQL planner may mark fields nullable here so recursive+ /// references do not inherit unsound anchor-term nullability assumptions.table_schema: SchemaRef,}impl CteWorkTable {- /// construct a new CteWorkTable with the given name and schema- /// This schema must match the schema of the recursive term of the query- /// Since the scan method will contain an physical plan that assumes this schema+ /// Construct a new CteWorkTable with the given name and self-reference schema.pub fn new(name: &str, table_schema: SchemaRef) -> Self {Self {name: name.to_owned(),@@ -56,7 +58,7 @@ impl CteWorkTable {&self.name}- /// The schema of the recursive term of the query+ /// The schema exposed by scans of the recursive self-reference.pub fn schema(&self) -> SchemaRef {Arc::clone(&self.table_schema)}diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 5496db3a8d..dd741ee6ff 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -1798,11 +1798,15 @@ impl DefaultPhysicalPlanner {}}LogicalPlan::RecursiveQuery(RecursiveQuery {- name, is_distinct, ..+ name,+ is_distinct,+ schema,+ ..}) => {let [static_term, recursive_term] = children.two()?;Arc::new(RecursiveQueryExec::try_new(name.clone(),+ Arc::clone(schema.inner()),static_term,recursive_term,*is_distinct,diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex 17e3dba14b..a7cec182f7 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -1014,7 +1014,7 @@ async fn parquet_recursive_projection_pushdown() -> Result<()> {SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]RecursiveQueryExec: name=number_series, is_distinct=falseCoalescePartitionsExec- ProjectionExec: expr=[id@0 as id, 1 as level]+ ProjectionExec: expr=[CAST(id@0 AS Int64) as id, CAST(1 AS Int64) as level]FilterExec: id@0 = 1RepartitionExec: partitioning=RoundRobinBatch(NUM_CORES), input_partitions=1DataSourceExec: file_groups={1 group: [[TMP_DIR/hierarchy.parquet]]}, projection=[id], file_type=parquet, predicate=id@0 = 1, pruning_predicate=id_null_count@2 != row_count@3 AND id_min@0 <= 1 AND 1 <= id_max@1, required_guarantees=[id in (1)]diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex 29bc448c8f..2ecb12c30a 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -192,12 +192,13 @@ impl LogicalPlanBuilder {// Ensure that the recursive term has the same field types as the static termlet coerced_recursive_term =coerce_plan_expr_for_schema(recursive_term, self.plan.schema())?;- Ok(Self::from(LogicalPlan::RecursiveQuery(RecursiveQuery {+ let recursive_query = RecursiveQuery::try_new(name,- static_term: self.plan,- recursive_term: Arc::new(coerced_recursive_term),+ self.plan,+ Arc::new(coerced_recursive_term),is_distinct,- })))+ )?;+ Ok(Self::from(LogicalPlan::RecursiveQuery(recursive_query)))}/// Create a values list based relation, and the schema is inferred from data, consumingdiff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex cef20dcd5a..1bfecd06c2 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -354,10 +354,7 @@ impl LogicalPlan {LogicalPlan::Copy(CopyTo { output_schema, .. }) => output_schema,LogicalPlan::Ddl(ddl) => ddl.schema(),LogicalPlan::Unnest(Unnest { schema, .. }) => schema,- LogicalPlan::RecursiveQuery(RecursiveQuery { static_term, .. }) => {- // we take the schema of the static term as the schema of the entire recursive query- static_term.schema()- }+ LogicalPlan::RecursiveQuery(RecursiveQuery { schema, .. }) => schema,}}@@ -741,7 +738,14 @@ impl LogicalPlan {};Ok(LogicalPlan::Distinct(distinct))}- LogicalPlan::RecursiveQuery(_) => Ok(self),+ LogicalPlan::RecursiveQuery(RecursiveQuery {+ name,+ static_term,+ recursive_term,+ is_distinct,+ schema: _,+ }) => RecursiveQuery::try_new(name, static_term, recursive_term, is_distinct)+ .map(LogicalPlan::RecursiveQuery),LogicalPlan::Analyze(_) => Ok(self),LogicalPlan::Explain(_) => Ok(self),LogicalPlan::TableScan(_) => Ok(self),@@ -1081,12 +1085,13 @@ impl LogicalPlan {}) => {self.assert_no_expressions(expr)?;let (static_term, recursive_term) = self.only_two_inputs(inputs)?;- Ok(LogicalPlan::RecursiveQuery(RecursiveQuery {- name: name.clone(),- static_term: Arc::new(static_term),- recursive_term: Arc::new(recursive_term),- is_distinct: *is_distinct,- }))+ RecursiveQuery::try_new(+ name.clone(),+ Arc::new(static_term),+ Arc::new(recursive_term),+ *is_distinct,+ )+ .map(LogicalPlan::RecursiveQuery)}LogicalPlan::Analyze(a) => {self.assert_no_expressions(expr)?;@@ -2262,7 +2267,7 @@ impl PartialOrd for EmptyRelation {/// intermediate table, then empty the intermediate table.////// [Postgres Docs]: https://www.postgresql.org/docs/current/queries-with.html#QUERIES-WITH-RECURSIVE-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash)]+#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct RecursiveQuery {/// Name of the querypub name: String,@@ -2274,6 +2279,90 @@ pub struct RecursiveQuery {/// Should the output of the recursive term be deduplicated (`UNION`) or/// not (`UNION ALL`).pub is_distinct: bool,+ /// Schema exposed to parent plans after reconciling the static and recursive terms.+ pub schema: DFSchemaRef,+}++impl PartialOrd for RecursiveQuery {+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {+ match self.name.partial_cmp(&other.name) {+ Some(Ordering::Equal) => {+ match self.static_term.partial_cmp(&other.static_term) {+ Some(Ordering::Equal) => {+ match self.recursive_term.partial_cmp(&other.recursive_term) {+ Some(Ordering::Equal) => {+ self.is_distinct.partial_cmp(&other.is_distinct)+ }+ cmp => cmp,+ }+ }+ cmp => cmp,+ }+ }+ cmp => cmp,+ }+ // If the query definition compares equal but the derived schema differs,+ // return `None` instead of contradicting `PartialEq` with `Some(Equal)`.+ // TODO (https://github.com/apache/datafusion/issues/17477) avoid recomparing all fields+ .filter(|cmp| *cmp != Ordering::Equal || self == other)+ }+}++impl RecursiveQuery {+ pub fn try_new(+ name: String,+ static_term: Arc<LogicalPlan>,+ recursive_term: Arc<LogicalPlan>,+ is_distinct: bool,+ ) -> Result<Self> {+ let schema =+ recursive_query_output_schema(static_term.schema(), recursive_term.schema())?;+ Ok(Self {+ name,+ static_term,+ recursive_term,+ is_distinct,+ schema,+ })+ }+}++/// Compute a recursive query's output schema by considering both its static and+/// recursive terms.+///+/// Field names, types, and metadata come from the static term. A field is+/// nullable if either the static or the recursive term produces a nullable+/// value in that position, matching how `UNION` reconciles branch nullability.+///+/// Functional dependencies are intentionally dropped: the recursive term+/// appends rows that can duplicate values the static term guarantees unique, so+/// any FDs carried by the static term may not hold over the combined output.+fn recursive_query_output_schema(+ static_schema: &DFSchemaRef,+ recursive_schema: &DFSchemaRef,+) -> Result<DFSchemaRef> {+ if static_schema.fields().len() != recursive_schema.fields().len() {+ return Err(DataFusionError::Plan(format!(+ "Non-recursive term and recursive term must have the same number of columns ({} != {})",+ static_schema.fields().len(),+ recursive_schema.fields().len()+ )));+ }++ let fields = static_schema+ .iter()+ .zip(recursive_schema.fields())+ .map(|((qualifier, static_field), recursive_field)| {+ let nullable = static_field.is_nullable() || recursive_field.is_nullable();+ (+ qualifier.cloned(),+ static_field.as_ref().clone().with_nullable(nullable).into(),+ )+ })+ .collect::<Vec<_>>();++ DFSchema::new_with_metadata(fields, static_schema.metadata().clone())+ .map(DFSchemaRef::new)}/// Values expression. See@@ -4671,6 +4760,74 @@ mod tests {.build()}+ fn recursive_term_scan(name: &str, fields: Vec<Field>) -> Result<Arc<LogicalPlan>> {+ Ok(Arc::new(+ table_scan(Some(name), &Schema::new(fields), None)?.build()?,+ ))+ }++ #[test]+ fn recursive_query_widens_nullability_per_column() -> Result<()> {+ // Column `a` is non-nullable in both terms and must stay non-nullable;+ // column `b` is non-nullable in the static term but nullable in the+ // recursive term, so the output must widen it to nullable.+ let static_term = recursive_term_scan(+ "static",+ vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ],+ )?;+ let recursive_term = recursive_term_scan(+ "rec",+ vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, true),+ ],+ )?;++ let query =+ RecursiveQuery::try_new("t".to_string(), static_term, recursive_term, false)?;++ // Names and types are taken from the static term.+ assert_eq!(query.schema.field(0).name(), "a");+ assert_eq!(query.schema.field(1).name(), "b");+ assert_eq!(query.schema.field(0).data_type(), &DataType::Int32);+ assert_eq!(query.schema.field(1).data_type(), &DataType::Int32);+ // Nullability is widened independently per column.+ assert!(!query.schema.field(0).is_nullable());+ assert!(query.schema.field(1).is_nullable());+ // `schema()` returns the widened recursive-query schema.+ assert_eq!(+ LogicalPlan::RecursiveQuery(query.clone()).schema(),+ &query.schema+ );+ Ok(())+ }++ #[test]+ fn recursive_query_rejects_column_count_mismatch() -> Result<()> {+ let static_term =+ recursive_term_scan("static", vec![Field::new("a", DataType::Int32, false)])?;+ let recursive_term = recursive_term_scan(+ "rec",+ vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ],+ )?;++ let err =+ RecursiveQuery::try_new("t".to_string(), static_term, recursive_term, false)+ .unwrap_err();+ assert!(+ err.strip_backtrace()+ .contains("must have the same number of columns"),+ "unexpected error: {err}"+ );+ Ok(())+ }+#[test]fn test_display_indent() -> Result<()> {let plan = display_plan()?;diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rsindex 2c6be54705..e0cdec9e2c 100644--- a/datafusion/expr/src/logical_plan/tree_node.rs+++ b/datafusion/expr/src/logical_plan/tree_node.rs@@ -337,13 +337,18 @@ impl TreeNode for LogicalPlan {static_term,recursive_term,is_distinct,+ schema,}) => (static_term, recursive_term).map_elements(f)?.update_data(|(static_term, recursive_term)| {+ // Ordinary child rewrites preserve derived schemas. Call+ // `LogicalPlan::recompute_schema` when child schemas should+ // be reconciled again.LogicalPlan::RecursiveQuery(RecursiveQuery {name,static_term,recursive_term,is_distinct,+ schema,})},),diff --git a/datafusion/expr/src/planner.rs b/datafusion/expr/src/planner.rsindex 00f1973572..7aaf3a98cb 100644--- a/datafusion/expr/src/planner.rs+++ b/datafusion/expr/src/planner.rs@@ -61,7 +61,8 @@ pub trait ContextProvider {not_impl_err!("Table Functions are not supported")}- /// Provides an intermediate table that is used to store the results of a CTE during execution+ /// Provides an intermediate table that is used to expose a recursive CTE+ /// self-reference during planning and execution.////// CTE stands for "Common Table Expression"///@@ -72,6 +73,9 @@ pub trait ContextProvider {/// of the sql crate (for example [`CteWorkTable`]).////// The [`ContextProvider`] provides a way to "hide" this dependency.+ /// The schema argument is the schema to expose for scans of the recursive+ /// self-reference, which may be more conservative than the final recursive+ /// query output schema.////// [`SqlToRel`]: https://docs.rs/datafusion/latest/datafusion/sql/planner/struct.SqlToRel.html/// [`CteWorkTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/cte_worktable/struct.CteWorkTable.htmldiff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rsindex b1dc820cfb..7289ac43e5 100644--- a/datafusion/physical-plan/src/recursive_query.rs+++ b/datafusion/physical-plan/src/recursive_query.rs@@ -35,7 +35,7 @@ use crate::{};use arrow::array::{BooleanArray, BooleanBuilder};use arrow::compute::filter_record_batch;-use arrow::datatypes::{Field, Schema, SchemaRef};+use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::{@@ -84,6 +84,7 @@ impl RecursiveQueryExec {/// Create a new RecursiveQueryExecpub fn try_new(name: String,+ output_schema: SchemaRef,static_term: Arc<dyn ExecutionPlan>,recursive_term: Arc<dyn ExecutionPlan>,is_distinct: bool,@@ -91,8 +92,6 @@ impl RecursiveQueryExec {// Each recursive query needs its own work tablelet work_table = Arc::new(WorkTable::new(name.clone()));// Use the same work table for both the WorkTableExec and the recursive term- let output_schema =- recursive_output_schema(&static_term.schema(), &recursive_term.schema());let static_term = project_plan_to_schema(static_term, &output_schema)?;let recursive_term = assign_work_table(recursive_term, &work_table)?;let recursive_term = project_plan_to_schema(recursive_term, &output_schema)?;@@ -177,6 +176,7 @@ impl ExecutionPlan for RecursiveQueryExec {) -> Result<Arc<dyn ExecutionPlan>> {RecursiveQueryExec::try_new(self.name.clone(),+ self.schema(),Arc::clone(&children[0]),Arc::clone(&children[1]),self.is_distinct,@@ -363,30 +363,6 @@ impl RecursiveQueryStream {}}-fn recursive_output_schema(- static_schema: &SchemaRef,- recursive_schema: &SchemaRef,-) -> SchemaRef {- let fields = static_schema- .fields()- .iter()- .zip(recursive_schema.fields())- .map(|(static_field, recursive_field)| {- Field::new(- static_field.name(),- static_field.data_type().clone(),- static_field.is_nullable() || recursive_field.is_nullable(),- )- .with_metadata(static_field.metadata().clone())- })- .collect::<Vec<_>>();-- Arc::new(Schema::new_with_metadata(- fields,- static_schema.metadata().clone(),- ))-}-fn assign_work_table(plan: Arc<dyn ExecutionPlan>,work_table: &Arc<WorkTable>,@@ -537,6 +513,7 @@ mod tests {let exec = RecursiveQueryExec::try_new("numbers".to_string(),+ static_term.schema(),Arc::clone(&static_term),Arc::clone(&recursive_term),false,@@ -558,9 +535,15 @@ mod tests {let static_term = empty_exec(vec![Field::new("value", DataType::Int32, false)]);let recursive_term =empty_exec(vec![Field::new("value + Int32(1)", DataType::Int32, true)]);+ let output_schema = Arc::new(Schema::new(vec![Field::new(+ "value",+ DataType::Int32,+ true,+ )]));let exec = RecursiveQueryExec::try_new("numbers".to_string(),+ Arc::clone(&output_schema),static_term,recursive_term,false,diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex 9bb6e74329..49593a6c6a 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -1193,12 +1193,15 @@ impl AsLogicalPlan for LogicalPlanNode {))?.try_into_logical_plan(ctx, extension_codec)?;- Ok(LogicalPlan::RecursiveQuery(RecursiveQuery {- name: recursive_query_node.name.clone(),- static_term: Arc::new(static_term),- recursive_term: Arc::new(recursive_term),- is_distinct: recursive_query_node.is_distinct,- }))+ // The output schema is derived state, so decoding goes through+ // the constructor after restoring the child terms.+ RecursiveQuery::try_new(+ recursive_query_node.name.clone(),+ Arc::new(static_term),+ Arc::new(recursive_term),+ recursive_query_node.is_distinct,+ )+ .map(LogicalPlan::RecursiveQuery)}LogicalPlanType::CteWorkTableScan(cte_work_table_scan_node) => {let CteWorkTableScanNode { name, schema } = cte_work_table_scan_node;diff --git a/datafusion/sql/src/cte.rs b/datafusion/sql/src/cte.rsindex 18766d7056..31cb22f4ef 100644--- a/datafusion/sql/src/cte.rs+++ b/datafusion/sql/src/cte.rs@@ -19,6 +19,7 @@ use std::sync::Arc;use crate::planner::{ContextProvider, PlannerContext, SqlToRel};+use arrow::datatypes::{Schema, SchemaRef};use datafusion_common::{Result, not_impl_err, plan_err,tree_node::{TreeNode, TreeNodeRecursion},@@ -127,15 +128,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {// in the case of DataFusion).//// Since we can't simply register a table during planning stage (it is- // an execution problem), we'll use a relation object that preserves the- // schema of the input perfectly and also knows which recursive CTE it is- // bound to.+ // an execution problem), we'll use a relation object that knows which+ // recursive CTE it is bound to.// ---------- Step 2: Create a temporary relation ------------------// Step 2.1: Create a table source for the temporary relation- let work_table_source = self- .context_provider- .create_cte_work_table(cte_name, Arc::clone(static_plan.schema().inner()))?;+ // Recursive self-references must expose conservative (nullable)+ // columns. Deriving them from the static term's possibly non-nullable+ // schema would let the recursive term treat values from previous+ // iterations as non-nullable.+ let work_table_source = self.context_provider.create_cte_work_table(+ cte_name,+ nullable_schema(static_plan.schema().inner()),+ )?;// Step 2.2: Create a temporary relation logical plan that will be used// as the input to the recursive term@@ -184,6 +189,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {}}+/// Return a copy of `schema` with every field marked nullable, preserving field+/// and schema metadata.+fn nullable_schema(schema: &Schema) -> SchemaRef {+ Arc::new(Schema::new_with_metadata(+ schema+ .fields()+ .iter()+ .map(|field| field.as_ref().clone().with_nullable(true))+ .collect::<Vec<_>>(),+ schema.metadata().clone(),+ ))+}+fn has_work_table_reference(plan: &LogicalPlan,work_table_source: &Arc<dyn TableSource>,diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.sltindex d13e0d4f08..8d85139766 100644--- a/datafusion/sqllogictest/test_files/cte.slt+++ b/datafusion/sqllogictest/test_files/cte.slt@@ -171,7 +171,7 @@ logical_plan07)--------TableScan: nodes projection=[id]physical_plan01)RecursiveQueryExec: name=nodes, is_distinct=false-02)--ProjectionExec: expr=[1 as id]+02)--ProjectionExec: expr=[CAST(1 AS Int64) as id]03)----PlaceholderRowExec04)--CoalescePartitionsExec05)----ProjectionExec: expr=[id@0 + 1 as id]@@ -699,7 +699,7 @@ WITH RECURSIVE region_sales AS (SELECTs.salesperson_id AS salesperson_id,SUM(s.sale_amount) AS amount,- SUM(0) as level+ 0 as levelFROMsales sGROUP BY@@ -1079,7 +1079,7 @@ logical_plan07)--------TableScan: numbers projection=[n]physical_plan01)RecursiveQueryExec: name=numbers, is_distinct=false-02)--ProjectionExec: expr=[1 as n]+02)--ProjectionExec: expr=[CAST(1 AS Int64) as n]03)----PlaceholderRowExec04)--CoalescePartitionsExec05)----ProjectionExec: expr=[n@0 + 1 as n]@@ -1104,7 +1104,7 @@ logical_plan07)--------TableScan: numbers projection=[n]physical_plan01)RecursiveQueryExec: name=numbers, is_distinct=false-02)--ProjectionExec: expr=[1 as n]+02)--ProjectionExec: expr=[CAST(1 AS Int64) as n]03)----PlaceholderRowExec04)--CoalescePartitionsExec05)----ProjectionExec: expr=[n@0 + 1 as n]@@ -1161,7 +1161,7 @@ logical_planphysical_plan01)GlobalLimitExec: skip=0, fetch=502)--RecursiveQueryExec: name=r, is_distinct=false-03)----ProjectionExec: expr=[0 as k, 0 as v]+03)----ProjectionExec: expr=[CAST(0 AS Int64) as k, CAST(0 AS Int64) as v]04)------PlaceholderRowExec05)----SortExec: TopK(fetch=1), expr=[v@1 ASC NULLS LAST], preserve_partitioning=[false]06)------WorkTableExec: name=r@@ -1300,6 +1300,186 @@ DROP TABLE cte_schema_reread;statement okDROP TABLE cte_schema_records;+##########+## Recursive CTE nullability widening+##+## A recursive term can introduce NULLs that the static (anchor) term never+## produces. The recursive CTE output schema must therefore widen nullability+## across both terms, otherwise nullability-based optimizer simplifications+## (e.g. removing IS NULL / IS NOT NULL predicates) produce wrong results.+##########++# recursive self-reference must use conservative nullability even when the+# anchor term uses non-null literals. Otherwise optimizer nullability-based+# simplification can remove this semantically required IS NOT NULL guard.+query II rowsort+WITH RECURSIVE t(a, b) AS (+ SELECT 0 AS a, 0 AS b+ UNION ALL+ SELECT b AS a, CAST(NULL AS INT) AS b FROM t WHERE a IS NOT NULL+)+SELECT * FROM t+----+0 0+0 NULL+NULL NULL++# outer IS NOT NULL filters must see recursive output as nullable, not just the+# non-null anchor literal.+query II rowsort+WITH RECURSIVE t(a, b) AS (+ SELECT 0 AS a, 0 AS b+ UNION ALL+ SELECT b AS a, CAST(NULL AS INT) AS b FROM t WHERE a IS NOT NULL+)+SELECT * FROM t WHERE a IS NOT NULL+----+0 0+0 NULL++# outer IS NULL filters must see recursive output as nullable, not just the+# non-null anchor literal.+query I+WITH RECURSIVE t(a) AS (+ SELECT 1 AS a+ UNION ALL+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL+)+SELECT * FROM t WHERE a IS NULL+----+NULL++# deduplicating recursive CTE must preserve widened nullability for outer filters.+query I+WITH RECURSIVE t(a) AS (+ SELECT 1 AS a+ UNION+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL+)+SELECT * FROM t WHERE a IS NULL+----+NULL++# recursive output nullability must be tracked per column, not just for the+# first column.+query II rowsort+WITH RECURSIVE t(a, b) AS (+ SELECT 0 AS a, 0 AS b+ UNION ALL+ SELECT b AS a, CAST(NULL AS INT) AS b FROM t WHERE a IS NOT NULL+)+SELECT * FROM t WHERE b IS NULL+----+0 NULL+NULL NULL++# recursive output nullability must survive recursive term type coercion.+query I+WITH RECURSIVE t(a) AS (+ SELECT 1::INT AS a+ UNION ALL+ SELECT CAST(NULL AS BIGINT) AS a FROM t WHERE a IS NOT NULL+)+SELECT * FROM t WHERE a IS NULL+----+NULL++# DESCRIBE should expose the widened recursive output nullability.+query TTT+DESCRIBE WITH RECURSIVE t(a) AS (+ SELECT 1 AS a+ UNION ALL+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL+)+SELECT * FROM t+----+a Int64 YES++# recursive self-reference must not simplify away IS NULL guards when the+# anchor term is nullable and the recursive term is non-null.+query I rowsort+WITH RECURSIVE t(a) AS (+ SELECT CAST(NULL AS INT) AS a+ UNION ALL+ SELECT 1 AS a FROM t WHERE a IS NULL+)+SELECT * FROM t+----+1+NULL++# widened recursive nullability must survive aggregate physical planning.+query III+WITH RECURSIVE t(a) AS (+ SELECT 1 AS a+ UNION ALL+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL+)+SELECT COUNT(*), COUNT(a), SUM(a) FROM t+----+2 1 1++# outer filters must still see widened nullability through a derived projection.+query I+WITH RECURSIVE t(a) AS (+ SELECT 1 AS a+ UNION ALL+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL+)+SELECT x FROM (SELECT a AS x FROM t) WHERE x IS NULL+----+NULL++# per-column nullability widening must survive type coercion in multi-column+# recursive terms.+query II+WITH RECURSIVE t(a, b) AS (+ SELECT 1::INT AS a, 2::INT AS b+ UNION ALL+ SELECT a + 1 AS a, CAST(NULL AS BIGINT) AS b FROM t WHERE a < 2+)+SELECT * FROM t WHERE b IS NULL+----+2 NULL++# join planning must preserve recursive output nullability for null-sensitive+# predicates above the recursive query.+query II+WITH RECURSIVE t(a) AS (+ SELECT 1 AS a+ UNION ALL+ SELECT CAST(NULL AS INT) AS a FROM t WHERE a IS NOT NULL+)+SELECT t.a, u.b FROM t LEFT JOIN (SELECT 1 AS b) u ON t.a = u.b WHERE u.b IS NULL+----+NULL NULL++# A recursive CTE must not inherit the static term's uniqueness / primary-key+# functional dependencies: the recursive term can append rows that duplicate+# the static term's "unique" keys, so an outer DISTINCT must not be optimized+# away based on a stale dependency.+statement ok+CREATE TABLE recursive_cte_pk(id INT NOT NULL PRIMARY KEY);++statement ok+INSERT INTO recursive_cte_pk VALUES (2), (1);++query I rowsort+SELECT DISTINCT id FROM (+ WITH RECURSIVE t(id) AS (+ SELECT id FROM recursive_cte_pk+ UNION ALL+ SELECT id - 1 FROM t WHERE id > 1+ )+ SELECT id FROM t+)+----+1+2++statement ok+DROP TABLE recursive_cte_pk;+statement count 0set datafusion.execution.enable_recursive_ctes = false;diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.sltindex 5bb4817be9..d8e90e294f 100644--- a/datafusion/sqllogictest/test_files/explain_tree.slt+++ b/datafusion/sqllogictest/test_files/explain_tree.slt@@ -1577,7 +1577,7 @@ physical_plan04)┌─────────────┴─────────────┐┌─────────────┴─────────────┐05)│ ProjectionExec ││ CoalescePartitionsExec │06)│ -------------------- ││ │-07)│ id: 1 ││ │+07)│ id: CAST(1 AS Int64) ││ │08)└─────────────┬─────────────┘└─────────────┬─────────────┘09)┌─────────────┴─────────────┐┌─────────────┴─────────────┐10)│ PlaceholderRowExec ││ ProjectionExec │