Apache DataFusion SQL Query Engine
fix: validate recursive CTE column coercion (#25956)
Goutam Adwant7de176016126
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex f443043d38..fe97ab8752 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -61,7 +61,9 @@ use datafusion_common::{get_target_functional_dependencies, internal_datafusion_err, plan_datafusion_err,plan_err,};-use datafusion_expr_common::type_coercion::binary::type_union_resolution;+use datafusion_expr_common::type_coercion::binary::{+ type_union_coercion, type_union_resolution,+};use indexmap::IndexSet;@@ -88,6 +90,95 @@ impl LogicalPlanBuilderOptions {}}+fn recursive_type_can_be_coerced(from: &DataType, into: &DataType) -> bool {+ if from == into || *from == DataType::Null {+ return true;+ }+ // Preserve the integer coercion supported by recursive queries since #9794.+ if from.is_integer() && into.is_integer() {+ return true;+ }+ if matches!(+ (from, into),+ (DataType::Date32, DataType::Date64)+ | (DataType::Utf8, DataType::LargeUtf8 | DataType::Utf8View)+ | (DataType::LargeUtf8, DataType::Utf8View)+ | (+ DataType::Binary,+ DataType::LargeBinary | DataType::BinaryView+ )+ | (DataType::LargeBinary, DataType::BinaryView)+ | (+ DataType::FixedSizeBinary(_),+ DataType::Binary | DataType::BinaryView+ )+ ) {+ return true;+ }+ // Compound types and changes of type category need an explicit seed type.+ // UNION's common-type rules can hide lossy casts within nested fields.+ if !from.is_numeric() || !into.is_numeric() {+ return false;+ }+ // The decimal range/scale checks below are more precise than UNION coercion,+ // which reserves 20 digits for Int64 even though its values fit in 19.+ if !into.is_decimal() && type_union_coercion(into, from).as_ref() != Some(into) {+ return false;+ }++ match (from, into) {+ (+ DataType::Int16+ | DataType::UInt16+ | DataType::Int32+ | DataType::UInt32+ | DataType::Int64+ | DataType::UInt64,+ DataType::Float16,+ )+ | (+ DataType::Int32 | DataType::UInt32 | DataType::Int64 | DataType::UInt64,+ DataType::Float32,+ )+ | (DataType::Int64 | DataType::UInt64, DataType::Float64) => false,+ (from, into) if from.is_decimal() && into.is_floating() => false,+ (from, into) if into.is_decimal() => {+ if from.is_floating() {+ return false;+ }+ let Some((into_precision, into_scale)) = decimal_precision_scale(into) else {+ return false;+ };+ let from_precision_scale = match from {+ DataType::Int8 | DataType::UInt8 => Some((3, 0)),+ DataType::Int16 | DataType::UInt16 => Some((5, 0)),+ DataType::Int32 | DataType::UInt32 => Some((10, 0)),+ DataType::Int64 => Some((19, 0)),+ DataType::UInt64 => Some((20, 0)),+ _ => decimal_precision_scale(from),+ };+ let Some((from_precision, from_scale)) = from_precision_scale else {+ return false;+ };+ into_scale >= from_scale+ && into_precision - into_scale >= from_precision - from_scale+ }+ _ => true,+ }+}++fn decimal_precision_scale(data_type: &DataType) -> Option<(i16, i16)> {+ match data_type {+ DataType::Decimal32(precision, scale)+ | DataType::Decimal64(precision, scale)+ | DataType::Decimal128(precision, scale)+ | DataType::Decimal256(precision, scale) => {+ Some((*precision as i16, *scale as i16))+ }+ _ => None,+ }+}+/// Builder for logical plans////// # Example building a simple plan@@ -191,6 +282,23 @@ impl LogicalPlanBuilder {recursive_fields_len);}+ for (index, (static_field, recursive_field)) in self+ .plan+ .schema()+ .fields()+ .iter()+ .zip(recursive_term.schema().fields())+ .enumerate()+ {+ let static_type = static_field.data_type();+ let recursive_type = recursive_field.data_type();+ if !recursive_type_can_be_coerced(recursive_type, static_type) {+ return plan_err!(+ "Recursive CTE '{name}' column {} has type {static_type} in the non-recursive term but {recursive_type} in the recursive term; cast the non-recursive term to the intended type",+ index + 1+ );+ }+ }// Ensure that the recursive term has the same field types as the static termlet coerced_recursive_term =coerce_plan_expr_for_schema(recursive_term, self.plan.schema())?;diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.sltindex e0f8a4d4da..a7679e1483 100644--- a/datafusion/sqllogictest/test_files/cte.slt+++ b/datafusion/sqllogictest/test_files/cte.slt@@ -977,6 +977,182 @@ WITH RECURSIVE my_cte AS(1 Int323 Int32+# Test issue: https://github.com/apache/datafusion/issues/25893+# A fractional recursive value must not be truncated to the seed's integer type.+# EXPLAIN checks planning without running a potentially non-terminating query.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Int64 in the non-recursive term but Float64 in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT 1::BIGINT AS a+ UNION ALL+ SELECT a::DOUBLE + 0.5::DOUBLE FROM my_cte WHERE a < 3+) SELECT * FROM my_cte;++# Decimal recursion must likewise not be coerced to an integer seed.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Int64 in the non-recursive term but Decimal128.* in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT 1::BIGINT AS a+ UNION ALL+ SELECT a::DECIMAL(10, 1) + 0.5::DECIMAL(10, 1) FROM my_cte WHERE a < 3+) SELECT * FROM my_cte;++# Casting the seed to the intended result type preserves fractional values.+query RT+WITH RECURSIVE my_cte AS (+ SELECT 1::DOUBLE AS a+ UNION ALL+ SELECT a + 0.5::DOUBLE FROM my_cte WHERE a < 2+) SELECT a, arrow_typeof(a) FROM my_cte ORDER BY a;+----+1 Float64+1.5 Float64+2 Float64++# Safe decimal widening remains supported.+query I+WITH RECURSIVE my_cte AS (+ SELECT 1::DECIMAL(20, 5) AS a+ UNION ALL+ SELECT 2::DECIMAL(10, 2) FROM my_cte WHERE a < 2+) SELECT count(*) FROM my_cte;+----+2++# Every signed BIGINT value fits in DECIMAL(19, 0), even though UNION's+# common-type rules reserve 20 digits for both signed and unsigned BIGINT.+query TT rowsort+WITH RECURSIVE my_cte AS (+ SELECT 0::DECIMAL(19, 0) AS lo, 0::DECIMAL(19, 0) AS hi+ UNION ALL+ SELECT '-9223372036854775808'::BIGINT, '9223372036854775807'::BIGINT+ FROM my_cte WHERE lo = 0+) SELECT lo::VARCHAR, hi::VARCHAR FROM my_cte;+----+-9223372036854775808 9223372036854775807+0 0++# Unsigned BIGINT requires all 20 integer digits.+query T rowsort+WITH RECURSIVE my_cte AS (+ SELECT 0::DECIMAL(20, 0) AS a+ UNION ALL+ SELECT arrow_cast('18446744073709551615', 'UInt64') FROM my_cte WHERE a = 0+) SELECT a::VARCHAR FROM my_cte;+----+0+18446744073709551615++# Checking decimal destinations before UNION coercion must still reject types+# whose complete range does not fit, even when the recursive value itself fits.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Int64 in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT 1::DECIMAL(18, 0) AS a+ UNION ALL+ SELECT 2::BIGINT FROM my_cte WHERE a < 2+) SELECT * FROM my_cte;++query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but UInt64 in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT 1::DECIMAL(19, 0) AS a+ UNION ALL+ SELECT arrow_cast(2, 'UInt64') FROM my_cte WHERE a < 2+) SELECT * FROM my_cte;++# Fractional digits leave too little room for the complete signed BIGINT range.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Int64 in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT 1::DECIMAL(19, 1) AS a+ UNION ALL+ SELECT 2::BIGINT FROM my_cte WHERE a < 2+) SELECT * FROM my_cte;++# Decimal-to-decimal coercion must not discard fractional digits either.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Decimal128.* in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT 1::DECIMAL(20, 1) AS a+ UNION ALL+ SELECT 2::DECIMAL(10, 2) FROM my_cte WHERE a < 2+) SELECT * FROM my_cte;++# Safe widening within date and string types remains supported.+query I+WITH RECURSIVE my_cte AS (+ SELECT arrow_cast(DATE '1970-01-01', 'Date64') AS a+ UNION ALL+ SELECT DATE '1970-01-02' FROM my_cte+ WHERE a = arrow_cast(DATE '1970-01-01', 'Date64')+) SELECT count(*) FROM my_cte;+----+2++query I+WITH RECURSIVE my_cte AS (+ SELECT arrow_cast('a', 'LargeUtf8') AS a+ UNION ALL+ SELECT 'b' FROM my_cte WHERE a = 'a'+) SELECT count(*) FROM my_cte;+----+2++query I+WITH RECURSIVE my_cte AS (+ SELECT arrow_cast('a', 'Utf8View') AS a+ UNION ALL+ SELECT arrow_cast('b', 'LargeUtf8') FROM my_cte WHERE a = 'a'+) SELECT count(*) FROM my_cte;+----+2++query I+WITH RECURSIVE my_cte AS (+ SELECT arrow_cast(X'616263', 'Binary') AS a+ UNION ALL+ SELECT arrow_cast(X'646566', 'FixedSizeBinary(3)') FROM my_cte+ WHERE a = X'616263'+) SELECT count(*) FROM my_cte;+----+2++# An integer wider than the floating-point mantissa can round back to the seed.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Float64 in the non-recursive term but Int64 in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT 9007199254740992::DOUBLE AS a+ UNION ALL+ SELECT a::BIGINT + 1 FROM my_cte WHERE a = 9007199254740992::DOUBLE+) SELECT * FROM my_cte;++# A floating-point recursive value cannot be represented at arbitrary decimal scale.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Float64 in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT 1::DECIMAL(30, 15) AS a+ UNION ALL+ SELECT a::DOUBLE + 0.1234567891234567::DOUBLE FROM my_cte WHERE a < 2+) SELECT * FROM my_cte;++# Nested numeric coercions must not bypass the column type check.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Struct.* in the non-recursive term but Struct.* in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT named_struct('n', 9007199254740992::DOUBLE) AS s+ UNION ALL+ SELECT named_struct('n', s['n']::BIGINT + 1) FROM my_cte+ WHERE s['n'] = 9007199254740992::DOUBLE+) SELECT * FROM my_cte;++# A date seed must not silently coerce an integer recursive result.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Date32 in the non-recursive term but Int64 in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT DATE '1970-01-01' AS a+ UNION ALL+ SELECT 4294967296::BIGINT FROM my_cte WHERE a = DATE '1970-01-01'+) SELECT * FROM my_cte;++# A high-scale decimal seed has too few integer digits for an Int64 result.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Int64 in the recursive term+EXPLAIN WITH RECURSIVE my_cte AS (+ SELECT 1::DECIMAL(38, 37) AS a+ UNION ALL+ SELECT 123::BIGINT FROM my_cte WHERE a < 2+) SELECT * FROM my_cte;+# Test issue: https://github.com/apache/datafusion/issues/9794# Non-recursive term and recursive term have different number of columnsquery error DataFusion error: Error during planning: Non\-recursive term and recursive term must have the same number of columns \(1 != 3\)@@ -987,8 +1163,8 @@ WITH RECURSIVE my_cte AS () SELECT * FROM my_cte;# Test issue: https://github.com/apache/datafusion/issues/9794-# Non-recursive term and recursive term have different types, and cannot be casted-query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'abc' to value of Int64 type+# Non-recursive term and recursive term have incompatible types.+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Int64 in the non-recursive term but Utf8 in the recursive termWITH RECURSIVE my_cte AS (SELECT 1 AS aUNION ALL