Skip to content

Apache DataFusion SQL Query Engine

fix: validate recursive CTE column coercion (#25956)

Goutam Adwant7de176016126
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
index f443043d38..fe97ab8752 100644
--- a/datafusion/expr/src/logical_plan/builder.rs
+++ b/datafusion/expr/src/logical_plan/builder.rs
@@ -61,7 +61,9 @@ use datafusion_common::{
get_target_functional_dependencies, internal_datafusion_err, plan_datafusion_err,
plan_err,
};
-use datafusion_expr_common::type_coercion::binary::type_union_resolution;
+use datafusion_expr_common::type_coercion::binary::{
+ type_union_coercion, type_union_resolution,
+};
use indexmap::IndexSet;
@@ -88,6 +90,95 @@ impl LogicalPlanBuilderOptions {
}
}
+fn recursive_type_can_be_coerced(from: &DataType, into: &DataType) -> bool {
+ if from == into || *from == DataType::Null {
+ return true;
+ }
+ // Preserve the integer coercion supported by recursive queries since #9794.
+ if from.is_integer() && into.is_integer() {
+ return true;
+ }
+ if matches!(
+ (from, into),
+ (DataType::Date32, DataType::Date64)
+ | (DataType::Utf8, DataType::LargeUtf8 | DataType::Utf8View)
+ | (DataType::LargeUtf8, DataType::Utf8View)
+ | (
+ DataType::Binary,
+ DataType::LargeBinary | DataType::BinaryView
+ )
+ | (DataType::LargeBinary, DataType::BinaryView)
+ | (
+ DataType::FixedSizeBinary(_),
+ DataType::Binary | DataType::BinaryView
+ )
+ ) {
+ return true;
+ }
+ // Compound types and changes of type category need an explicit seed type.
+ // UNION's common-type rules can hide lossy casts within nested fields.
+ if !from.is_numeric() || !into.is_numeric() {
+ return false;
+ }
+ // The decimal range/scale checks below are more precise than UNION coercion,
+ // which reserves 20 digits for Int64 even though its values fit in 19.
+ if !into.is_decimal() && type_union_coercion(into, from).as_ref() != Some(into) {
+ return false;
+ }
+
+ match (from, into) {
+ (
+ DataType::Int16
+ | DataType::UInt16
+ | DataType::Int32
+ | DataType::UInt32
+ | DataType::Int64
+ | DataType::UInt64,
+ DataType::Float16,
+ )
+ | (
+ DataType::Int32 | DataType::UInt32 | DataType::Int64 | DataType::UInt64,
+ DataType::Float32,
+ )
+ | (DataType::Int64 | DataType::UInt64, DataType::Float64) => false,
+ (from, into) if from.is_decimal() && into.is_floating() => false,
+ (from, into) if into.is_decimal() => {
+ if from.is_floating() {
+ return false;
+ }
+ let Some((into_precision, into_scale)) = decimal_precision_scale(into) else {
+ return false;
+ };
+ let from_precision_scale = match from {
+ DataType::Int8 | DataType::UInt8 => Some((3, 0)),
+ DataType::Int16 | DataType::UInt16 => Some((5, 0)),
+ DataType::Int32 | DataType::UInt32 => Some((10, 0)),
+ DataType::Int64 => Some((19, 0)),
+ DataType::UInt64 => Some((20, 0)),
+ _ => decimal_precision_scale(from),
+ };
+ let Some((from_precision, from_scale)) = from_precision_scale else {
+ return false;
+ };
+ into_scale >= from_scale
+ && into_precision - into_scale >= from_precision - from_scale
+ }
+ _ => true,
+ }
+}
+
+fn decimal_precision_scale(data_type: &DataType) -> Option<(i16, i16)> {
+ match data_type {
+ DataType::Decimal32(precision, scale)
+ | DataType::Decimal64(precision, scale)
+ | DataType::Decimal128(precision, scale)
+ | DataType::Decimal256(precision, scale) => {
+ Some((*precision as i16, *scale as i16))
+ }
+ _ => None,
+ }
+}
+
/// Builder for logical plans
///
/// # Example building a simple plan
@@ -191,6 +282,23 @@ impl LogicalPlanBuilder {
recursive_fields_len
);
}
+ for (index, (static_field, recursive_field)) in self
+ .plan
+ .schema()
+ .fields()
+ .iter()
+ .zip(recursive_term.schema().fields())
+ .enumerate()
+ {
+ let static_type = static_field.data_type();
+ let recursive_type = recursive_field.data_type();
+ if !recursive_type_can_be_coerced(recursive_type, static_type) {
+ return plan_err!(
+ "Recursive CTE '{name}' column {} has type {static_type} in the non-recursive term but {recursive_type} in the recursive term; cast the non-recursive term to the intended type",
+ index + 1
+ );
+ }
+ }
// Ensure that the recursive term has the same field types as the static term
let coerced_recursive_term =
coerce_plan_expr_for_schema(recursive_term, self.plan.schema())?;
diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.slt
index e0f8a4d4da..a7679e1483 100644
--- a/datafusion/sqllogictest/test_files/cte.slt
+++ b/datafusion/sqllogictest/test_files/cte.slt
@@ -977,6 +977,182 @@ WITH RECURSIVE my_cte AS(
1 Int32
3 Int32
+# Test issue: https://github.com/apache/datafusion/issues/25893
+# A fractional recursive value must not be truncated to the seed's integer type.
+# EXPLAIN checks planning without running a potentially non-terminating query.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Int64 in the non-recursive term but Float64 in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT 1::BIGINT AS a
+ UNION ALL
+ SELECT a::DOUBLE + 0.5::DOUBLE FROM my_cte WHERE a < 3
+) SELECT * FROM my_cte;
+
+# Decimal recursion must likewise not be coerced to an integer seed.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Int64 in the non-recursive term but Decimal128.* in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT 1::BIGINT AS a
+ UNION ALL
+ SELECT a::DECIMAL(10, 1) + 0.5::DECIMAL(10, 1) FROM my_cte WHERE a < 3
+) SELECT * FROM my_cte;
+
+# Casting the seed to the intended result type preserves fractional values.
+query RT
+WITH RECURSIVE my_cte AS (
+ SELECT 1::DOUBLE AS a
+ UNION ALL
+ SELECT a + 0.5::DOUBLE FROM my_cte WHERE a < 2
+) SELECT a, arrow_typeof(a) FROM my_cte ORDER BY a;
+----
+1 Float64
+1.5 Float64
+2 Float64
+
+# Safe decimal widening remains supported.
+query I
+WITH RECURSIVE my_cte AS (
+ SELECT 1::DECIMAL(20, 5) AS a
+ UNION ALL
+ SELECT 2::DECIMAL(10, 2) FROM my_cte WHERE a < 2
+) SELECT count(*) FROM my_cte;
+----
+2
+
+# Every signed BIGINT value fits in DECIMAL(19, 0), even though UNION's
+# common-type rules reserve 20 digits for both signed and unsigned BIGINT.
+query TT rowsort
+WITH RECURSIVE my_cte AS (
+ SELECT 0::DECIMAL(19, 0) AS lo, 0::DECIMAL(19, 0) AS hi
+ UNION ALL
+ SELECT '-9223372036854775808'::BIGINT, '9223372036854775807'::BIGINT
+ FROM my_cte WHERE lo = 0
+) SELECT lo::VARCHAR, hi::VARCHAR FROM my_cte;
+----
+-9223372036854775808 9223372036854775807
+0 0
+
+# Unsigned BIGINT requires all 20 integer digits.
+query T rowsort
+WITH RECURSIVE my_cte AS (
+ SELECT 0::DECIMAL(20, 0) AS a
+ UNION ALL
+ SELECT arrow_cast('18446744073709551615', 'UInt64') FROM my_cte WHERE a = 0
+) SELECT a::VARCHAR FROM my_cte;
+----
+0
+18446744073709551615
+
+# Checking decimal destinations before UNION coercion must still reject types
+# whose complete range does not fit, even when the recursive value itself fits.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Int64 in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT 1::DECIMAL(18, 0) AS a
+ UNION ALL
+ SELECT 2::BIGINT FROM my_cte WHERE a < 2
+) SELECT * FROM my_cte;
+
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but UInt64 in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT 1::DECIMAL(19, 0) AS a
+ UNION ALL
+ SELECT arrow_cast(2, 'UInt64') FROM my_cte WHERE a < 2
+) SELECT * FROM my_cte;
+
+# Fractional digits leave too little room for the complete signed BIGINT range.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Int64 in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT 1::DECIMAL(19, 1) AS a
+ UNION ALL
+ SELECT 2::BIGINT FROM my_cte WHERE a < 2
+) SELECT * FROM my_cte;
+
+# Decimal-to-decimal coercion must not discard fractional digits either.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Decimal128.* in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT 1::DECIMAL(20, 1) AS a
+ UNION ALL
+ SELECT 2::DECIMAL(10, 2) FROM my_cte WHERE a < 2
+) SELECT * FROM my_cte;
+
+# Safe widening within date and string types remains supported.
+query I
+WITH RECURSIVE my_cte AS (
+ SELECT arrow_cast(DATE '1970-01-01', 'Date64') AS a
+ UNION ALL
+ SELECT DATE '1970-01-02' FROM my_cte
+ WHERE a = arrow_cast(DATE '1970-01-01', 'Date64')
+) SELECT count(*) FROM my_cte;
+----
+2
+
+query I
+WITH RECURSIVE my_cte AS (
+ SELECT arrow_cast('a', 'LargeUtf8') AS a
+ UNION ALL
+ SELECT 'b' FROM my_cte WHERE a = 'a'
+) SELECT count(*) FROM my_cte;
+----
+2
+
+query I
+WITH RECURSIVE my_cte AS (
+ SELECT arrow_cast('a', 'Utf8View') AS a
+ UNION ALL
+ SELECT arrow_cast('b', 'LargeUtf8') FROM my_cte WHERE a = 'a'
+) SELECT count(*) FROM my_cte;
+----
+2
+
+query I
+WITH RECURSIVE my_cte AS (
+ SELECT arrow_cast(X'616263', 'Binary') AS a
+ UNION ALL
+ SELECT arrow_cast(X'646566', 'FixedSizeBinary(3)') FROM my_cte
+ WHERE a = X'616263'
+) SELECT count(*) FROM my_cte;
+----
+2
+
+# An integer wider than the floating-point mantissa can round back to the seed.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Float64 in the non-recursive term but Int64 in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT 9007199254740992::DOUBLE AS a
+ UNION ALL
+ SELECT a::BIGINT + 1 FROM my_cte WHERE a = 9007199254740992::DOUBLE
+) SELECT * FROM my_cte;
+
+# A floating-point recursive value cannot be represented at arbitrary decimal scale.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Float64 in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT 1::DECIMAL(30, 15) AS a
+ UNION ALL
+ SELECT a::DOUBLE + 0.1234567891234567::DOUBLE FROM my_cte WHERE a < 2
+) SELECT * FROM my_cte;
+
+# Nested numeric coercions must not bypass the column type check.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Struct.* in the non-recursive term but Struct.* in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT named_struct('n', 9007199254740992::DOUBLE) AS s
+ UNION ALL
+ SELECT named_struct('n', s['n']::BIGINT + 1) FROM my_cte
+ WHERE s['n'] = 9007199254740992::DOUBLE
+) SELECT * FROM my_cte;
+
+# A date seed must not silently coerce an integer recursive result.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Date32 in the non-recursive term but Int64 in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT DATE '1970-01-01' AS a
+ UNION ALL
+ SELECT 4294967296::BIGINT FROM my_cte WHERE a = DATE '1970-01-01'
+) SELECT * FROM my_cte;
+
+# A high-scale decimal seed has too few integer digits for an Int64 result.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Decimal128.* in the non-recursive term but Int64 in the recursive term
+EXPLAIN WITH RECURSIVE my_cte AS (
+ SELECT 1::DECIMAL(38, 37) AS a
+ UNION ALL
+ SELECT 123::BIGINT FROM my_cte WHERE a < 2
+) SELECT * FROM my_cte;
+
# Test issue: https://github.com/apache/datafusion/issues/9794
# Non-recursive term and recursive term have different number of columns
query error DataFusion error: Error during planning: Non\-recursive term and recursive term must have the same number of columns \(1 != 3\)
@@ -987,8 +1163,8 @@ WITH RECURSIVE my_cte AS (
) SELECT * FROM my_cte;
# Test issue: https://github.com/apache/datafusion/issues/9794
-# Non-recursive term and recursive term have different types, and cannot be casted
-query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'abc' to value of Int64 type
+# Non-recursive term and recursive term have incompatible types.
+query error DataFusion error: Error during planning: Recursive CTE 'my_cte' column 1 has type Int64 in the non-recursive term but Utf8 in the recursive term
WITH RECURSIVE my_cte AS (
SELECT 1 AS a
UNION ALL