Apache DataFusion SQL Query Engine
catch errors when simplifying cast(lit(...), ...) and bubble those up (#18332)
Adrian Garcia Badaracco70e91556d220
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex 85e9d9b6a0..56fe95fffd 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -571,7 +571,18 @@ impl TreeNodeRewriter for ConstEvaluator<'_> {ConstSimplifyResult::NotSimplified(s, m) => {Ok(Transformed::no(Expr::Literal(s, m)))}- ConstSimplifyResult::SimplifyRuntimeError(_, expr) => {+ ConstSimplifyResult::SimplifyRuntimeError(err, expr) => {+ // For CAST expressions with literal inputs, propagate the error at plan time rather than deferring to execution time.+ // This provides clearer error messages and fails fast.+ if let Expr::Cast(Cast { ref expr, .. })+ | Expr::TryCast(TryCast { ref expr, .. }) = expr+ {+ if matches!(expr.as_ref(), Expr::Literal(_, _)) {+ return Err(err);+ }+ }+ // For other expressions (like CASE, COALESCE), preserve the original+ // to allow short-circuit evaluation at execution timeOk(Transformed::yes(expr))}},@@ -4968,6 +4979,56 @@ mod tests {);}+ #[test]+ fn simplify_cast_literal() {+ // Test that CAST(literal) expressions are evaluated at plan time++ // CAST(123 AS Int64) should become 123i64+ let expr = Expr::Cast(Cast::new(Box::new(lit(123i32)), DataType::Int64));+ let expected = lit(123i64);+ assert_eq!(simplify(expr), expected);++ // CAST(1761630189642 AS Timestamp(Nanosecond, Some("+00:00")))+ // Integer to timestamp cast+ let expr = Expr::Cast(Cast::new(+ Box::new(lit(1761630189642i64)),+ DataType::Timestamp(+ arrow::datatypes::TimeUnit::Nanosecond,+ Some("+00:00".into()),+ ),+ ));+ // Should evaluate to a timestamp literal+ let result = simplify(expr);+ match result {+ Expr::Literal(ScalarValue::TimestampNanosecond(Some(val), tz), _) => {+ assert_eq!(val, 1761630189642i64);+ assert_eq!(tz.as_deref(), Some("+00:00"));+ }+ other => panic!("Expected TimestampNanosecond literal, got: {other:?}"),+ }++ // Test CAST of invalid string to timestamp - should return an error at plan time+ // This represents the case from the issue: CAST(Utf8("1761630189642") AS Timestamp)+ // "1761630189642" is NOT a valid timestamp string format+ let expr = Expr::Cast(Cast::new(+ Box::new(lit("1761630189642")),+ DataType::Timestamp(+ arrow::datatypes::TimeUnit::Nanosecond,+ Some("+00:00".into()),+ ),+ ));++ // The simplification should now fail with an error at plan time+ let schema = test_schema();+ let props = ExecutionProps::new();+ let simplifier =+ ExprSimplifier::new(SimplifyContext::new(&props).with_schema(schema));+ let result = simplifier.simplify(expr);+ assert!(result.is_err(), "Expected error for invalid cast");+ let err_msg = result.unwrap_err().to_string();+ assert_contains!(err_msg, "Error parsing timestamp");+ }+fn if_not_null(expr: Expr, then: bool) -> Expr {Expr::Case(Case {expr: Some(expr.is_not_null().into()),diff --git a/datafusion/sqllogictest/test_files/arrow_typeof.slt b/datafusion/sqllogictest/test_files/arrow_typeof.sltindex ac32ef821b..cbb20acb2d 100644--- a/datafusion/sqllogictest/test_files/arrow_typeof.slt+++ b/datafusion/sqllogictest/test_files/arrow_typeof.slt@@ -316,7 +316,7 @@ select arrow_cast(interval '30 minutes', 'Duration(Second)');----0 days 0 hours 30 mins 0 secs-query error DataFusion error: This feature is not implemented: Unsupported CAST from Utf8 to Duration\(s\)+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*This feature is not implemented: Unsupported CAST from Utf8 to Duration\(s\)select arrow_cast('30 minutes', 'Duration(Second)');@@ -337,7 +337,7 @@ select arrow_cast(timestamp '2000-01-01T00:00:00Z', 'Timestamp(Nanosecond, Some(----2000-01-01T00:00:00+08:00-statement error DataFusion error: Arrow error: Parser error: Invalid timezone "\+25:00": failed to parse timezone+statement error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Parser error: Invalid timezone "\+25:00": failed to parse timezoneselect arrow_cast(timestamp '2000-01-01T00:00:00', 'Timestamp(Nanosecond, Some( "+25:00" ))');@@ -406,7 +406,7 @@ select arrow_cast([1], 'FixedSizeList(1, Int64)');----[1]-query error DataFusion error: Arrow error: Cast error: Cannot cast to FixedSizeList\(4\): value at index 0 has length 3+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast to FixedSizeList\(4\): value at index 0 has length 3select arrow_cast(make_array(1, 2, 3), 'FixedSizeList(4, Int64)');query ?diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.sltindex a581bcb539..e7ca7a5ae1 100644--- a/datafusion/sqllogictest/test_files/cte.slt+++ b/datafusion/sqllogictest/test_files/cte.slt@@ -764,7 +764,7 @@ WITH RECURSIVE my_cte AS (# Test issue: https://github.com/apache/datafusion/issues/9794# Non-recursive term and recursive term have different types, and cannot be casted-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'abc' to value of Int64 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'abc' to value of Int64 typeWITH RECURSIVE my_cte AS (SELECT 1 AS aUNION ALLdiff --git a/datafusion/sqllogictest/test_files/errors.slt b/datafusion/sqllogictest/test_files/errors.sltindex 3e60423df8..41f747df5b 100644--- a/datafusion/sqllogictest/test_files/errors.slt+++ b/datafusion/sqllogictest/test_files/errors.slt@@ -145,7 +145,7 @@ SELECTLIMIT 5;-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'foo' to value of Int64 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'foo' to value of Int64 typecreate table foo as values (1), ('foo');query error DataFusion error: Error during planning: Substring without for/from is not validdiff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.sltindex 949edb8376..a3234b4e7e 100644--- a/datafusion/sqllogictest/test_files/map.slt+++ b/datafusion/sqllogictest/test_files/map.slt@@ -155,7 +155,7 @@ SELECT MAKE_MAP('POST', 41, 'HEAD', 53, 'PATCH', 30);----{POST: 41, HEAD: 53, PATCH: 30}-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'ab' to value of Int64 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'ab' to value of Int64 typeSELECT MAKE_MAP('POST', 41, 'HEAD', 'ab', 'PATCH', 30);# Map keys can not be NULL@@ -523,7 +523,7 @@ SELECT MAP { 'a': 1, 'b': 3 };----{a: 1, b: 3}-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'a' to value of Int64 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'a' to value of Int64 typeSELECT MAP { 'a': 1, 2: 3 };# accessing map with non-string key@@ -670,7 +670,7 @@ SELECT map_entries(MAP { 'a': 1, 'b': 3 });----[{key: a, value: 1}, {key: b, value: 3}]-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'a' to value of Int64 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'a' to value of Int64 typeSELECT map_entries(MAP { 'a': 1, 2: 3 });query ?@@ -721,7 +721,7 @@ SELECT map_keys(MAP { 'a': 1, 'b': 3 });----[a, b]-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'a' to value of Int64 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'a' to value of Int64 typeSELECT map_keys(MAP { 'a': 1, 2: 3 });query ?@@ -768,7 +768,7 @@ NULL# Tests for map_values-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'a' to value of Int64 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'a' to value of Int64 typeSELECT map_values(MAP { 'a': 1, 2: 3 });query ?diff --git a/datafusion/sqllogictest/test_files/nullif.slt b/datafusion/sqllogictest/test_files/nullif.sltindex 6acb9aea26..7b4c59b263 100644--- a/datafusion/sqllogictest/test_files/nullif.slt+++ b/datafusion/sqllogictest/test_files/nullif.slt@@ -112,7 +112,7 @@ select nullif(1.0, 2);----1-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'a' to value of Int64 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'a' to value of Int64 typeselect nullif(2, 'a');query Tdiff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.sltindex cd1f90c42e..5b2587bdc3 100644--- a/datafusion/sqllogictest/test_files/select.slt+++ b/datafusion/sqllogictest/test_files/select.slt@@ -1775,7 +1775,7 @@ DROP TABLE test;query error DataFusion error: Arrow error: Parser error: Error parsing timestamp from 'I AM NOT A TIMESTAMP': error parsing dateSELECT to_timestamp('I AM NOT A TIMESTAMP');-query error DataFusion error: Arrow error: Cast error: Cannot cast string '' to value of Int32 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string '' to value of Int32 typeSELECT CAST('' AS int);# See issue: https://github.com/apache/datafusion/issues/8978diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex 0e3c5145d1..dce5fe036b 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -492,7 +492,7 @@ Struct("r": nullable Utf8, "c": nullable Float64)statement okdrop table t;-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'a' to value of Float64 type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'a' to value of Float64 typecreate table t as values({r: 'a', c: 1}), ({c: 2.3, r: 'b'});##################################@@ -554,14 +554,14 @@ statement okdrop table t;# row() with incorrect order-statement error DataFusion error: Arrow error: Cast error: Cannot cast string 'blue' to value of Float32 type-create table t(a struct(r varchar, c int), b struct(r varchar, c float)) as values+statement error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'blue' to value of Float32 type+create table t(a struct(r varchar, c int), b struct(r varchar, c float)) as values(row('red', 1), row(2.3, 'blue')),(row('purple', 1), row('green', 2.3));# out of order struct literal# TODO: This query should not fail-statement error DataFusion error: Arrow error: Cast error: Cannot cast string 'b' to value of Int32 type+statement error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'b' to value of Int32 typecreate table t(a struct(r varchar, c int)) as values ({r: 'a', c: 1}), ({c: 2, r: 'b'});##################################diff --git a/datafusion/sqllogictest/test_files/timestamps.slt b/datafusion/sqllogictest/test_files/timestamps.sltindex 250d4e9830..cdacad0fda 100644--- a/datafusion/sqllogictest/test_files/timestamps.slt+++ b/datafusion/sqllogictest/test_files/timestamps.slt@@ -691,11 +691,11 @@ select----08:09:10.123456789 13:14:15.123456 13:14:15.123 13:14:15-query error DataFusion error: Arrow error: Cast error: Cannot cast string 'not a time' to value of Time64\(ns\) type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'not a time' to value of Time64\(ns\) typeSELECT TIME 'not a time' as time;# invalid time-query error DataFusion error: Arrow error: Cast error: Cannot cast string '24:01:02' to value of Time64\(ns\) type+query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string '24:01:02' to value of Time64\(ns\) typeSELECT TIME '24:01:02' as time;# invalid timezone@@ -3271,7 +3271,7 @@ statement error The to_local_time function can only accept Timestamp as the argselect to_local_time('2024-04-01T00:00:20Z');# invalid timezone-statement error DataFusion error: Arrow error: Parser error: Invalid timezone "Europe/timezone": failed to parse timezone+statement error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Parser error: Invalid timezone "Europe/timezone": failed to parse timezoneselect to_local_time('2024-04-01T00:00:20Z'::timestamp AT TIME ZONE 'Europe/timezone');# valid query