Skip to content

Apache DataFusion SQL Query Engine

test: Add regression tests for some `unwrap_cast` related bugs that don't reproduce (#27018)

Adam Gutglick17401bbac0b9
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt
index ce525005de..878e477ed3 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt
@@ -25,6 +25,7 @@
## https://github.com/apache/datafusion/issues/25166
## https://github.com/apache/datafusion/issues/25167
## https://github.com/apache/datafusion/issues/25170
+## https://github.com/apache/datafusion/issues/25941
##
## Cross-engine notes below quote PostgreSQL with the session `TimeZone` named
## in the comment. The subset the two engines agree on is machine checked
@@ -488,14 +489,11 @@ RESET datafusion.execution.time_zone
##########
## SECTION 8: comparison and equality across tz-aware and tz-naive
##
-## The pins here record three behaviours. A tz-naive side is read in the other
+## The pins here record two behaviours. A tz-naive side is read in the other
## operand's zone, where PostgreSQL uses the session zone:
## https://github.com/apache/datafusion/issues/13212
## `'...Z'::timestamptz` is tz-naive while the session zone is unset
## (SECTION 0): https://github.com/apache/datafusion/issues/25166
-## And under a non-UTC session zone, unwrap_cast rewrites a tz-naive column
-## compared against a tz-aware literal into a different comparison:
-## https://github.com/apache/datafusion/issues/25095
##########
statement ok
@@ -622,6 +620,114 @@ set datafusion.explain.logical_plan_only = false
statement ok
RESET datafusion.execution.time_zone
+# tz-naive column cast to a tz-aware type and compared against a tz-aware
+# literal under a non-UTC session zone. 2024-11-01 00:00 in Asia/Singapore
+# (+08:00) is 2024-10-31 16:00 UTC, so the literal comparison must agree with the
+# column-vs-column comparison. PostgreSQL and DuckDB under
+# TimeZone='Asia/Singapore' return 1, 0, 1, 1.
+# Regression test for https://github.com/apache/datafusion/issues/25095
+statement ok
+SET datafusion.execution.time_zone = 'Asia/Singapore'
+
+statement ok
+CREATE TABLE issue_25095_t AS SELECT TIMESTAMP '2024-11-01T00:00:00' AS ts
+
+statement ok
+CREATE TABLE issue_25095_u AS SELECT '2024-10-31T16:00:00Z'::timestamptz AS tstz
+
+# Explicit cast, matching instant
+query I
+SELECT count(*) FROM issue_25095_t WHERE ts::timestamptz = '2024-10-31T16:00:00Z'::timestamptz
+----
+1
+
+# Explicit cast, wrong instant
+query I
+SELECT count(*) FROM issue_25095_t WHERE ts::timestamptz = '2024-11-01T00:00:00Z'::timestamptz
+----
+0
+
+# Implicit coercion instead of an explicit cast
+query I
+SELECT count(*) FROM issue_25095_t WHERE ts = '2024-10-31T16:00:00Z'::timestamptz
+----
+1
+
+# Control: column vs column, nothing to unwrap
+query I
+SELECT count(*) FROM issue_25095_t, issue_25095_u WHERE issue_25095_t.ts::timestamptz = issue_25095_u.tstz
+----
+1
+
+statement ok
+RESET datafusion.execution.time_zone
+
+statement ok
+DROP TABLE issue_25095_t
+
+statement ok
+DROP TABLE issue_25095_u
+
+# tz-naive column cast to a fixed offset zone and compared against a literal in
+# that zone. Both sides display as 2024-01-01T10:00:00+05:00, so the literal
+# comparison must return the same row as the column comparison.
+# Regression test for https://github.com/apache/datafusion/issues/25941
+statement ok
+CREATE TABLE issue_25941_t AS VALUES (TIMESTAMP '2024-01-01 10:00:00')
+
+statement ok
+CREATE TABLE issue_25941_l AS SELECT arrow_cast('2024-01-01T10:00:00+05:00', 'Timestamp(Nanosecond, Some("+05:00"))') AS v
+
+query PP
+SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("+05:00"))') AS lhs, v AS rhs FROM issue_25941_t, issue_25941_l
+----
+2024-01-01T10:00:00+05:00 2024-01-01T10:00:00+05:00
+
+# Compared against a column
+query I
+SELECT count(*) FROM issue_25941_t, issue_25941_l
+WHERE arrow_cast(issue_25941_t.column1, 'Timestamp(Nanosecond, Some("+05:00"))') = issue_25941_l.v
+----
+1
+
+# Compared against the same value as a literal
+query I
+SELECT count(*) FROM issue_25941_t
+WHERE arrow_cast(column1, 'Timestamp(Nanosecond, Some("+05:00"))')
+ = arrow_cast('2024-01-01T10:00:00+05:00', 'Timestamp(Nanosecond, Some("+05:00"))')
+----
+1
+
+# The same comparison through `::timestamptz` under a +05:00 session zone.
+# See https://github.com/apache/datafusion/issues/25941
+statement ok
+SET datafusion.execution.time_zone = '+05:00'
+
+statement ok
+CREATE TABLE issue_25941_t2 AS VALUES (TIMESTAMP '2024-01-01 10:00:00')
+
+query P
+SELECT column1::timestamptz FROM issue_25941_t2
+----
+2024-01-01T10:00:00+05:00
+
+query I
+SELECT count(*) FROM issue_25941_t2 WHERE column1::timestamptz = TIMESTAMPTZ '2024-01-01 10:00:00+05:00'
+----
+1
+
+statement ok
+RESET datafusion.execution.time_zone
+
+statement ok
+DROP TABLE issue_25941_t
+
+statement ok
+DROP TABLE issue_25941_l
+
+statement ok
+DROP TABLE issue_25941_t2
+
# DIVERGES FROM POSTGRESQL: the tz-naive literal is read in the column's zone
# (America/Denver), so it matches the July row. PostgreSQL reads a tz-naive
# value in the session zone instead.
diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
index b0efc10861..341b2784a5 100644
--- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
@@ -1086,3 +1086,40 @@ set datafusion.execution.parquet.pushdown_filters = false;
statement ok
DROP TABLE case_without_else;
+
+###
+# The physical simplifier must not unwrap a narrowing TRY_CAST either
+# (max_passes = 0 so only the physical rewrite runs).
+# See https://github.com/apache/datafusion/issues/24702
+###
+
+statement ok
+COPY (VALUES (5::bigint), (7::bigint), (9999999999::bigint), (10000000000::bigint))
+TO 'test_files/scratch/parquet_filter_pushdown/try_cast_narrowing.parquet'
+STORED AS PARQUET
+OPTIONS ('format.max_row_group_size' '2');
+
+statement ok
+CREATE EXTERNAL TABLE try_cast_narrowing_pq STORED AS PARQUET
+LOCATION 'test_files/scratch/parquet_filter_pushdown/try_cast_narrowing.parquet';
+
+statement ok
+set datafusion.execution.parquet.pushdown_filters = true;
+
+statement ok
+set datafusion.optimizer.max_passes = 0;
+
+query I rowsort
+SELECT column1 FROM try_cast_narrowing_pq WHERE try_cast(column1 AS INT) > 1;
+----
+5
+7
+
+statement ok
+RESET datafusion.optimizer.max_passes;
+
+statement ok
+set datafusion.execution.parquet.pushdown_filters = false;
+
+statement ok
+DROP TABLE try_cast_narrowing_pq;
diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt b/datafusion/sqllogictest/test_files/simplify_expr.slt
index b2a289fa7e..1d6f3847b0 100644
--- a/datafusion/sqllogictest/test_files/simplify_expr.slt
+++ b/datafusion/sqllogictest/test_files/simplify_expr.slt
@@ -380,6 +380,85 @@ physical_plan
statement ok
drop table decimal_prices;
+# Decimal scale narrowing rounds (many-to-one), so the CAST must stay on the
+# column. See https://github.com/apache/datafusion/issues/23095
+statement ok
+create table decimal_scale_narrowing(d decimal(20,4)) as values (1.2345), (1.2399), (1.2500);
+
+query RR
+select d, arrow_cast(d, 'Decimal128(20, 2)') as d2 from decimal_scale_narrowing order by d;
+----
+1.2345 1.23
+1.2399 1.24
+1.2500 1.25
+
+query R
+select d from decimal_scale_narrowing
+where arrow_cast(d, 'Decimal128(20, 2)') = arrow_cast(1.23, 'Decimal128(20, 2)');
+----
+1.2345
+
+query TT
+explain select d from decimal_scale_narrowing
+where arrow_cast(d, 'Decimal128(20, 2)') = arrow_cast(1.23, 'Decimal128(20, 2)');
+----
+logical_plan
+01)Filter: CAST(decimal_scale_narrowing.d AS Decimal128(20, 2)) = Decimal128(1.23,20,2)
+02)--TableScan: decimal_scale_narrowing projection=[d]
+physical_plan
+01)FilterExec: CAST(d@0 AS Decimal128(20, 2)) = 1.23
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# Ordered and IN comparisons (1.2399 rounds to 1.24)
+query R
+select d from decimal_scale_narrowing where cast(d as decimal(20,2)) < cast(1.24 as decimal(20,2));
+----
+1.2345
+
+query R
+select d from decimal_scale_narrowing
+where cast(d as decimal(20,2)) in (cast(1.23 as decimal(20,2)), cast(1.25 as decimal(20,2)))
+order by d;
+----
+1.2345
+1.2500
+
+statement ok
+drop table decimal_scale_narrowing;
+
+# A negative decimal scale must not panic in the cast rewrite.
+# See https://github.com/apache/datafusion/issues/24896
+query B
+select arrow_cast(1, 'Decimal128(10, -2)') = 100;
+----
+false
+
+query B
+select arrow_cast(100, 'Decimal128(10, -2)') = 100;
+----
+true
+
+statement ok
+create table decimal_negative_scale(v bigint) as values (1), (100), (200);
+
+query I
+select v from decimal_negative_scale where arrow_cast(v, 'Decimal128(10, -2)') = 100;
+----
+100
+
+query TT
+explain select v from decimal_negative_scale where arrow_cast(v, 'Decimal128(10, -2)') = 100;
+----
+logical_plan
+01)Filter: CAST(CAST(decimal_negative_scale.v AS Decimal128(10, -2)) AS Decimal128(20, 0)) = Decimal128(100,20,0)
+02)--TableScan: decimal_negative_scale projection=[v]
+physical_plan
+01)FilterExec: CAST(CAST(v@0 AS Decimal128(10, -2)) AS Decimal128(20, 0)) = 100
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+drop table decimal_negative_scale;
+
# Integer -> decimal scale-up is admitted by the same type-based rule: the
# rewrite compares the integer column directly, so a source value that Arrow's
# strict cast rejects (and TRY_CAST renders NULL) simply fails the predicate
@@ -434,6 +513,72 @@ physical_plan
statement ok
drop table int_prices;
+# A narrowing TRY_CAST yields NULL on overflow, so it must not be unwrapped.
+# (A TRY_CAST to an overflowing decimal still is, by design; see above.)
+# See https://github.com/apache/datafusion/issues/24702
+statement ok
+create table try_cast_narrowing as values (5::bigint), (9999999999::bigint);
+
+query IIB
+select column1, try_cast(column1 as int) as c, try_cast(column1 as int) > 1 as p
+from try_cast_narrowing order by column1;
+----
+5 5 true
+9999999999 NULL NULL
+
+query I
+select column1 from try_cast_narrowing where try_cast(column1 as int) > 1;
+----
+5
+
+query TT
+explain select column1 from try_cast_narrowing where try_cast(column1 as int) > 1;
+----
+logical_plan
+01)Filter: TRY_CAST(try_cast_narrowing.column1 AS Int32) > Int32(1)
+02)--TableScan: try_cast_narrowing projection=[column1]
+physical_plan
+01)FilterExec: TRY_CAST(column1@0 AS Int32) > 1
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+drop table try_cast_narrowing;
+
+# 4294967301 (2^32 + 5) overflows Int32, so IN and NOT IN are both NULL
+statement ok
+create table try_cast_narrowing_in as values (5::bigint), (4294967301::bigint);
+
+query IIBB
+select column1, try_cast(column1 as int) as c,
+ try_cast(column1 as int) in (5) as p_in,
+ try_cast(column1 as int) not in (5) as p_not_in
+from try_cast_narrowing_in order by column1;
+----
+5 5 true false
+4294967301 NULL NULL NULL
+
+query I
+select column1 from try_cast_narrowing_in where try_cast(column1 as int) in (5);
+----
+5
+
+query I
+select column1 from try_cast_narrowing_in where try_cast(column1 as int) not in (5);
+----
+
+query TT
+explain select column1 from try_cast_narrowing_in where try_cast(column1 as int) in (5);
+----
+logical_plan
+01)Filter: TRY_CAST(try_cast_narrowing_in.column1 AS Int32) = Int32(5)
+02)--TableScan: try_cast_narrowing_in projection=[column1]
+physical_plan
+01)FilterExec: TRY_CAST(column1@0 AS Int32) = 5
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+drop table try_cast_narrowing_in;
+
# ------------------------------------------------------------------------
# Unwrapping Date32 <-> Date64 casts in comparison predicates.
#