Apache DataFusion SQL Query Engine
test: Add regression tests for some `unwrap_cast` related bugs that don't reproduce (#27018)
Adam Gutglick17401bbac0b9
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.sltindex ce525005de..878e477ed3 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt@@ -25,6 +25,7 @@## https://github.com/apache/datafusion/issues/25166## https://github.com/apache/datafusion/issues/25167## https://github.com/apache/datafusion/issues/25170+## https://github.com/apache/datafusion/issues/25941#### Cross-engine notes below quote PostgreSQL with the session `TimeZone` named## in the comment. The subset the two engines agree on is machine checked@@ -488,14 +489,11 @@ RESET datafusion.execution.time_zone############ SECTION 8: comparison and equality across tz-aware and tz-naive##-## The pins here record three behaviours. A tz-naive side is read in the other+## The pins here record two behaviours. A tz-naive side is read in the other## operand's zone, where PostgreSQL uses the session zone:## https://github.com/apache/datafusion/issues/13212## `'...Z'::timestamptz` is tz-naive while the session zone is unset## (SECTION 0): https://github.com/apache/datafusion/issues/25166-## And under a non-UTC session zone, unwrap_cast rewrites a tz-naive column-## compared against a tz-aware literal into a different comparison:-## https://github.com/apache/datafusion/issues/25095##########statement ok@@ -622,6 +620,114 @@ set datafusion.explain.logical_plan_only = falsestatement okRESET datafusion.execution.time_zone+# tz-naive column cast to a tz-aware type and compared against a tz-aware+# literal under a non-UTC session zone. 2024-11-01 00:00 in Asia/Singapore+# (+08:00) is 2024-10-31 16:00 UTC, so the literal comparison must agree with the+# column-vs-column comparison. PostgreSQL and DuckDB under+# TimeZone='Asia/Singapore' return 1, 0, 1, 1.+# Regression test for https://github.com/apache/datafusion/issues/25095+statement ok+SET datafusion.execution.time_zone = 'Asia/Singapore'++statement ok+CREATE TABLE issue_25095_t AS SELECT TIMESTAMP '2024-11-01T00:00:00' AS ts++statement ok+CREATE TABLE issue_25095_u AS SELECT '2024-10-31T16:00:00Z'::timestamptz AS tstz++# Explicit cast, matching instant+query I+SELECT count(*) FROM issue_25095_t WHERE ts::timestamptz = '2024-10-31T16:00:00Z'::timestamptz+----+1++# Explicit cast, wrong instant+query I+SELECT count(*) FROM issue_25095_t WHERE ts::timestamptz = '2024-11-01T00:00:00Z'::timestamptz+----+0++# Implicit coercion instead of an explicit cast+query I+SELECT count(*) FROM issue_25095_t WHERE ts = '2024-10-31T16:00:00Z'::timestamptz+----+1++# Control: column vs column, nothing to unwrap+query I+SELECT count(*) FROM issue_25095_t, issue_25095_u WHERE issue_25095_t.ts::timestamptz = issue_25095_u.tstz+----+1++statement ok+RESET datafusion.execution.time_zone++statement ok+DROP TABLE issue_25095_t++statement ok+DROP TABLE issue_25095_u++# tz-naive column cast to a fixed offset zone and compared against a literal in+# that zone. Both sides display as 2024-01-01T10:00:00+05:00, so the literal+# comparison must return the same row as the column comparison.+# Regression test for https://github.com/apache/datafusion/issues/25941+statement ok+CREATE TABLE issue_25941_t AS VALUES (TIMESTAMP '2024-01-01 10:00:00')++statement ok+CREATE TABLE issue_25941_l AS SELECT arrow_cast('2024-01-01T10:00:00+05:00', 'Timestamp(Nanosecond, Some("+05:00"))') AS v++query PP+SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("+05:00"))') AS lhs, v AS rhs FROM issue_25941_t, issue_25941_l+----+2024-01-01T10:00:00+05:00 2024-01-01T10:00:00+05:00++# Compared against a column+query I+SELECT count(*) FROM issue_25941_t, issue_25941_l+WHERE arrow_cast(issue_25941_t.column1, 'Timestamp(Nanosecond, Some("+05:00"))') = issue_25941_l.v+----+1++# Compared against the same value as a literal+query I+SELECT count(*) FROM issue_25941_t+WHERE arrow_cast(column1, 'Timestamp(Nanosecond, Some("+05:00"))')+ = arrow_cast('2024-01-01T10:00:00+05:00', 'Timestamp(Nanosecond, Some("+05:00"))')+----+1++# The same comparison through `::timestamptz` under a +05:00 session zone.+# See https://github.com/apache/datafusion/issues/25941+statement ok+SET datafusion.execution.time_zone = '+05:00'++statement ok+CREATE TABLE issue_25941_t2 AS VALUES (TIMESTAMP '2024-01-01 10:00:00')++query P+SELECT column1::timestamptz FROM issue_25941_t2+----+2024-01-01T10:00:00+05:00++query I+SELECT count(*) FROM issue_25941_t2 WHERE column1::timestamptz = TIMESTAMPTZ '2024-01-01 10:00:00+05:00'+----+1++statement ok+RESET datafusion.execution.time_zone++statement ok+DROP TABLE issue_25941_t++statement ok+DROP TABLE issue_25941_l++statement ok+DROP TABLE issue_25941_t2+# DIVERGES FROM POSTGRESQL: the tz-naive literal is read in the column's zone# (America/Denver), so it matches the July row. PostgreSQL reads a tz-naive# value in the session zone instead.diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.sltindex b0efc10861..341b2784a5 100644--- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt+++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt@@ -1086,3 +1086,40 @@ set datafusion.execution.parquet.pushdown_filters = false;statement okDROP TABLE case_without_else;++###+# The physical simplifier must not unwrap a narrowing TRY_CAST either+# (max_passes = 0 so only the physical rewrite runs).+# See https://github.com/apache/datafusion/issues/24702+###++statement ok+COPY (VALUES (5::bigint), (7::bigint), (9999999999::bigint), (10000000000::bigint))+TO 'test_files/scratch/parquet_filter_pushdown/try_cast_narrowing.parquet'+STORED AS PARQUET+OPTIONS ('format.max_row_group_size' '2');++statement ok+CREATE EXTERNAL TABLE try_cast_narrowing_pq STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_filter_pushdown/try_cast_narrowing.parquet';++statement ok+set datafusion.execution.parquet.pushdown_filters = true;++statement ok+set datafusion.optimizer.max_passes = 0;++query I rowsort+SELECT column1 FROM try_cast_narrowing_pq WHERE try_cast(column1 AS INT) > 1;+----+5+7++statement ok+RESET datafusion.optimizer.max_passes;++statement ok+set datafusion.execution.parquet.pushdown_filters = false;++statement ok+DROP TABLE try_cast_narrowing_pq;diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt b/datafusion/sqllogictest/test_files/simplify_expr.sltindex b2a289fa7e..1d6f3847b0 100644--- a/datafusion/sqllogictest/test_files/simplify_expr.slt+++ b/datafusion/sqllogictest/test_files/simplify_expr.slt@@ -380,6 +380,85 @@ physical_planstatement okdrop table decimal_prices;+# Decimal scale narrowing rounds (many-to-one), so the CAST must stay on the+# column. See https://github.com/apache/datafusion/issues/23095+statement ok+create table decimal_scale_narrowing(d decimal(20,4)) as values (1.2345), (1.2399), (1.2500);++query RR+select d, arrow_cast(d, 'Decimal128(20, 2)') as d2 from decimal_scale_narrowing order by d;+----+1.2345 1.23+1.2399 1.24+1.2500 1.25++query R+select d from decimal_scale_narrowing+where arrow_cast(d, 'Decimal128(20, 2)') = arrow_cast(1.23, 'Decimal128(20, 2)');+----+1.2345++query TT+explain select d from decimal_scale_narrowing+where arrow_cast(d, 'Decimal128(20, 2)') = arrow_cast(1.23, 'Decimal128(20, 2)');+----+logical_plan+01)Filter: CAST(decimal_scale_narrowing.d AS Decimal128(20, 2)) = Decimal128(1.23,20,2)+02)--TableScan: decimal_scale_narrowing projection=[d]+physical_plan+01)FilterExec: CAST(d@0 AS Decimal128(20, 2)) = 1.23+02)--DataSourceExec: partitions=1, partition_sizes=[1]++# Ordered and IN comparisons (1.2399 rounds to 1.24)+query R+select d from decimal_scale_narrowing where cast(d as decimal(20,2)) < cast(1.24 as decimal(20,2));+----+1.2345++query R+select d from decimal_scale_narrowing+where cast(d as decimal(20,2)) in (cast(1.23 as decimal(20,2)), cast(1.25 as decimal(20,2)))+order by d;+----+1.2345+1.2500++statement ok+drop table decimal_scale_narrowing;++# A negative decimal scale must not panic in the cast rewrite.+# See https://github.com/apache/datafusion/issues/24896+query B+select arrow_cast(1, 'Decimal128(10, -2)') = 100;+----+false++query B+select arrow_cast(100, 'Decimal128(10, -2)') = 100;+----+true++statement ok+create table decimal_negative_scale(v bigint) as values (1), (100), (200);++query I+select v from decimal_negative_scale where arrow_cast(v, 'Decimal128(10, -2)') = 100;+----+100++query TT+explain select v from decimal_negative_scale where arrow_cast(v, 'Decimal128(10, -2)') = 100;+----+logical_plan+01)Filter: CAST(CAST(decimal_negative_scale.v AS Decimal128(10, -2)) AS Decimal128(20, 0)) = Decimal128(100,20,0)+02)--TableScan: decimal_negative_scale projection=[v]+physical_plan+01)FilterExec: CAST(CAST(v@0 AS Decimal128(10, -2)) AS Decimal128(20, 0)) = 100+02)--DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+drop table decimal_negative_scale;+# Integer -> decimal scale-up is admitted by the same type-based rule: the# rewrite compares the integer column directly, so a source value that Arrow's# strict cast rejects (and TRY_CAST renders NULL) simply fails the predicate@@ -434,6 +513,72 @@ physical_planstatement okdrop table int_prices;+# A narrowing TRY_CAST yields NULL on overflow, so it must not be unwrapped.+# (A TRY_CAST to an overflowing decimal still is, by design; see above.)+# See https://github.com/apache/datafusion/issues/24702+statement ok+create table try_cast_narrowing as values (5::bigint), (9999999999::bigint);++query IIB+select column1, try_cast(column1 as int) as c, try_cast(column1 as int) > 1 as p+from try_cast_narrowing order by column1;+----+5 5 true+9999999999 NULL NULL++query I+select column1 from try_cast_narrowing where try_cast(column1 as int) > 1;+----+5++query TT+explain select column1 from try_cast_narrowing where try_cast(column1 as int) > 1;+----+logical_plan+01)Filter: TRY_CAST(try_cast_narrowing.column1 AS Int32) > Int32(1)+02)--TableScan: try_cast_narrowing projection=[column1]+physical_plan+01)FilterExec: TRY_CAST(column1@0 AS Int32) > 1+02)--DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+drop table try_cast_narrowing;++# 4294967301 (2^32 + 5) overflows Int32, so IN and NOT IN are both NULL+statement ok+create table try_cast_narrowing_in as values (5::bigint), (4294967301::bigint);++query IIBB+select column1, try_cast(column1 as int) as c,+ try_cast(column1 as int) in (5) as p_in,+ try_cast(column1 as int) not in (5) as p_not_in+from try_cast_narrowing_in order by column1;+----+5 5 true false+4294967301 NULL NULL NULL++query I+select column1 from try_cast_narrowing_in where try_cast(column1 as int) in (5);+----+5++query I+select column1 from try_cast_narrowing_in where try_cast(column1 as int) not in (5);+----++query TT+explain select column1 from try_cast_narrowing_in where try_cast(column1 as int) in (5);+----+logical_plan+01)Filter: TRY_CAST(try_cast_narrowing_in.column1 AS Int32) = Int32(5)+02)--TableScan: try_cast_narrowing_in projection=[column1]+physical_plan+01)FilterExec: TRY_CAST(column1@0 AS Int32) = 5+02)--DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+drop table try_cast_narrowing_in;+# ------------------------------------------------------------------------# Unwrapping Date32 <-> Date64 casts in comparison predicates.#