Apache DataFusion SQL Query Engine
fix: preserve EXTRACT year comparisons with unrepresentable bounds (#25140)
Stefan Wang6079a1e6ef81
diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rsindex 4bfadf7999..c7f7ef88c0 100644--- a/datafusion/functions/src/datetime/date_part.rs+++ b/datafusion/functions/src/datetime/date_part.rs@@ -326,30 +326,37 @@ fn date_to_scalar(date: NaiveDate, target_type: &DataType) -> Option<ScalarValuelet naive_midnight = date.and_hms_opt(0, 0, 0)?;let tz: Option<Tz> = tz_opt.clone().and_then(|s| s.parse().ok());+ // Midnight has no representation when the date falls outside the+ // unit's range, or when the zone skips it. Reporting no bound keeps+ // the caller on the original predicate; a NULL bound here would make+ // the rewritten comparison NULL on every row.match unit {Second => ScalarValue::TimestampSecond(- TimestampSecondType::from_naive_datetime(naive_midnight, tz.as_ref()),+ Some(TimestampSecondType::from_naive_datetime(+ naive_midnight,+ tz.as_ref(),+ )?),tz_opt.clone(),),Millisecond => ScalarValue::TimestampMillisecond(- TimestampMillisecondType::from_naive_datetime(+ Some(TimestampMillisecondType::from_naive_datetime(naive_midnight,tz.as_ref(),- ),+ )?),tz_opt.clone(),),Microsecond => ScalarValue::TimestampMicrosecond(- TimestampMicrosecondType::from_naive_datetime(+ Some(TimestampMicrosecondType::from_naive_datetime(naive_midnight,tz.as_ref(),- ),+ )?),tz_opt.clone(),),Nanosecond => ScalarValue::TimestampNanosecond(- TimestampNanosecondType::from_naive_datetime(+ Some(TimestampNanosecondType::from_naive_datetime(naive_midnight,tz.as_ref(),- ),+ )?),tz_opt.clone(),),}diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.sltindex 5bbd93f8ad..18a81bd94f 100644--- a/datafusion/sqllogictest/test_files/datetime/date_part.slt+++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt@@ -1859,3 +1859,81 @@ logical_planphysical_plan01)FilterExec: c@0 >= 2024-01-01 AND c@0 < 2025-01-0102)--DataSourceExec: partitions=1, partition_sizes=[1]++# A bound whose January 1 falls outside the timestamp unit's range must leave the+# predicate alone rather than rewriting it against a NULL bound.++statement ok+CREATE VIEW year_bound AS SELECT * FROM (VALUES+ ('2020-06-01T12:00:00'::timestamp),+ ('1995-06-01T12:00:00'::timestamp)) s(ts);++query I+SELECT count(*) FROM year_bound WHERE EXTRACT(YEAR FROM ts) < 9999;+----+2++query I+SELECT count(*) FROM year_bound WHERE EXTRACT(YEAR FROM ts) <= 2262;+----+2++query I+SELECT count(*) FROM year_bound WHERE EXTRACT(YEAR FROM ts) >= 1000 OR EXTRACT(YEAR FROM ts) = 2020;+----+2++query IB rowsort+SELECT EXTRACT(YEAR FROM ts), EXTRACT(YEAR FROM ts) < 9999 FROM year_bound;+----+1995 true+2020 true++# A bound inside the nanosecond range still rewrites as before.++query I+SELECT count(*) FROM year_bound WHERE EXTRACT(YEAR FROM ts) >= 1678;+----+2++statement ok+DROP VIEW year_bound;++# Lima skipped midnight on 1994-01-01.+statement ok+CREATE TABLE year_bound_lima AS+SELECT+ arrow_cast(ts, 'Timestamp(Second, Some("America/Lima"))') AS ts_sec,+ arrow_cast(ts, 'Timestamp(Millisecond, Some("America/Lima"))') AS ts_milli,+ arrow_cast(ts, 'Timestamp(Microsecond, Some("America/Lima"))') AS ts_micro,+ arrow_cast(ts, 'Timestamp(Nanosecond, Some("America/Lima"))') AS ts_nano+FROM (VALUES+ (NULL),+ ('1994-01-01T04:59:59Z'),+ ('1994-01-01T05:00:00Z')) AS input(ts);++query PBBBB+SELECT ts_nano,+ EXTRACT(YEAR FROM ts_sec) = 1994,+ EXTRACT(YEAR FROM ts_milli) = 1994,+ EXTRACT(YEAR FROM ts_micro) = 1994,+ EXTRACT(YEAR FROM ts_nano) = 1994+FROM year_bound_lima+ORDER BY ts_nano NULLS FIRST;+----+NULL NULL NULL NULL NULL+1993-12-31T23:59:59-05:00 false false false false+1994-01-01T01:00:00-04:00 true true true true++query P+SELECT ts_nano FROM year_bound_lima WHERE EXTRACT(YEAR FROM ts_nano) = 1994;+----+1994-01-01T01:00:00-04:00++query I+SELECT count(*) FROM year_bound_lima WHERE EXTRACT(YEAR FROM ts_nano) = 1993;+----+1++statement ok+DROP TABLE year_bound_lima;