Skip to content

Apache DataFusion SQL Query Engine

fix: accept all timestamp precisions in `generate_series`/`range` (#25173)

Adrian Garcia Badaraccod967e68458b7
diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rs
index cf5e026581..94e69e17c3 100644
--- a/datafusion/functions-table/src/generate_series.rs
+++ b/datafusion/functions-table/src/generate_series.rs
@@ -534,6 +534,54 @@ fn reach_end_int64(val: i64, end: i64, step: i64, include_end: bool) -> bool {
}
}
+/// `i64::MIN` nanoseconds since the Unix epoch, rendered as a timestamp.
+const NANOS_RANGE_MIN: &str = "1677-09-21T00:12:43.145224192";
+/// `i64::MAX` nanoseconds since the Unix epoch, rendered as a timestamp.
+const NANOS_RANGE_MAX: &str = "2262-04-11T23:47:16.854775807";
+
+/// Reads a timestamp literal of any [`TimeUnit`] as nanoseconds since the
+/// epoch, plus its timezone. A NULL yields `None`. Errors if the value does not
+/// fit in an `i64` of nanoseconds.
+fn timestamp_arg_to_nanos(
+ expr: &Expr,
+ arg_desc: &str,
+ name: &str,
+) -> Result<(Option<i64>, Option<Arc<str>>)> {
+ let Expr::Literal(scalar, _) = expr else {
+ return plan_err!(
+ "{arg_desc} for {name} must be a TIMESTAMP or NULL constant, got {expr}"
+ );
+ };
+
+ let (value, nanos_per_unit, tz) = match scalar {
+ ScalarValue::Null => return Ok((None, None)),
+ ScalarValue::TimestampSecond(v, tz) => (v, 1_000_000_000i64, tz),
+ ScalarValue::TimestampMillisecond(v, tz) => (v, 1_000_000, tz),
+ ScalarValue::TimestampMicrosecond(v, tz) => (v, 1_000, tz),
+ ScalarValue::TimestampNanosecond(v, tz) => (v, 1, tz),
+ other => {
+ return plan_err!(
+ "{arg_desc} for {name} must be a TIMESTAMP or NULL constant, got {:?}",
+ other.data_type()
+ );
+ }
+ };
+
+ let Some(value) = value else {
+ return Ok((None, tz.clone()));
+ };
+
+ let nanos = value.checked_mul(nanos_per_unit).ok_or_else(|| {
+ plan_datafusion_err!(
+ "{arg_desc} for {name} is out of range of nanosecond timestamps: \
+ {value} ({:?}) is outside {NANOS_RANGE_MIN} to {NANOS_RANGE_MAX}",
+ scalar.data_type()
+ )
+ })?;
+
+ Ok((Some(nanos), tz.clone()))
+}
+
fn validate_interval_step(step: IntervalMonthDayNano) -> Result<()> {
if step.months == 0 && step.days == 0 && step.nanoseconds == 0 {
return plan_err!("Step interval cannot be zero");
@@ -673,38 +721,29 @@ impl GenerateSeriesFuncImpl {
);
}
- // Parse start timestamp
- let (start_ts, tz) = match &exprs[0] {
- Expr::Literal(ScalarValue::TimestampNanosecond(ts, tz), _) => {
- (*ts, tz.clone())
- }
- other => {
- return plan_err!(
- "First argument must be a timestamp or NULL, got {:?}",
- other
- );
- }
- };
+ let (start_ts, tz) =
+ timestamp_arg_to_nanos(&exprs[0], "First argument", self.name)?;
+ let (end_ts, _end_tz) =
+ timestamp_arg_to_nanos(&exprs[1], "Second argument", self.name)?;
- // Parse end timestamp
- let end_ts = match &exprs[1] {
- Expr::Literal(ScalarValue::Null, _) => None,
- Expr::Literal(ScalarValue::TimestampNanosecond(ts, _), _) => *ts,
- other => {
- return plan_err!(
- "Second argument must be a timestamp or NULL, got {:?}",
- other
- );
- }
- };
+ // The output timezone (which also drives month/day step arithmetic)
+ // comes from the start. The end's zone does not change its instant.
// Parse step interval
let step_interval = match &exprs[2] {
Expr::Literal(ScalarValue::Null, _) => None,
Expr::Literal(ScalarValue::IntervalMonthDayNano(interval), _) => *interval,
+ Expr::Literal(scalar, _) => {
+ return plan_err!(
+ "Third argument for {} must be an INTERVAL or NULL constant, got {:?}",
+ self.name,
+ scalar.data_type()
+ );
+ }
other => {
return plan_err!(
- "Third argument must be an interval or NULL, got {:?}",
+ "Third argument for {} must be an INTERVAL or NULL constant, got {}",
+ self.name,
other
);
}
@@ -765,9 +804,17 @@ impl GenerateSeriesFuncImpl {
args: GenSeriesArgs::ContainsNull { name: self.name },
}));
}
+ Expr::Literal(scalar, _) => {
+ return plan_err!(
+ "First argument for {} must be a DATE or NULL constant, got {:?}",
+ self.name,
+ scalar.data_type()
+ );
+ }
other => {
return plan_err!(
- "First argument must be a date or NULL, got {:?}",
+ "First argument for {} must be a DATE or NULL constant, got {}",
+ self.name,
other
);
}
@@ -783,9 +830,17 @@ impl GenerateSeriesFuncImpl {
args: GenSeriesArgs::ContainsNull { name: self.name },
}));
}
+ Expr::Literal(scalar, _) => {
+ return plan_err!(
+ "Second argument for {} must be a DATE or NULL constant, got {:?}",
+ self.name,
+ scalar.data_type()
+ );
+ }
other => {
return plan_err!(
- "Second argument must be a date or NULL, got {:?}",
+ "Second argument for {} must be a DATE or NULL constant, got {}",
+ self.name,
other
);
}
@@ -803,9 +858,17 @@ impl GenerateSeriesFuncImpl {
args: GenSeriesArgs::ContainsNull { name: self.name },
}));
}
+ Expr::Literal(scalar, _) => {
+ return plan_err!(
+ "Third argument for {} must be an INTERVAL or NULL constant, got {:?}",
+ self.name,
+ scalar.data_type()
+ );
+ }
other => {
return plan_err!(
- "Third argument must be an interval or NULL, got {:?}",
+ "Third argument for {} must be an INTERVAL or NULL constant, got {}",
+ self.name,
other
);
}
diff --git a/datafusion/sqllogictest/test_files/table_functions.slt b/datafusion/sqllogictest/test_files/table_functions.slt
index e67d898d71..3b9164df3d 100644
--- a/datafusion/sqllogictest/test_files/table_functions.slt
+++ b/datafusion/sqllogictest/test_files/table_functions.slt
@@ -259,6 +259,184 @@ SELECT * FROM generate_series(DATE '2262-04-10', DATE '2262-04-11', INTERVAL '2'
----
2262-04-10T00:00:00
+#
+# Timestamp bounds of every precision
+# https://github.com/apache/datafusion/issues/25169
+#
+
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Second, None)'), INTERVAL '1' DAY)
+----
+2024-01-01T00:00:00
+2024-01-02T00:00:00
+2024-01-03T00:00:00
+
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Millisecond, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)
+----
+2024-01-01T00:00:00
+2024-01-02T00:00:00
+2024-01-03T00:00:00
+
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Microsecond, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)
+----
+2024-01-01T00:00:00
+2024-01-02T00:00:00
+2024-01-03T00:00:00
+
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Nanosecond, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Nanosecond, None)'), INTERVAL '1' DAY)
+----
+2024-01-01T00:00:00
+2024-01-02T00:00:00
+2024-01-03T00:00:00
+
+query P
+SELECT * FROM range(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Second, None)'), INTERVAL '1' DAY)
+----
+2024-01-01T00:00:00
+2024-01-02T00:00:00
+
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-01T00:00:01', 'Timestamp(Second, None)'), INTERVAL '250' MILLISECOND)
+----
+2024-01-01T00:00:00
+2024-01-01T00:00:00.250
+2024-01-01T00:00:00.500
+2024-01-01T00:00:00.750
+2024-01-01T00:00:01
+
+# Mixed precisions
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)
+----
+2024-01-01T00:00:00
+2024-01-02T00:00:00
+2024-01-03T00:00:00
+
+query P
+SELECT * FROM range(arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Millisecond, None)'), arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), INTERVAL '-1' DAY)
+----
+2024-01-03T00:00:00
+2024-01-02T00:00:00
+
+# Output is always nanoseconds
+query T
+SELECT DISTINCT arrow_typeof(value) FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-02T00:00:00', 'Timestamp(Second, None)'), INTERVAL '1' DAY)
+----
+Timestamp(ns)
+
+query T
+SELECT DISTINCT arrow_typeof(value) FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), arrow_cast(TIMESTAMP '2024-01-02T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), INTERVAL '1' DAY)
+----
+Timestamp(ns, "+05:00")
+
+# Mixed timezones: the output uses the start's timezone
+query T
+SELECT DISTINCT arrow_typeof(value) FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00Z', 'Timestamp(Microsecond, Some("America/New_York"))'), INTERVAL '1' DAY)
+----
+Timestamp(ns, "+05:00")
+
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00Z', 'Timestamp(Microsecond, Some("America/New_York"))'), INTERVAL '1' DAY)
+----
+2024-01-01T00:00:00+05:00
+2024-01-02T00:00:00+05:00
+2024-01-03T00:00:00+05:00
+
+# Bounds compare as instants: 2024-01-01T05:00Z to 2024-01-02T19:00Z
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00Z', 'Timestamp(Microsecond, Some("America/New_York"))'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), INTERVAL '1' DAY)
+----
+2024-01-01T00:00:00-05:00
+2024-01-02T00:00:00-05:00
+
+# NULL bounds
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(NULL, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)
+----
+
+query P
+SELECT * FROM range(arrow_cast(NULL, 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)
+----
+
+query P
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), NULL, INTERVAL '1' DAY)
+----
+
+# Bounds outside the nanosecond range
+statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps: 9223372036854775807 \(Timestamp\(Second, None\)\) is outside 1677\-09\-21T00:12:43\.145224192 to 2262\-04\-11T23:47:16\.854775807
+SELECT * FROM generate_series(arrow_cast(9223372036854775807, 'Timestamp(Second, None)'), arrow_cast(0, 'Timestamp(Second, None)'), INTERVAL '-1' DAY)
+
+statement error DataFusion error: Error during planning: Second argument for generate_series is out of range of nanosecond timestamps: \-9223372036854775808 \(Timestamp\(Millisecond, None\)\) is outside 1677\-09\-21T00:12:43\.145224192 to 2262\-04\-11T23:47:16\.854775807
+SELECT * FROM generate_series(arrow_cast(0, 'Timestamp(Millisecond, None)'), arrow_cast(-9223372036854775808, 'Timestamp(Millisecond, None)'), INTERVAL '-1' DAY)
+
+statement error DataFusion error: Error during planning: First argument for range is out of range of nanosecond timestamps: 9223372036854775807 \(Timestamp\(Microsecond, None\)\) is outside
+SELECT * FROM range(arrow_cast(9223372036854775807, 'Timestamp(Microsecond, None)'), arrow_cast(0, 'Timestamp(Microsecond, None)'), INTERVAL '-1' DAY)
+
+# Edge of the nanosecond range for each unit
+query P
+SELECT * FROM generate_series(arrow_cast(9223372036, 'Timestamp(Second, None)'), arrow_cast(9223372036, 'Timestamp(Second, None)'), INTERVAL '1' DAY)
+----
+2262-04-11T23:47:16
+
+query P
+SELECT * FROM generate_series(arrow_cast(-9223372036854775, 'Timestamp(Microsecond, None)'), arrow_cast(-9223372036854775, 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)
+----
+1677-09-21T00:12:43.145225
+
+query P
+SELECT * FROM generate_series(arrow_cast(-9223372036, 'Timestamp(Second, None)'), arrow_cast(-9223372036, 'Timestamp(Second, None)'), INTERVAL '1' DAY)
+----
+1677-09-21T00:12:44
+
+query P
+SELECT * FROM generate_series(arrow_cast(9223372036854, 'Timestamp(Millisecond, None)'), arrow_cast(9223372036854, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)
+----
+2262-04-11T23:47:16.854
+
+query P
+SELECT * FROM generate_series(arrow_cast(-9223372036854, 'Timestamp(Millisecond, None)'), arrow_cast(-9223372036854, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)
+----
+1677-09-21T00:12:43.146
+
+query P
+SELECT * FROM generate_series(arrow_cast(9223372036854775, 'Timestamp(Microsecond, None)'), arrow_cast(9223372036854775, 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)
+----
+2262-04-11T23:47:16.854775
+
+statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps
+SELECT * FROM generate_series(arrow_cast(9223372037, 'Timestamp(Second, None)'), arrow_cast(9223372037, 'Timestamp(Second, None)'), INTERVAL '1' DAY)
+
+statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps
+SELECT * FROM generate_series(arrow_cast(-9223372036854776, 'Timestamp(Microsecond, None)'), arrow_cast(-9223372036854776, 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)
+
+statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps
+SELECT * FROM generate_series(arrow_cast(-9223372037, 'Timestamp(Second, None)'), arrow_cast(-9223372037, 'Timestamp(Second, None)'), INTERVAL '1' DAY)
+
+statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps
+SELECT * FROM generate_series(arrow_cast(9223372036855, 'Timestamp(Millisecond, None)'), arrow_cast(9223372036855, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)
+
+statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps
+SELECT * FROM generate_series(arrow_cast(-9223372036855, 'Timestamp(Millisecond, None)'), arrow_cast(-9223372036855, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)
+
+statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps
+SELECT * FROM generate_series(arrow_cast(9223372036854776, 'Timestamp(Microsecond, None)'), arrow_cast(9223372036854776, 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)
+
+# Invalid argument types
+statement error DataFusion error: Error during planning: Second argument for generate_series must be a TIMESTAMP or NULL constant, got Int64
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), 5, INTERVAL '1' DAY)
+
+statement error DataFusion error: Error during planning: Third argument for generate_series must be an INTERVAL or NULL constant, got Utf8
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Second, None)'), 'not an interval')
+
+statement error DataFusion error: Error during planning: Second argument for generate_series must be a DATE or NULL constant, got Int64
+SELECT * FROM generate_series(DATE '2024-01-01', 5, INTERVAL '1' DAY)
+
+statement error DataFusion error: Error during planning: Second argument for generate_series must be a TIMESTAMP or NULL constant, got \(<subquery>\)
+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), (SELECT TIMESTAMP '2024-01-03T00:00:00'), INTERVAL '1' DAY)
+
# UDF and UDTF `generate_series` can be used simultaneously
query ? rowsort
SELECT generate_series(1, t1.end) FROM generate_series(3, 5) as t1(end)