Apache DataFusion SQL Query Engine
fix: accept all timestamp precisions in `generate_series`/`range` (#25173)
Adrian Garcia Badaraccod967e68458b7
diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rsindex cf5e026581..94e69e17c3 100644--- a/datafusion/functions-table/src/generate_series.rs+++ b/datafusion/functions-table/src/generate_series.rs@@ -534,6 +534,54 @@ fn reach_end_int64(val: i64, end: i64, step: i64, include_end: bool) -> bool {}}+/// `i64::MIN` nanoseconds since the Unix epoch, rendered as a timestamp.+const NANOS_RANGE_MIN: &str = "1677-09-21T00:12:43.145224192";+/// `i64::MAX` nanoseconds since the Unix epoch, rendered as a timestamp.+const NANOS_RANGE_MAX: &str = "2262-04-11T23:47:16.854775807";++/// Reads a timestamp literal of any [`TimeUnit`] as nanoseconds since the+/// epoch, plus its timezone. A NULL yields `None`. Errors if the value does not+/// fit in an `i64` of nanoseconds.+fn timestamp_arg_to_nanos(+ expr: &Expr,+ arg_desc: &str,+ name: &str,+) -> Result<(Option<i64>, Option<Arc<str>>)> {+ let Expr::Literal(scalar, _) = expr else {+ return plan_err!(+ "{arg_desc} for {name} must be a TIMESTAMP or NULL constant, got {expr}"+ );+ };++ let (value, nanos_per_unit, tz) = match scalar {+ ScalarValue::Null => return Ok((None, None)),+ ScalarValue::TimestampSecond(v, tz) => (v, 1_000_000_000i64, tz),+ ScalarValue::TimestampMillisecond(v, tz) => (v, 1_000_000, tz),+ ScalarValue::TimestampMicrosecond(v, tz) => (v, 1_000, tz),+ ScalarValue::TimestampNanosecond(v, tz) => (v, 1, tz),+ other => {+ return plan_err!(+ "{arg_desc} for {name} must be a TIMESTAMP or NULL constant, got {:?}",+ other.data_type()+ );+ }+ };++ let Some(value) = value else {+ return Ok((None, tz.clone()));+ };++ let nanos = value.checked_mul(nanos_per_unit).ok_or_else(|| {+ plan_datafusion_err!(+ "{arg_desc} for {name} is out of range of nanosecond timestamps: \+ {value} ({:?}) is outside {NANOS_RANGE_MIN} to {NANOS_RANGE_MAX}",+ scalar.data_type()+ )+ })?;++ Ok((Some(nanos), tz.clone()))+}+fn validate_interval_step(step: IntervalMonthDayNano) -> Result<()> {if step.months == 0 && step.days == 0 && step.nanoseconds == 0 {return plan_err!("Step interval cannot be zero");@@ -673,38 +721,29 @@ impl GenerateSeriesFuncImpl {);}- // Parse start timestamp- let (start_ts, tz) = match &exprs[0] {- Expr::Literal(ScalarValue::TimestampNanosecond(ts, tz), _) => {- (*ts, tz.clone())- }- other => {- return plan_err!(- "First argument must be a timestamp or NULL, got {:?}",- other- );- }- };+ let (start_ts, tz) =+ timestamp_arg_to_nanos(&exprs[0], "First argument", self.name)?;+ let (end_ts, _end_tz) =+ timestamp_arg_to_nanos(&exprs[1], "Second argument", self.name)?;- // Parse end timestamp- let end_ts = match &exprs[1] {- Expr::Literal(ScalarValue::Null, _) => None,- Expr::Literal(ScalarValue::TimestampNanosecond(ts, _), _) => *ts,- other => {- return plan_err!(- "Second argument must be a timestamp or NULL, got {:?}",- other- );- }- };+ // The output timezone (which also drives month/day step arithmetic)+ // comes from the start. The end's zone does not change its instant.// Parse step intervallet step_interval = match &exprs[2] {Expr::Literal(ScalarValue::Null, _) => None,Expr::Literal(ScalarValue::IntervalMonthDayNano(interval), _) => *interval,+ Expr::Literal(scalar, _) => {+ return plan_err!(+ "Third argument for {} must be an INTERVAL or NULL constant, got {:?}",+ self.name,+ scalar.data_type()+ );+ }other => {return plan_err!(- "Third argument must be an interval or NULL, got {:?}",+ "Third argument for {} must be an INTERVAL or NULL constant, got {}",+ self.name,other);}@@ -765,9 +804,17 @@ impl GenerateSeriesFuncImpl {args: GenSeriesArgs::ContainsNull { name: self.name },}));}+ Expr::Literal(scalar, _) => {+ return plan_err!(+ "First argument for {} must be a DATE or NULL constant, got {:?}",+ self.name,+ scalar.data_type()+ );+ }other => {return plan_err!(- "First argument must be a date or NULL, got {:?}",+ "First argument for {} must be a DATE or NULL constant, got {}",+ self.name,other);}@@ -783,9 +830,17 @@ impl GenerateSeriesFuncImpl {args: GenSeriesArgs::ContainsNull { name: self.name },}));}+ Expr::Literal(scalar, _) => {+ return plan_err!(+ "Second argument for {} must be a DATE or NULL constant, got {:?}",+ self.name,+ scalar.data_type()+ );+ }other => {return plan_err!(- "Second argument must be a date or NULL, got {:?}",+ "Second argument for {} must be a DATE or NULL constant, got {}",+ self.name,other);}@@ -803,9 +858,17 @@ impl GenerateSeriesFuncImpl {args: GenSeriesArgs::ContainsNull { name: self.name },}));}+ Expr::Literal(scalar, _) => {+ return plan_err!(+ "Third argument for {} must be an INTERVAL or NULL constant, got {:?}",+ self.name,+ scalar.data_type()+ );+ }other => {return plan_err!(- "Third argument must be an interval or NULL, got {:?}",+ "Third argument for {} must be an INTERVAL or NULL constant, got {}",+ self.name,other);}diff --git a/datafusion/sqllogictest/test_files/table_functions.slt b/datafusion/sqllogictest/test_files/table_functions.sltindex e67d898d71..3b9164df3d 100644--- a/datafusion/sqllogictest/test_files/table_functions.slt+++ b/datafusion/sqllogictest/test_files/table_functions.slt@@ -259,6 +259,184 @@ SELECT * FROM generate_series(DATE '2262-04-10', DATE '2262-04-11', INTERVAL '2'----2262-04-10T00:00:00+#+# Timestamp bounds of every precision+# https://github.com/apache/datafusion/issues/25169+#++query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Second, None)'), INTERVAL '1' DAY)+----+2024-01-01T00:00:00+2024-01-02T00:00:00+2024-01-03T00:00:00++query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Millisecond, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)+----+2024-01-01T00:00:00+2024-01-02T00:00:00+2024-01-03T00:00:00++query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Microsecond, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)+----+2024-01-01T00:00:00+2024-01-02T00:00:00+2024-01-03T00:00:00++query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Nanosecond, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Nanosecond, None)'), INTERVAL '1' DAY)+----+2024-01-01T00:00:00+2024-01-02T00:00:00+2024-01-03T00:00:00++query P+SELECT * FROM range(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Second, None)'), INTERVAL '1' DAY)+----+2024-01-01T00:00:00+2024-01-02T00:00:00++query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-01T00:00:01', 'Timestamp(Second, None)'), INTERVAL '250' MILLISECOND)+----+2024-01-01T00:00:00+2024-01-01T00:00:00.250+2024-01-01T00:00:00.500+2024-01-01T00:00:00.750+2024-01-01T00:00:01++# Mixed precisions+query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)+----+2024-01-01T00:00:00+2024-01-02T00:00:00+2024-01-03T00:00:00++query P+SELECT * FROM range(arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Millisecond, None)'), arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), INTERVAL '-1' DAY)+----+2024-01-03T00:00:00+2024-01-02T00:00:00++# Output is always nanoseconds+query T+SELECT DISTINCT arrow_typeof(value) FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-02T00:00:00', 'Timestamp(Second, None)'), INTERVAL '1' DAY)+----+Timestamp(ns)++query T+SELECT DISTINCT arrow_typeof(value) FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), arrow_cast(TIMESTAMP '2024-01-02T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), INTERVAL '1' DAY)+----+Timestamp(ns, "+05:00")++# Mixed timezones: the output uses the start's timezone+query T+SELECT DISTINCT arrow_typeof(value) FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00Z', 'Timestamp(Microsecond, Some("America/New_York"))'), INTERVAL '1' DAY)+----+Timestamp(ns, "+05:00")++query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00Z', 'Timestamp(Microsecond, Some("America/New_York"))'), INTERVAL '1' DAY)+----+2024-01-01T00:00:00+05:00+2024-01-02T00:00:00+05:00+2024-01-03T00:00:00+05:00++# Bounds compare as instants: 2024-01-01T05:00Z to 2024-01-02T19:00Z+query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00Z', 'Timestamp(Microsecond, Some("America/New_York"))'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00Z', 'Timestamp(Second, Some("+05:00"))'), INTERVAL '1' DAY)+----+2024-01-01T00:00:00-05:00+2024-01-02T00:00:00-05:00++# NULL bounds+query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(NULL, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)+----++query P+SELECT * FROM range(arrow_cast(NULL, 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)+----++query P+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), NULL, INTERVAL '1' DAY)+----++# Bounds outside the nanosecond range+statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps: 9223372036854775807 \(Timestamp\(Second, None\)\) is outside 1677\-09\-21T00:12:43\.145224192 to 2262\-04\-11T23:47:16\.854775807+SELECT * FROM generate_series(arrow_cast(9223372036854775807, 'Timestamp(Second, None)'), arrow_cast(0, 'Timestamp(Second, None)'), INTERVAL '-1' DAY)++statement error DataFusion error: Error during planning: Second argument for generate_series is out of range of nanosecond timestamps: \-9223372036854775808 \(Timestamp\(Millisecond, None\)\) is outside 1677\-09\-21T00:12:43\.145224192 to 2262\-04\-11T23:47:16\.854775807+SELECT * FROM generate_series(arrow_cast(0, 'Timestamp(Millisecond, None)'), arrow_cast(-9223372036854775808, 'Timestamp(Millisecond, None)'), INTERVAL '-1' DAY)++statement error DataFusion error: Error during planning: First argument for range is out of range of nanosecond timestamps: 9223372036854775807 \(Timestamp\(Microsecond, None\)\) is outside+SELECT * FROM range(arrow_cast(9223372036854775807, 'Timestamp(Microsecond, None)'), arrow_cast(0, 'Timestamp(Microsecond, None)'), INTERVAL '-1' DAY)++# Edge of the nanosecond range for each unit+query P+SELECT * FROM generate_series(arrow_cast(9223372036, 'Timestamp(Second, None)'), arrow_cast(9223372036, 'Timestamp(Second, None)'), INTERVAL '1' DAY)+----+2262-04-11T23:47:16++query P+SELECT * FROM generate_series(arrow_cast(-9223372036854775, 'Timestamp(Microsecond, None)'), arrow_cast(-9223372036854775, 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)+----+1677-09-21T00:12:43.145225++query P+SELECT * FROM generate_series(arrow_cast(-9223372036, 'Timestamp(Second, None)'), arrow_cast(-9223372036, 'Timestamp(Second, None)'), INTERVAL '1' DAY)+----+1677-09-21T00:12:44++query P+SELECT * FROM generate_series(arrow_cast(9223372036854, 'Timestamp(Millisecond, None)'), arrow_cast(9223372036854, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)+----+2262-04-11T23:47:16.854++query P+SELECT * FROM generate_series(arrow_cast(-9223372036854, 'Timestamp(Millisecond, None)'), arrow_cast(-9223372036854, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)+----+1677-09-21T00:12:43.146++query P+SELECT * FROM generate_series(arrow_cast(9223372036854775, 'Timestamp(Microsecond, None)'), arrow_cast(9223372036854775, 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)+----+2262-04-11T23:47:16.854775++statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps+SELECT * FROM generate_series(arrow_cast(9223372037, 'Timestamp(Second, None)'), arrow_cast(9223372037, 'Timestamp(Second, None)'), INTERVAL '1' DAY)++statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps+SELECT * FROM generate_series(arrow_cast(-9223372036854776, 'Timestamp(Microsecond, None)'), arrow_cast(-9223372036854776, 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)++statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps+SELECT * FROM generate_series(arrow_cast(-9223372037, 'Timestamp(Second, None)'), arrow_cast(-9223372037, 'Timestamp(Second, None)'), INTERVAL '1' DAY)++statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps+SELECT * FROM generate_series(arrow_cast(9223372036855, 'Timestamp(Millisecond, None)'), arrow_cast(9223372036855, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)++statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps+SELECT * FROM generate_series(arrow_cast(-9223372036855, 'Timestamp(Millisecond, None)'), arrow_cast(-9223372036855, 'Timestamp(Millisecond, None)'), INTERVAL '1' DAY)++statement error DataFusion error: Error during planning: First argument for generate_series is out of range of nanosecond timestamps+SELECT * FROM generate_series(arrow_cast(9223372036854776, 'Timestamp(Microsecond, None)'), arrow_cast(9223372036854776, 'Timestamp(Microsecond, None)'), INTERVAL '1' DAY)++# Invalid argument types+statement error DataFusion error: Error during planning: Second argument for generate_series must be a TIMESTAMP or NULL constant, got Int64+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), 5, INTERVAL '1' DAY)++statement error DataFusion error: Error during planning: Third argument for generate_series must be an INTERVAL or NULL constant, got Utf8+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), arrow_cast(TIMESTAMP '2024-01-03T00:00:00', 'Timestamp(Second, None)'), 'not an interval')++statement error DataFusion error: Error during planning: Second argument for generate_series must be a DATE or NULL constant, got Int64+SELECT * FROM generate_series(DATE '2024-01-01', 5, INTERVAL '1' DAY)++statement error DataFusion error: Error during planning: Second argument for generate_series must be a TIMESTAMP or NULL constant, got \(<subquery>\)+SELECT * FROM generate_series(arrow_cast(TIMESTAMP '2024-01-01T00:00:00', 'Timestamp(Second, None)'), (SELECT TIMESTAMP '2024-01-03T00:00:00'), INTERVAL '1' DAY)+# UDF and UDTF `generate_series` can be used simultaneouslyquery ? rowsortSELECT generate_series(1, t1.end) FROM generate_series(3, 5) as t1(end)