Apache DataFusion SQL Query Engine
perf: precompile formats in to_time (#23964)
linfeng44aae9486590
diff --git a/datafusion/functions/src/datetime/to_time.rs b/datafusion/functions/src/datetime/to_time.rsindex 94aa49fbba..45664e9416 100644--- a/datafusion/functions/src/datetime/to_time.rs+++ b/datafusion/functions/src/datetime/to_time.rs@@ -22,7 +22,7 @@ use arrow::array::types::Time64NanosecondType;use arrow::array::{Array, PrimitiveArray, StringArrayType};use arrow::datatypes::DataType;use arrow::datatypes::DataType::*;-use chrono::NaiveTime;+use chrono::format::{Item, Parsed, StrftimeItems, parse};use datafusion_common::{Result, ScalarValue, exec_err};use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,@@ -141,6 +141,7 @@ impl ScalarUDFImpl for ToTimeFunc {/// Convert string arguments to time (standalone function, not a method on ToTimeFunc)fn string_to_time(args: &[ColumnarValue]) -> Result<ColumnarValue> {let formats = collect_formats(args)?;+ let formats = compile_formats(&formats);match &args[0] {ColumnarValue::Scalar(ScalarValue::Utf8(s))@@ -207,10 +208,25 @@ fn timestamp_to_time(arg: &ColumnarValue) -> Result<ColumnarValue> {arg.cast_to(&Time64(arrow::datatypes::TimeUnit::Nanosecond), None)}+struct CompiledTimeFormat<'a> {+ source: &'a str,+ items: Vec<Item<'a>>,+}++fn compile_formats<'a>(formats: &[&'a str]) -> Vec<CompiledTimeFormat<'a>> {+ formats+ .iter()+ .map(|source| CompiledTimeFormat {+ source,+ items: StrftimeItems::new(source).collect(),+ })+ .collect()+}+/// Parse time array using the provided formatsfn parse_time_array<'a, A: StringArrayType<'a>>(array: &A,- formats: &[&str],+ formats: &[CompiledTimeFormat<'_>],) -> Result<PrimitiveArray<Time64NanosecondType>> {let mut values = Vec::with_capacity(array.len());for i in 0..array.len() {@@ -224,10 +240,12 @@ fn parse_time_array<'a, A: StringArrayType<'a>>(}/// Parse time string using provided formats-fn parse_time_with_formats(s: &str, formats: &[&str]) -> Result<i64> {+fn parse_time_with_formats(s: &str, formats: &[CompiledTimeFormat<'_>]) -> Result<i64> {for format in formats {- if let Ok(time) = NaiveTime::parse_from_str(s, format) {- // Use Arrow's time_to_time64ns function instead of custom implementation+ let mut parsed = Parsed::new();+ if parse(&mut parsed, s, format.items.iter()).is_ok()+ && let Ok(time) = parsed.to_naive_time()+ {return Ok(time_to_time64ns(time));}}@@ -235,5 +253,8 @@ fn parse_time_with_formats(s: &str, formats: &[&str]) -> Result<i64> {"Error parsing '{}' as time. Tried formats: {:?}",s,formats+ .iter()+ .map(|format| format.source)+ .collect::<Vec<_>>())}diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.sltindex 002111d3f2..d73bc6eb06 100644--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt@@ -3529,6 +3529,28 @@ select to_time(time_str) from time_strings;statement okdrop table time_strings;+# Table input with multiple formats+# `%Q` is intentionally invalid; subsequent formats should still be tried.+query D rowsort+select to_time(+ time_str,+ '%Q',+ '%H:%M:%S',+ '%H-%M-%S',+ '%H/%M/%S'+) from (+ values+ ('12:30:45'),+ ('14-25-30'),+ ('09/05/01'),+ (NULL)+) as formatted_time_strings(time_str);+----+09:05:01+12:30:45+14:25:30+NULL+# Error casesquery error Error parsing 'not_a_time' as time