Apache DataFusion SQL Query Engine
perf: Cast entire Date32 array to Date64 on 1st failure (#21948)
Huy Mace4f43ff7afa0
diff --git a/datafusion/functions/src/datetime/to_char.rs b/datafusion/functions/src/datetime/to_char.rsindex cdd4b7d5d0..5accddd07f 100644--- a/datafusion/functions/src/datetime/to_char.rs+++ b/datafusion/functions/src/datetime/to_char.rs@@ -248,6 +248,11 @@ fn to_char_array(args: &[ColumnarValue]) -> Result<ColumnarValue> {StringBuilder::with_capacity(data_array.len(), data_array.len() * fmt_len);let mut buffer = String::with_capacity(fmt_len);+ // Lazily computed Date64 cast of the entire array, used when a Date32+ // format string contains time specifiers that the Date32 formatter+ // cannot handle. Cast once and reuse for all subsequent rows+ let mut date64_array: Option<ArrayRef> = None;+for idx in 0..data_array.len() {if format_array.is_null(idx) || data_array.is_null(idx) {builder.append_null();@@ -266,13 +271,18 @@ fn to_char_array(args: &[ColumnarValue]) -> Result<ColumnarValue> {// buffer and `append_value` on success.match formatter.value(idx).write(&mut buffer) {Ok(()) => builder.append_value(&buffer),- // Retry with Date64 (see comment in to_char_scalar).Err(_) if data_type == &Date32 => {buffer.clear();- let date64_value = cast(&data_array.slice(idx, 1), &Date64)?;- let retry_fmt =- ArrayFormatter::try_new(date64_value.as_ref(), &format_options)?;- retry_fmt.value(0).write(&mut buffer)?;+ let date64_ref = match &date64_array {+ Some(arr) => arr.as_ref(),+ None => {+ date64_array = Some(cast(data_array.as_ref(), &Date64)?);+ date64_array.as_ref().unwrap().as_ref()+ }+ };+ let retry_options = build_format_options(&Date64, format)?;+ let retry_fmt = ArrayFormatter::try_new(date64_ref, &retry_options)?;+ retry_fmt.value(idx).write(&mut buffer)?;builder.append_value(&buffer);}Err(e) => return Err(e.into()),@@ -307,11 +317,21 @@ mod tests {#[test]fn test_array_array() {- let array_array_data = vec![(- Arc::new(Date32Array::from(vec![18506, 18507])) as ArrayRef,- StringArray::from(vec!["%Y::%m::%d", "%Y::%m::%d %S::%M::%H %f"]),- StringArray::from(vec!["2020::09::01", "2020::09::02 00::00::00 000000000"]),- )];+ let array_array_data = vec![+ (+ Arc::new(Date32Array::from(vec![18506, 18507])) as ArrayRef,+ StringArray::from(vec!["%Y::%m::%d", "%Y::%m::%d %S::%M::%H %f"]),+ StringArray::from(vec![+ "2020::09::01",+ "2020::09::02 00::00::00 000000000",+ ]),+ ),+ (+ Arc::new(Date32Array::from(vec![18506, 18507])) as ArrayRef,+ StringArray::from(vec!["%Y::%m::%d %H:%M:%S", "%d-%m-%Y %H:%M"]),+ StringArray::from(vec!["2020::09::01 00:00:00", "02-09-2020 00:00"]),+ ),+ ];for (value, format, expected) in array_array_data {let batch_len = value.len();