Apache DataFusion SQL Query Engine
fix: validate temporal display formats (#26045)
mikae4f9b030e1f2
diff --git a/datafusion-cli/src/print_format.rs b/datafusion-cli/src/print_format.rsindex ff92874813..594550ac07 100644--- a/datafusion-cli/src/print_format.rs+++ b/datafusion-cli/src/print_format.rs@@ -206,8 +206,8 @@ mod tests {use super::*;use std::sync::Arc;- use arrow::array::Int32Array;- use arrow::datatypes::{DataType, Field, Schema};+ use arrow::array::{Int32Array, Time64NanosecondArray};+ use arrow::datatypes::{DataType, Field, Schema, TimeUnit};use insta::{allow_duplicates, assert_snapshot};#[test]@@ -489,6 +489,48 @@ mod tests {assert_eq!(output, "")}+ #[test]+ fn invalid_temporal_format_errors_before_table_output() {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "time",+ DataType::Time64(TimeUnit::Nanosecond),+ false,+ )]));+ let batch = RecordBatch::try_new(+ schema.clone(),+ vec![Arc::new(Time64NanosecondArray::from(vec![1_000, 2_000]))],+ )+ .unwrap();+ let empty_batch = RecordBatch::new_empty(schema.clone());+ let format_options = FormatOptions {+ time_format: Some("%Y".to_owned()),+ ..Default::default()+ };++ for batches in [vec![batch], vec![empty_batch]] {+ let mut output = Vec::new();+ let error = PrintFormat::Table+ .print_batches(+ &mut output,+ schema.clone(),+ &batches,+ MaxRows::Limited(1),+ true,+ &format_options,+ )+ .expect_err("incompatible time formats must fail before writing output");++ assert!(+ error.to_string().contains("datafusion.format.time_format"),+ "unexpected error: {error}"+ );+ assert!(+ output.is_empty(),+ "format errors must not write partial output"+ );+ }+ }+#[derive(Debug)]struct PrintBatchesTest {format: PrintFormat,diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 302cb0b255..ee74faa5bd 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -2133,6 +2133,62 @@ config_namespace! {impl<'a> TryFrom<&'a FormatOptions> for arrow::util::display::FormatOptions<'a> {type Error = DataFusionError;fn try_from(options: &'a FormatOptions) -> Result<Self> {+ for (name, format) in [+ ("date_format", options.date_format.as_deref()),+ ("datetime_format", options.datetime_format.as_deref()),+ ("timestamp_format", options.timestamp_format.as_deref()),+ (+ "timestamp_tz_format",+ options.timestamp_tz_format.as_deref(),+ ),+ ("time_format", options.time_format.as_deref()),+ ] {+ if let Some(format) = format {+ let items = chrono::format::StrftimeItems::new(format).parse().map_err(+ |error| {+ DataFusionError::Configuration(format!(+ "Invalid datafusion.format.{name}: {error}"+ ))+ },+ )?;++ // Parsing validates directives, but some directives require fields+ // that are absent from a particular temporal type (for example,+ // `%H` for a date). Chrono reports those errors while formatting.+ let mut rendered = String::new();+ let result = match name {+ "date_format" => chrono::NaiveDate::from_ymd_opt(2001, 2, 3)+ .expect("valid sample date")+ .format_with_items(items.iter())+ .write_to(&mut rendered),+ "datetime_format" | "timestamp_format" => {+ chrono::NaiveDate::from_ymd_opt(2001, 2, 3)+ .expect("valid sample date")+ .and_hms_opt(4, 5, 6)+ .expect("valid sample time")+ .format_with_items(items.iter())+ .write_to(&mut rendered)+ }+ "timestamp_tz_format" => {+ chrono::DateTime::from_timestamp(981_173_106, 0)+ .expect("valid sample timestamp")+ .format_with_items(items.iter())+ .write_to(&mut rendered)+ }+ "time_format" => chrono::NaiveTime::from_hms_opt(4, 5, 6)+ .expect("valid sample time")+ .format_with_items(items.iter())+ .write_to(&mut rendered),+ _ => unreachable!("all temporal format names are handled above"),+ };+ if result.is_err() {+ return Err(DataFusionError::Configuration(format!(+ "Invalid datafusion.format.{name}: format is incompatible with its temporal type"+ )));+ }+ }+ }+Ok(Self::new().with_display_error(options.safe).with_null(&options.null)@@ -4242,6 +4298,132 @@ mod tests {ConfigEntry, ConfigExtension, ConfigField, ConfigFileType, ExtensionOptions,Extensions, TableOptions,};++ #[test]+ fn invalid_temporal_format_is_rejected_before_rendering() {+ use crate::config::FormatOptions;++ let invalid_options = [+ (+ "date_format",+ FormatOptions {+ date_format: Some("%".to_owned()),+ ..Default::default()+ },+ ),+ (+ "datetime_format",+ FormatOptions {+ datetime_format: Some("%".to_owned()),+ ..Default::default()+ },+ ),+ (+ "timestamp_format",+ FormatOptions {+ timestamp_format: Some("%".to_owned()),+ ..Default::default()+ },+ ),+ (+ "timestamp_tz_format",+ FormatOptions {+ timestamp_tz_format: Some("%".to_owned()),+ ..Default::default()+ },+ ),+ (+ "time_format",+ FormatOptions {+ time_format: Some("%".to_owned()),+ ..Default::default()+ },+ ),+ ];++ for (name, options) in invalid_options {+ let error = arrow::util::display::FormatOptions::try_from(&options)+ .expect_err("invalid format strings must be rejected before rendering");+ assert!(+ error+ .to_string()+ .contains(&format!("datafusion.format.{name}")),+ "unexpected validation error: {error}"+ );+ }++ let incompatible_options = [+ (+ "time_format",+ FormatOptions {+ time_format: Some("%Y".to_owned()),+ ..Default::default()+ },+ ),+ (+ "date_format",+ FormatOptions {+ date_format: Some("%H".to_owned()),+ ..Default::default()+ },+ ),+ (+ "timestamp_format",+ FormatOptions {+ timestamp_format: Some("%+".to_owned()),+ ..Default::default()+ },+ ),+ ];++ for (name, options) in incompatible_options {+ let error = arrow::util::display::FormatOptions::try_from(&options)+ .expect_err(+ "type-incompatible formats must be rejected before rendering",+ );+ assert!(+ error+ .to_string()+ .contains(&format!("datafusion.format.{name}")),+ "unexpected validation error: {error}"+ );+ }++ arrow::util::display::FormatOptions::try_from(&FormatOptions::default())+ .expect("the default temporal formats must remain valid");++ let options = FormatOptions {+ timestamp_tz_format: Some("%+".to_owned()),+ ..Default::default()+ };+ let options: arrow::util::display::FormatOptions = (&options)+ .try_into()+ .expect("the timezone-aware ISO 8601 format must be valid");+ let schema = arrow::datatypes::Schema::new(vec![arrow::datatypes::Field::new(+ "timestamp",+ arrow::datatypes::DataType::Timestamp(+ arrow::datatypes::TimeUnit::Nanosecond,+ Some("UTC".into()),+ ),+ false,+ )]);+ let batch = arrow::record_batch::RecordBatch::try_new(+ std::sync::Arc::new(schema),+ vec![std::sync::Arc::new(+ arrow::array::TimestampNanosecondArray::from(vec![0])+ .with_timezone("UTC"),+ )],+ )+ .expect("valid timestamp batch");+ let formatted =+ arrow::util::pretty::pretty_format_batches_with_options(&[batch], &options)+ .expect("valid timezone-aware format must render")+ .to_string();+ assert!(+ formatted.contains("1970-01-01T00:00:00+00:00"),+ "unexpected formatted output: {formatted}"+ );+ }use std::any::Any;use std::collections::HashMap;