Skip to content

Apache DataFusion SQL Query Engine

fix: validate temporal display formats (#26045)

mikae4f9b030e1f2
diff --git a/datafusion-cli/src/print_format.rs b/datafusion-cli/src/print_format.rs
index ff92874813..594550ac07 100644
--- a/datafusion-cli/src/print_format.rs
+++ b/datafusion-cli/src/print_format.rs
@@ -206,8 +206,8 @@ mod tests {
use super::*;
use std::sync::Arc;
- use arrow::array::Int32Array;
- use arrow::datatypes::{DataType, Field, Schema};
+ use arrow::array::{Int32Array, Time64NanosecondArray};
+ use arrow::datatypes::{DataType, Field, Schema, TimeUnit};
use insta::{allow_duplicates, assert_snapshot};
#[test]
@@ -489,6 +489,48 @@ mod tests {
assert_eq!(output, "")
}
+ #[test]
+ fn invalid_temporal_format_errors_before_table_output() {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "time",
+ DataType::Time64(TimeUnit::Nanosecond),
+ false,
+ )]));
+ let batch = RecordBatch::try_new(
+ schema.clone(),
+ vec![Arc::new(Time64NanosecondArray::from(vec![1_000, 2_000]))],
+ )
+ .unwrap();
+ let empty_batch = RecordBatch::new_empty(schema.clone());
+ let format_options = FormatOptions {
+ time_format: Some("%Y".to_owned()),
+ ..Default::default()
+ };
+
+ for batches in [vec![batch], vec![empty_batch]] {
+ let mut output = Vec::new();
+ let error = PrintFormat::Table
+ .print_batches(
+ &mut output,
+ schema.clone(),
+ &batches,
+ MaxRows::Limited(1),
+ true,
+ &format_options,
+ )
+ .expect_err("incompatible time formats must fail before writing output");
+
+ assert!(
+ error.to_string().contains("datafusion.format.time_format"),
+ "unexpected error: {error}"
+ );
+ assert!(
+ output.is_empty(),
+ "format errors must not write partial output"
+ );
+ }
+ }
+
#[derive(Debug)]
struct PrintBatchesTest {
format: PrintFormat,
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
index 302cb0b255..ee74faa5bd 100644
--- a/datafusion/common/src/config.rs
+++ b/datafusion/common/src/config.rs
@@ -2133,6 +2133,62 @@ config_namespace! {
impl<'a> TryFrom<&'a FormatOptions> for arrow::util::display::FormatOptions<'a> {
type Error = DataFusionError;
fn try_from(options: &'a FormatOptions) -> Result<Self> {
+ for (name, format) in [
+ ("date_format", options.date_format.as_deref()),
+ ("datetime_format", options.datetime_format.as_deref()),
+ ("timestamp_format", options.timestamp_format.as_deref()),
+ (
+ "timestamp_tz_format",
+ options.timestamp_tz_format.as_deref(),
+ ),
+ ("time_format", options.time_format.as_deref()),
+ ] {
+ if let Some(format) = format {
+ let items = chrono::format::StrftimeItems::new(format).parse().map_err(
+ |error| {
+ DataFusionError::Configuration(format!(
+ "Invalid datafusion.format.{name}: {error}"
+ ))
+ },
+ )?;
+
+ // Parsing validates directives, but some directives require fields
+ // that are absent from a particular temporal type (for example,
+ // `%H` for a date). Chrono reports those errors while formatting.
+ let mut rendered = String::new();
+ let result = match name {
+ "date_format" => chrono::NaiveDate::from_ymd_opt(2001, 2, 3)
+ .expect("valid sample date")
+ .format_with_items(items.iter())
+ .write_to(&mut rendered),
+ "datetime_format" | "timestamp_format" => {
+ chrono::NaiveDate::from_ymd_opt(2001, 2, 3)
+ .expect("valid sample date")
+ .and_hms_opt(4, 5, 6)
+ .expect("valid sample time")
+ .format_with_items(items.iter())
+ .write_to(&mut rendered)
+ }
+ "timestamp_tz_format" => {
+ chrono::DateTime::from_timestamp(981_173_106, 0)
+ .expect("valid sample timestamp")
+ .format_with_items(items.iter())
+ .write_to(&mut rendered)
+ }
+ "time_format" => chrono::NaiveTime::from_hms_opt(4, 5, 6)
+ .expect("valid sample time")
+ .format_with_items(items.iter())
+ .write_to(&mut rendered),
+ _ => unreachable!("all temporal format names are handled above"),
+ };
+ if result.is_err() {
+ return Err(DataFusionError::Configuration(format!(
+ "Invalid datafusion.format.{name}: format is incompatible with its temporal type"
+ )));
+ }
+ }
+ }
+
Ok(Self::new()
.with_display_error(options.safe)
.with_null(&options.null)
@@ -4242,6 +4298,132 @@ mod tests {
ConfigEntry, ConfigExtension, ConfigField, ConfigFileType, ExtensionOptions,
Extensions, TableOptions,
};
+
+ #[test]
+ fn invalid_temporal_format_is_rejected_before_rendering() {
+ use crate::config::FormatOptions;
+
+ let invalid_options = [
+ (
+ "date_format",
+ FormatOptions {
+ date_format: Some("%".to_owned()),
+ ..Default::default()
+ },
+ ),
+ (
+ "datetime_format",
+ FormatOptions {
+ datetime_format: Some("%".to_owned()),
+ ..Default::default()
+ },
+ ),
+ (
+ "timestamp_format",
+ FormatOptions {
+ timestamp_format: Some("%".to_owned()),
+ ..Default::default()
+ },
+ ),
+ (
+ "timestamp_tz_format",
+ FormatOptions {
+ timestamp_tz_format: Some("%".to_owned()),
+ ..Default::default()
+ },
+ ),
+ (
+ "time_format",
+ FormatOptions {
+ time_format: Some("%".to_owned()),
+ ..Default::default()
+ },
+ ),
+ ];
+
+ for (name, options) in invalid_options {
+ let error = arrow::util::display::FormatOptions::try_from(&options)
+ .expect_err("invalid format strings must be rejected before rendering");
+ assert!(
+ error
+ .to_string()
+ .contains(&format!("datafusion.format.{name}")),
+ "unexpected validation error: {error}"
+ );
+ }
+
+ let incompatible_options = [
+ (
+ "time_format",
+ FormatOptions {
+ time_format: Some("%Y".to_owned()),
+ ..Default::default()
+ },
+ ),
+ (
+ "date_format",
+ FormatOptions {
+ date_format: Some("%H".to_owned()),
+ ..Default::default()
+ },
+ ),
+ (
+ "timestamp_format",
+ FormatOptions {
+ timestamp_format: Some("%+".to_owned()),
+ ..Default::default()
+ },
+ ),
+ ];
+
+ for (name, options) in incompatible_options {
+ let error = arrow::util::display::FormatOptions::try_from(&options)
+ .expect_err(
+ "type-incompatible formats must be rejected before rendering",
+ );
+ assert!(
+ error
+ .to_string()
+ .contains(&format!("datafusion.format.{name}")),
+ "unexpected validation error: {error}"
+ );
+ }
+
+ arrow::util::display::FormatOptions::try_from(&FormatOptions::default())
+ .expect("the default temporal formats must remain valid");
+
+ let options = FormatOptions {
+ timestamp_tz_format: Some("%+".to_owned()),
+ ..Default::default()
+ };
+ let options: arrow::util::display::FormatOptions = (&options)
+ .try_into()
+ .expect("the timezone-aware ISO 8601 format must be valid");
+ let schema = arrow::datatypes::Schema::new(vec![arrow::datatypes::Field::new(
+ "timestamp",
+ arrow::datatypes::DataType::Timestamp(
+ arrow::datatypes::TimeUnit::Nanosecond,
+ Some("UTC".into()),
+ ),
+ false,
+ )]);
+ let batch = arrow::record_batch::RecordBatch::try_new(
+ std::sync::Arc::new(schema),
+ vec![std::sync::Arc::new(
+ arrow::array::TimestampNanosecondArray::from(vec![0])
+ .with_timezone("UTC"),
+ )],
+ )
+ .expect("valid timestamp batch");
+ let formatted =
+ arrow::util::pretty::pretty_format_batches_with_options(&[batch], &options)
+ .expect("valid timezone-aware format must render")
+ .to_string();
+ assert!(
+ formatted.contains("1970-01-01T00:00:00+00:00"),
+ "unexpected formatted output: {formatted}"
+ );
+ }
use std::any::Any;
use std::collections::HashMap;