Apache DataFusion SQL Query Engine
refactor: Improve `SessionContext::parse_duration` API (#20816)
Eren Avsarogullari9c3c01aa54fb
diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex 58d433e7dd..ad254d61b5 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -1184,7 +1184,7 @@ impl SessionContext {builder.with_object_list_cache_limit(limit)}"list_files_cache_ttl" => {- let duration = Self::parse_duration(value)?;+ let duration = Self::parse_duration(variable, value)?;builder.with_object_list_cache_ttl(Some(duration))}_ => return plan_err!("Unknown runtime configuration: {variable}"),@@ -1323,36 +1323,66 @@ impl SessionContext {}}- fn parse_duration(duration: &str) -> Result<Duration> {+ fn parse_duration(config_name: &str, duration: &str) -> Result<Duration> {+ if duration.trim().is_empty() {+ return Err(plan_datafusion_err!(+ "Duration should not be empty or blank for '{config_name}'"+ ));+ }+let mut minutes = None;let mut seconds = None;for duration in duration.split_inclusive(&['m', 's']) {let (number, unit) = duration.split_at(duration.len() - 1);let number: u64 = number.parse().map_err(|_| {- plan_datafusion_err!("Failed to parse number from duration '{duration}'")+ plan_datafusion_err!("Failed to parse number from duration '{duration}' for '{config_name}'")})?;match unit {"m" if minutes.is_none() && seconds.is_none() => minutes = Some(number),"s" if seconds.is_none() => seconds = Some(number),- _ => plan_err!(- "Invalid duration, unit must be either 'm' (minutes), or 's' (seconds), and be in the correct order"+ other => plan_err!(+ "Invalid duration unit: '{other}'. The unit must be either 'm' (minutes), or 's' (seconds), and be in the correct order for '{config_name}'")?,}}- let duration = Duration::from_secs(- minutes.unwrap_or_default() * 60 + seconds.unwrap_or_default(),- );+ let secs = Self::check_overflow(config_name, minutes, 60, seconds)?;+ let duration = Duration::from_secs(secs);if duration.is_zero() {- return plan_err!("Duration must be greater than 0 seconds");+ return plan_err!(+ "Duration must be greater than 0 seconds for '{config_name}'"+ );}Ok(duration)}+ fn check_overflow(+ config_name: &str,+ mins: Option<u64>,+ multiplier: u64,+ secs: Option<u64>,+ ) -> Result<u64> {+ let first_part_of_secs = mins.unwrap_or_default().checked_mul(multiplier);+ if first_part_of_secs.is_none() {+ plan_err!(+ "Duration has overflowed allowed maximum limit due to 'mins * {multiplier}' when setting '{config_name}'"+ )?+ }+ let second_part_of_secs = first_part_of_secs+ .unwrap()+ .checked_add(secs.unwrap_or_default());+ if second_part_of_secs.is_none() {+ plan_err!(+ "Duration has overflowed allowed maximum limit due to 'mins * {multiplier} + secs' when setting '{config_name}'"+ )?+ }+ Ok(second_part_of_secs.unwrap())+ }+async fn create_custom_table(&self,cmd: &CreateExternalTable,@@ -2803,6 +2833,8 @@ mod tests {#[test]fn test_parse_duration() {+ const LIST_FILES_CACHE_TTL: &str = "datafusion.runtime.list_files_cache_ttl";+// Valid durationsfor (duration, want) in [("1s", Duration::from_secs(1)),@@ -2810,14 +2842,81 @@ mod tests {("1m0s", Duration::from_secs(60)),("1m1s", Duration::from_secs(61)),] {- let have = SessionContext::parse_duration(duration).unwrap();+ let have =+ SessionContext::parse_duration(LIST_FILES_CACHE_TTL, duration).unwrap();assert_eq!(want, have);}// Invalid durations- for duration in ["0s", "0m", "1s0m", "1s1m"] {- let have = SessionContext::parse_duration(duration);+ for duration in [+ "0s", "0m", "1s0m", "1s1m", "XYZ", "1h", "XYZm2s", "", " ", "-1m", "1m 1s",+ "1m1s ", " 1m1s",+ ] {+ let have = SessionContext::parse_duration(LIST_FILES_CACHE_TTL, duration);assert!(have.is_err());+ assert!(+ have.unwrap_err()+ .message()+ .to_string()+ .contains(LIST_FILES_CACHE_TTL)+ );+ }+ }++ #[test]+ fn test_parse_duration_with_overflow_check() {+ const LIST_FILES_CACHE_TTL: &str = "datafusion.runtime.list_files_cache_ttl";++ // Valid durations which are close to max allowed limit+ for (duration, want) in [+ (+ "18446744073709551615s",+ Duration::from_secs(18446744073709551615),+ ),+ (+ "307445734561825860m",+ Duration::from_secs(307445734561825860 * 60),+ ),+ (+ "307445734561825860m10s",+ Duration::from_secs(307445734561825860 * 60 + 10),+ ),+ (+ "1m18446744073709551555s",+ Duration::from_secs(60 + 18446744073709551555),+ ),+ ] {+ let have =+ SessionContext::parse_duration(LIST_FILES_CACHE_TTL, duration).unwrap();+ assert_eq!(want, have);+ }++ // Invalid durations which overflow max allowed limit+ for (duration, error_message_prefix) in [+ (+ "18446744073709551616s",+ "Failed to parse number from duration",+ ),+ (+ "307445734561825861m",+ "Duration has overflowed allowed maximum limit due to",+ ),+ (+ "307445734561825860m60s",+ "Duration has overflowed allowed maximum limit due to",+ ),+ (+ "1m18446744073709551556s",+ "Duration has overflowed allowed maximum limit due to",+ ),+ ] {+ let have = SessionContext::parse_duration(LIST_FILES_CACHE_TTL, duration);+ assert!(have.is_err());+ let error_message = have.unwrap_err().message().to_string();+ assert!(+ error_message.contains(error_message_prefix)+ && error_message.contains(LIST_FILES_CACHE_TTL)+ );}}diff --git a/datafusion/sqllogictest/test_files/set_variable.slt b/datafusion/sqllogictest/test_files/set_variable.sltindex 7be353f057..375d349251 100644--- a/datafusion/sqllogictest/test_files/set_variable.slt+++ b/datafusion/sqllogictest/test_files/set_variable.slt@@ -450,3 +450,33 @@ datafusion.runtime.temp_directorystatement error DataFusion error: Error during planning: Unsupported value NullSET datafusion.runtime.memory_limit = NULL++statement error DataFusion error: Error during planning: Unsupported value Null+SET datafusion.runtime.list_files_cache_ttl = NULL++statement error DataFusion error: Error during planning: Duration should not be empty or blank for 'datafusion.runtime.list_files_cache_ttl'+SET datafusion.runtime.list_files_cache_ttl = ' '++statement ok+SET datafusion.runtime.list_files_cache_ttl = '18446744073709551615s'++statement error DataFusion error: Error during planning: Failed to parse number from duration '18446744073709551616s' for 'datafusion.runtime.list_files_cache_ttl'+SET datafusion.runtime.list_files_cache_ttl = '18446744073709551616s'++statement ok+SET datafusion.runtime.list_files_cache_ttl = '307445734561825860m'++statement ok+SET datafusion.runtime.list_files_cache_ttl = '307445734561825860m10s'++statement error DataFusion error: Error during planning: Duration has overflowed allowed maximum limit due to 'mins \* 60' when setting 'datafusion\.runtime\.list_files_cache_ttl'+SET datafusion.runtime.list_files_cache_ttl = '307445734561825861m'++statement error DataFusion error: Error during planning: Duration has overflowed allowed maximum limit due to 'mins \* 60 \+ secs' when setting 'datafusion\.runtime\.list_files_cache_ttl'+SET datafusion.runtime.list_files_cache_ttl = '307445734561825860m60s'++statement ok+SET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551555s'++statement error DataFusion error: Error during planning: Duration has overflowed allowed maximum limit due to 'mins \* 60 \+ secs' when setting 'datafusion\.runtime\.list_files_cache_ttl'+SET datafusion.runtime.list_files_cache_ttl = '1m18446744073709551556s'diff --git a/docs/source/library-user-guide/upgrading/52.0.0.md b/docs/source/library-user-guide/upgrading/52.0.0.mdindex 4c659b6118..8bf2f803be 100644--- a/docs/source/library-user-guide/upgrading/52.0.0.md+++ b/docs/source/library-user-guide/upgrading/52.0.0.md@@ -51,7 +51,7 @@ those changes until the `ListingTableProvider` instance is dropped and recreatedYou can configure the maximum cache size and cache entry expiration time via configuration options:- `datafusion.runtime.list_files_cache_limit` - Limits the size of the cache in bytes-- `datafusion.runtime.list_files_cache_ttl` - Limits the TTL (time-to-live) of an entry in seconds+- `datafusion.runtime.list_files_cache_ttl` - Limits the TTL (time-to-live) of an entry in minutes and/or secondsDetailed configuration information can be found in the [DataFusion RuntimeConfiguration](https://datafusion.apache.org/user-guide/configs.html#runtime-configuration-settings) user's guide.