Apache DataFusion SQL Query Engine
fix: return an error for an invalid CSV null_regex instead of panicking (#25261)
S Prajwall Narayana98d84121cd86
diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rsindex a8c27369cb..1f7da7e849 100644--- a/datafusion/datasource-csv/src/file_format.rs+++ b/datafusion/datasource-csv/src/file_format.rs@@ -30,8 +30,8 @@ use arrow::error::ArrowError;use datafusion_common::config::{ConfigField, ConfigFileType, CsvOptions};use datafusion_common::file_options::csv_writer::CsvWriterOptions;use datafusion_common::{- DEFAULT_CSV_EXTENSION, DataFusionError, GetExt, Result, Statistics, exec_err,- not_impl_err,+ DEFAULT_CSV_EXTENSION, DataFusionError, GetExt, Result, Statistics,+ exec_datafusion_err, exec_err, not_impl_err,};use datafusion_common_runtime::SpawnedTask;use datafusion_datasource::TableSchema;@@ -540,6 +540,21 @@ impl CsvFormat {let mut record_number = -1;let initial_records_to_read = records_to_read;+ // Compile once rather than per chunk, and report a malformed pattern+ // instead of panicking on it.+ let null_regex = self+ .options+ .null_regex+ .as_ref()+ .map(|null_regex| {+ Regex::new(null_regex).map_err(|e| {+ exec_datafusion_err!(+ "Unable to parse CSV null regex '{null_regex}': {e}"+ )+ })+ })+ .transpose()?;+pin_mut!(stream);while let Some(chunk) = stream.next().await.transpose()? {@@ -557,10 +572,8 @@ impl CsvFormat {.with_quote(self.options.quote).with_truncated_rows(self.options.truncated_rows.unwrap_or(false));- if let Some(null_regex) = &self.options.null_regex {- let regex = Regex::new(null_regex.as_str())- .expect("Unable to parse CSV null regex.");- format = format.with_null_regex(regex);+ if let Some(regex) = &null_regex {+ format = format.with_null_regex(regex.clone());}if let Some(escape) = self.options.escape {diff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.sltindex b5b6130940..3562383c8b 100644--- a/datafusion/sqllogictest/test_files/csv_files.slt+++ b/datafusion/sqllogictest/test_files/csv_files.slt@@ -22,6 +22,14 @@ STORED AS CSVLOCATION '../core/tests/data/duplicate_header.csv'OPTIONS ('format.has_header' 'true');+# A malformed null_regex is reported as an error rather than panicking the+# query. Schema inference compiles the pattern, so CREATE is where it surfaces.+statement error Unable to parse CSV null regex+CREATE EXTERNAL TABLE csv_with_invalid_null_regex+STORED AS CSV+LOCATION '../core/tests/data/quote.csv'+OPTIONS ('format.has_header' 'true', 'format.null_regex' '(');+# create_external_table_with_quote_escapestatement okCREATE EXTERNAL TABLE csv_with_quote (