Skip to content

Apache DataFusion SQL Query Engine

fix: return an error for an invalid CSV null_regex instead of panicking (#25261)

S Prajwall Narayana98d84121cd86
diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs
index a8c27369cb..1f7da7e849 100644
--- a/datafusion/datasource-csv/src/file_format.rs
+++ b/datafusion/datasource-csv/src/file_format.rs
@@ -30,8 +30,8 @@ use arrow::error::ArrowError;
use datafusion_common::config::{ConfigField, ConfigFileType, CsvOptions};
use datafusion_common::file_options::csv_writer::CsvWriterOptions;
use datafusion_common::{
- DEFAULT_CSV_EXTENSION, DataFusionError, GetExt, Result, Statistics, exec_err,
- not_impl_err,
+ DEFAULT_CSV_EXTENSION, DataFusionError, GetExt, Result, Statistics,
+ exec_datafusion_err, exec_err, not_impl_err,
};
use datafusion_common_runtime::SpawnedTask;
use datafusion_datasource::TableSchema;
@@ -540,6 +540,21 @@ impl CsvFormat {
let mut record_number = -1;
let initial_records_to_read = records_to_read;
+ // Compile once rather than per chunk, and report a malformed pattern
+ // instead of panicking on it.
+ let null_regex = self
+ .options
+ .null_regex
+ .as_ref()
+ .map(|null_regex| {
+ Regex::new(null_regex).map_err(|e| {
+ exec_datafusion_err!(
+ "Unable to parse CSV null regex '{null_regex}': {e}"
+ )
+ })
+ })
+ .transpose()?;
+
pin_mut!(stream);
while let Some(chunk) = stream.next().await.transpose()? {
@@ -557,10 +572,8 @@ impl CsvFormat {
.with_quote(self.options.quote)
.with_truncated_rows(self.options.truncated_rows.unwrap_or(false));
- if let Some(null_regex) = &self.options.null_regex {
- let regex = Regex::new(null_regex.as_str())
- .expect("Unable to parse CSV null regex.");
- format = format.with_null_regex(regex);
+ if let Some(regex) = &null_regex {
+ format = format.with_null_regex(regex.clone());
}
if let Some(escape) = self.options.escape {
diff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.slt
index b5b6130940..3562383c8b 100644
--- a/datafusion/sqllogictest/test_files/csv_files.slt
+++ b/datafusion/sqllogictest/test_files/csv_files.slt
@@ -22,6 +22,14 @@ STORED AS CSV
LOCATION '../core/tests/data/duplicate_header.csv'
OPTIONS ('format.has_header' 'true');
+# A malformed null_regex is reported as an error rather than panicking the
+# query. Schema inference compiles the pattern, so CREATE is where it surfaces.
+statement error Unable to parse CSV null regex
+CREATE EXTERNAL TABLE csv_with_invalid_null_regex
+STORED AS CSV
+LOCATION '../core/tests/data/quote.csv'
+OPTIONS ('format.has_header' 'true', 'format.null_regex' '(');
+
# create_external_table_with_quote_escape
statement ok
CREATE EXTERNAL TABLE csv_with_quote (