Apache DataFusion SQL Query Engine
fix: apply CSV null_regex when reading, not only when inferring the schema (#25254)
S Prajwall Narayanac8c046491ed4
diff --git a/datafusion/core/tests/data/null_regex.csv b/datafusion/core/tests/data/null_regex.csvnew file mode 100644index 0000000000..387b1a66c7--- /dev/null+++ b/datafusion/core/tests/data/null_regex.csv@@ -0,0 +1,4 @@+c1,c2,c3+id0,10,alpha+id1,N/A,NULL+id2,30,gammadiff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rsindex b62589a3e8..3288f0e1d0 100644--- a/datafusion/datasource-csv/src/source.rs+++ b/datafusion/datasource-csv/src/source.rs@@ -48,6 +48,7 @@ use crate::file_format::CsvDecoder;use futures::{StreamExt, TryStreamExt};use object_store::buffered::BufWriter;use object_store::{GetOptions, GetResultPayload, ObjectStore};+use regex::Regex;use tokio::io::AsyncWriteExt;/// A Config for [`CsvOpener`]@@ -145,6 +146,11 @@ impl CsvSource {self.options.escape}+ /// Regex for fields that should be read as NULL+ pub fn null_regex(&self) -> Option<&str> {+ self.options.null_regex.as_deref()+ }+/// Initialize a CsvSource with escapepub fn with_escape(&self, escape: Option<u8>) -> Self {let mut conf = self.clone();@@ -185,10 +191,10 @@ impl CsvSource {impl CsvSource {fn open<R: Read>(&self, reader: R) -> Result<csv::Reader<R>> {- Ok(self.builder().build(reader)?)+ Ok(self.builder()?.build(reader)?)}- fn builder(&self) -> csv::ReaderBuilder {+ fn builder(&self) -> Result<csv::ReaderBuilder> {let mut builder =csv::ReaderBuilder::new(Arc::clone(self.table_schema.file_schema())).with_delimiter(self.delimiter())@@ -209,8 +215,14 @@ impl CsvSource {if let Some(comment) = self.comment() {builder = builder.with_comment(comment);}+ if let Some(null_regex) = self.null_regex() {+ let regex = Regex::new(null_regex).map_err(|e| {+ exec_datafusion_err!("Unable to parse CSV null regex '{null_regex}': {e}")+ })?;+ builder = builder.with_null_regex(regex);+ }- builder+ Ok(builder)}}@@ -452,7 +464,7 @@ impl FileOpener for CsvOpener {.await?.map_err(DataFusionError::from);- let decoder = config.builder().build_decoder();+ let decoder = config.builder()?.build_decoder();let input = file_compression_type.convert_stream(aligned_stream.boxed())?.fuse();@@ -486,7 +498,7 @@ impl FileOpener for CsvOpener {.boxed())}GetResultPayload::Stream(s) => {- let decoder = config.builder().build_decoder();+ let decoder = config.builder()?.build_decoder();let s = s.map_err(DataFusionError::from);let input = file_compression_type.convert_stream(s.boxed())?.fuse();@@ -658,3 +670,111 @@ impl CsvSource {Ok(DataSourceExec::from_data_source(conf))}}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::{Array, Int64Array, StringArray};+ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};++ fn csv_source(schema: SchemaRef, null_regex: Option<&str>) -> CsvSource {+ let options = CsvOptions {+ has_header: Some(true),+ null_regex: null_regex.map(str::to_string),+ ..CsvOptions::default()+ };+ let mut source = CsvSource::new(schema).with_csv_options(options);+ source.batch_size = Some(1024);+ source+ }++ #[test]+ fn null_regex_nulls_matching_string_values() {+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int64, true),+ Field::new("name", DataType::Utf8, true),+ ]));+ let source = csv_source(schema, Some("^(NULL|N/A)$"));++ let data = "id,name\n1,alice\n2,N/A\n3,carol\n";+ let batch = source+ .open(data.as_bytes())+ .unwrap()+ .next()+ .unwrap()+ .unwrap();++ let names = batch+ .column(1)+ .as_any()+ .downcast_ref::<StringArray>()+ .unwrap();+ assert_eq!(names.value(0), "alice");+ assert!(names.is_null(1), "N/A should be read as NULL");+ assert_eq!(names.value(2), "carol");+ }++ #[test]+ fn null_regex_nulls_matching_values_in_numeric_columns() {+ // Without the regex this fails to parse rather than producing NULL,+ // which is the case null_regex exists for.+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int64, true),+ Field::new("value", DataType::Int64, true),+ ]));+ let source = csv_source(schema, Some("^(NULL|N/A)$"));++ let data = "id,value\n1,10\n2,N/A\n3,30\n";+ let batch = source+ .open(data.as_bytes())+ .unwrap()+ .next()+ .unwrap()+ .unwrap();++ let values = batch+ .column(1)+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap();+ assert_eq!(values.value(0), 10);+ assert!(values.is_null(1), "N/A should be read as NULL");+ assert_eq!(values.value(2), 30);+ }++ #[test]+ fn without_null_regex_the_placeholder_is_read_verbatim() {+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int64, true),+ Field::new("name", DataType::Utf8, true),+ ]));+ let source = csv_source(schema, None);++ let data = "id,name\n1,alice\n2,N/A\n3,carol\n";+ let batch = source+ .open(data.as_bytes())+ .unwrap()+ .next()+ .unwrap()+ .unwrap();++ let names = batch+ .column(1)+ .as_any()+ .downcast_ref::<StringArray>()+ .unwrap();+ assert_eq!(names.value(1), "N/A");+ }++ #[test]+ fn invalid_null_regex_is_reported_as_an_error() {+ let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, true)]));+ let source = csv_source(schema, Some("("));++ let err = source.open(b"id\n1\n".as_slice()).unwrap_err();+ assert!(+ err.to_string().contains("Unable to parse CSV null regex"),+ "unexpected error: {err}"+ );+ }+}diff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.sltindex 3562383c8b..bef8e4f061 100644--- a/datafusion/sqllogictest/test_files/csv_files.slt+++ b/datafusion/sqllogictest/test_files/csv_files.slt@@ -22,6 +22,30 @@ STORED AS CSVLOCATION '../core/tests/data/duplicate_header.csv'OPTIONS ('format.has_header' 'true');+# null_regex applies to the data, not only to schema inference: a matching+# field is read as NULL whatever the column's type.+statement ok+CREATE EXTERNAL TABLE csv_with_null_regex (+c1 VARCHAR,+c2 INT,+c3 VARCHAR+) STORED AS CSV+LOCATION '../core/tests/data/null_regex.csv'+OPTIONS ('format.null_regex' '^(NULL|N/A)$',+ 'format.has_header' 'true');++query TIT+select * from csv_with_null_regex;+----+id0 10 alpha+id1 NULL NULL+id2 30 gamma++query II+select count(*), count(c2) from csv_with_null_regex;+----+3 2+# A malformed null_regex is reported as an error rather than panicking the# query. Schema inference compiles the pattern, so CREATE is where it surfaces.statement error Unable to parse CSV null regex