Apache DataFusion SQL Query Engine
fix: reject duplicate field names during CSV and Parquet schema inference (#24387)
batu12ba4b74d838
diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rsindex 90d7eb3b41..447d5244f9 100644--- a/datafusion/core/src/datasource/file_format/csv.rs+++ b/datafusion/core/src/datasource/file_format/csv.rs@@ -1628,4 +1628,31 @@ mod tests {Ok(())}++ #[tokio::test]+ async fn infer_schema_rejects_duplicate_header_names() -> Result<()> {+ let directory = tempfile::tempdir()?;+ let path = directory.path().join("duplicate_header.csv");+ std::fs::write(&path, "id,value,value\n1,10,100\n")?;++ let store = Arc::new(LocalFileSystem::new()) as _;+ let meta = crate::test::object_store::local_unpartitioned_file(&path);++ let ctx = SessionContext::new().state();+ let error = CsvFormat::default()+ .with_has_header(true)+ .infer_schema(&ctx, &store, std::slice::from_ref(&meta))+ .await+ .expect_err("duplicate header names must not infer a schema")+ .to_string();++ assert!(+ error.contains("duplicate unqualified field name")+ && error.contains("value")+ && error.contains("duplicate_header.csv"),+ "unexpected error: {error}"+ );++ Ok(())+ }}diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rsindex 0f5db4a057..bbd9d0937a 100644--- a/datafusion/core/src/datasource/file_format/parquet.rs+++ b/datafusion/core/src/datasource/file_format/parquet.rs@@ -1822,4 +1822,38 @@ mod tests {Ok(())}++ #[tokio::test]+ async fn infer_schema_rejects_duplicate_field_names() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int64, false),+ Field::new("value", DataType::Int64, false),+ Field::new("value", DataType::Int64, false),+ ]));+ let batch = RecordBatch::try_new(+ schema,+ vec![+ Arc::new(Int64Array::from(vec![1, 2, 3])) as ArrayRef,+ Arc::new(Int64Array::from(vec![10, 20, 30])) as ArrayRef,+ Arc::new(Int64Array::from(vec![100, 200, 300])) as ArrayRef,+ ],+ )?;++ let store = Arc::new(LocalFileSystem::new()) as _;+ let (meta, _files) = store_parquet(vec![batch], false).await?;++ let ctx = SessionContext::new().state();+ let error = ParquetFormat::default()+ .infer_schema(&ctx, &store, &meta)+ .await+ .expect_err("duplicate field names must not infer a schema")+ .to_string();++ assert!(+ error.contains("duplicate unqualified field name") && error.contains("value"),+ "unexpected error: {error}"+ );++ Ok(())+ }}diff --git a/datafusion/core/tests/data/duplicate_header.csv b/datafusion/core/tests/data/duplicate_header.csvnew file mode 100644index 0000000000..0b2719a8ec--- /dev/null+++ b/datafusion/core/tests/data/duplicate_header.csv@@ -0,0 +1,2 @@+id,value,value+1,10,100diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rsindex c0d22b80f0..a094fdc3bf 100644--- a/datafusion/datasource-csv/src/file_format.rs+++ b/datafusion/datasource-csv/src/file_format.rs@@ -41,6 +41,7 @@ use datafusion_datasource::file::FileSource;use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_format::{DEFAULT_SCHEMA_INFER_MAX_RECORD, FileFormat, FileFormatFactory,+ ensure_unique_field_names,};use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};@@ -397,12 +398,24 @@ impl FileFormat for CsvFormat {)})?;records_to_read -= records_read;- schemas.push(schema);+ schemas.push((&object.location, schema));if records_to_read == 0 {break;}}+ let mut seen = HashSet::new();+ for (location, schema) in &schemas {+ ensure_unique_field_names(schema, &mut seen).map_err(|err| {+ DataFusionError::Context(+ format!("Error when processing CSV file {location}"),+ Box::new(err),+ )+ })?;+ }+ drop(seen);++ let schemas = schemas.into_iter().map(|(_, schema)| schema);let merged_schema = Schema::try_merge(schemas)?;Ok(Arc::new(merged_schema))}diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex 6358201c06..1b25a2c632 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -17,6 +17,7 @@//! [`ParquetFormat`]: Parquet [`FileFormat`] abstractions+use std::collections::HashSet;use std::fmt;use std::fmt::Debug;use std::ops::Range;@@ -37,7 +38,9 @@ use datafusion_datasource::TableSchema;use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_sink_config::FileSinkConfig;-use datafusion_datasource::file_format::{FileFormat, FileFormatFactory};+use datafusion_datasource::file_format::{+ FileFormat, FileFormatFactory, ensure_unique_field_names,+};use datafusion_common::Statistics;use datafusion_common::config::{ConfigField, ConfigFileType, TableParquetOptions};@@ -388,6 +391,17 @@ impl FileFormat for ParquetFormat {schemas.sort_unstable_by(|(location1, _), (location2, _)| location1.cmp(location2));+ let mut seen = HashSet::new();+ for (location, schema) in &schemas {+ ensure_unique_field_names(schema, &mut seen).map_err(|err| {+ DataFusionError::Context(+ format!("Error when processing Parquet file {location}"),+ Box::new(err),+ )+ })?;+ }+ drop(seen);+let schemas = schemas.into_iter().map(|(_, schema)| schema);let schema = if self.skip_metadata() {diff --git a/datafusion/datasource/src/file_format.rs b/datafusion/datasource/src/file_format.rsindex dd30881610..2c2a94e8cc 100644--- a/datafusion/datasource/src/file_format.rs+++ b/datafusion/datasource/src/file_format.rs@@ -19,7 +19,7 @@//! See write.rs for write related helper methodsuse std::any::Any;-use std::collections::HashMap;+use std::collections::{HashMap, HashSet};use std::fmt;use std::sync::Arc;@@ -28,9 +28,11 @@ use crate::file_compression_type::FileCompressionType;use crate::file_scan_config::FileScanConfig;use crate::file_sink_config::FileSinkConfig;-use arrow::datatypes::SchemaRef;+use arrow::datatypes::{Schema, SchemaRef};use datafusion_common::file_options::file_type::FileType;-use datafusion_common::{GetExt, Result, Statistics, internal_err, not_impl_err};+use datafusion_common::{+ GetExt, Result, SchemaError, Statistics, internal_err, not_impl_err, schema_err,+};use datafusion_physical_expr::LexRequirement;use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::ExecutionPlan;@@ -42,6 +44,27 @@ use object_store::{ObjectMeta, ObjectStore};/// Default max records to scan to infer the schemapub const DEFAULT_SCHEMA_INFER_MAX_RECORD: usize = 1000;+/// Rejects an inferred schema that names the same field more than once.+///+/// [`Schema::try_merge`] coalesces fields by name, so callers validate each+/// inferred file schema before merging. `seen` is cleared before use so callers+/// can reuse its allocation across schemas.+pub fn ensure_unique_field_names<'a>(+ schema: &'a Schema,+ seen: &mut HashSet<&'a str>,+) -> Result<()> {+ seen.clear();+ seen.reserve(schema.fields().len());+ for field in schema.fields() {+ if !seen.insert(field.name()) {+ return schema_err!(SchemaError::DuplicateUnqualifiedField {+ name: field.name().clone(),+ });+ }+ }+ Ok(())+}+/// Metadata fetched from a file, including statistics and ordering.////// This struct is returned by [`FileFormat::infer_stats_and_ordering`] todiff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.sltindex af2c6d41af..a64f1df6e4 100644--- a/datafusion/sqllogictest/test_files/csv_files.slt+++ b/datafusion/sqllogictest/test_files/csv_files.slt@@ -15,6 +15,13 @@# specific language governing permissions and limitations# under the License.+# duplicate_header_is_rejected+statement error Schema contains duplicate unqualified field name value+CREATE EXTERNAL TABLE csv_with_duplicate_header+STORED AS CSV+LOCATION '../core/tests/data/duplicate_header.csv'+OPTIONS ('format.has_header' 'true');+# create_external_table_with_quote_escapestatement okCREATE EXTERNAL TABLE csv_with_quote (