Skip to content

Apache DataFusion SQL Query Engine

feat: add match_columns_by_header option for CSV (#27021)

Wyatt Herkamp3f837ad0db6f
diff --git a/Cargo.lock b/Cargo.lock
index 731f49b7d2..c55c3e5647 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2038,16 +2038,19 @@ dependencies = [
"arrow",
"async-trait",
"bytes",
+ "csv-core",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-proto-models",
"datafusion-session",
"futures",
+ "log",
"object_store",
"regex",
"tokio",
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs
index 73821acdcf..6f33782ef9 100644
--- a/datafusion/common/src/config.rs
+++ b/datafusion/common/src/config.rs
@@ -4134,6 +4134,17 @@ config_namespace! {
/// of columns, it creates a union schema containing all columns found across the files, and will
/// pad any files missing columns with null values for their rows.
pub truncated_rows: Option<bool>, default = None
+ /// Whether to map file columns to the table schema by header name instead of by position.
+ ///
+ /// Only applies when `has_header` is true. When enabled, each file's header row is read
+ /// and columns are matched by name: table columns missing from a file are filled with
+ /// nulls, and file columns not declared in the table are ignored. Names are matched exactly
+ /// (case-sensitive) and duplicate header names resolve to the first occurrence. Header
+ /// names containing quoted newlines are not supported. The header is located the same
+ /// way the CSV parser does: a leading UTF-8 BOM, blank lines, and comment lines are skipped.
+ ///
+ /// Defaults to `false` when `None`.
+ pub match_columns_by_header: Option<bool>, default = None
}
}
@@ -4259,6 +4270,13 @@ impl CsvOptions {
self
}
+ /// Whether to map file columns to the table schema by header name instead of by position.
+ /// Only applies when the file has a header.
+ pub fn with_match_columns_by_header(mut self, match_by_header: bool) -> Self {
+ self.match_columns_by_header = Some(match_by_header);
+ self
+ }
+
/// Set the compression level for the output file.
/// The valid range depends on the compression algorithm.
/// If not specified, the default level for the algorithm is used.
diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs
index 297f33cda4..72e737ba78 100644
--- a/datafusion/core/src/datasource/file_format/options.rs
+++ b/datafusion/core/src/datasource/file_format/options.rs
@@ -86,6 +86,12 @@ pub struct CsvReadOptions<'a> {
/// When set to true then it will allow records with less than the expected number of columns and fill the missing columns with nulls.
/// If the record’s schema is not nullable, then it will still return an error.
pub truncated_rows: bool,
+ /// Whether to map file columns to the table schema by header name instead of by position.
+ ///
+ /// Only applies when `has_header` is true. Table columns missing from a file are
+ /// filled with nulls and file columns not declared in the table are ignored.
+ /// Defaults to `false`.
+ pub match_columns_by_header: bool,
}
impl Default for CsvReadOptions<'_> {
@@ -113,6 +119,7 @@ impl<'a> CsvReadOptions<'a> {
comment: None,
null_regex: None,
truncated_rows: false,
+ match_columns_by_header: false,
}
}
@@ -228,6 +235,13 @@ impl<'a> CsvReadOptions<'a> {
self.truncated_rows = truncated_rows;
self
}
+
+ /// Configure whether file columns are matched to the table schema by header name
+ /// instead of by position. Only applies when `has_header` is true.
+ pub fn match_columns_by_header(mut self, match_columns_by_header: bool) -> Self {
+ self.match_columns_by_header = match_columns_by_header;
+ self
+ }
}
/// Options that control the reading of Parquet files.
@@ -643,7 +657,8 @@ impl ReadOptions<'_> for CsvReadOptions<'_> {
.with_schema_infer_max_rec(self.schema_infer_max_records)
.with_file_compression_type(self.file_compression_type.to_owned())
.with_null_regex(self.null_regex.clone())
- .with_truncated_rows(self.truncated_rows);
+ .with_truncated_rows(self.truncated_rows)
+ .with_match_columns_by_header(self.match_columns_by_header);
ListingOptions::new(Arc::new(file_format))
.with_file_extension(self.file_extension)
diff --git a/datafusion/core/src/datasource/physical_plan/csv.rs b/datafusion/core/src/datasource/physical_plan/csv.rs
index 004e1d60cb..aee03fc6f2 100644
--- a/datafusion/core/src/datasource/physical_plan/csv.rs
+++ b/datafusion/core/src/datasource/physical_plan/csv.rs
@@ -27,7 +27,7 @@ mod tests {
use datafusion_common::config::CsvOptions;
use datafusion_common::test_util::arrow_test_data;
use datafusion_common::test_util::batches_to_string;
- use datafusion_common::{Result, assert_batches_eq};
+ use datafusion_common::{Result, assert_batches_eq, assert_batches_sorted_eq};
use datafusion_execution::config::SessionConfig;
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::metrics::MetricsSet;
@@ -802,4 +802,512 @@ mod tests {
Arc::new(schema)
}
+
+ /// Files sharing the table `(a INT, b TEXT, c INT, d TEXT)` but written
+ /// with different header layouts
+ const HEADER_LAYOUT_FILES: [(&str, &str); 3] = [
+ ("full.csv", "a,b,c,d\n1,x,10,p\n4,y,40,s\n"),
+ ("missing_middle.csv", "a,d\n2,q\n5,t\n"),
+ ("reordered.csv", "d,extra,c,a\nr,ignored,30,3\n"),
+ ];
+
+ #[derive(Debug, Clone, Copy)]
+ enum HeaderLayoutRead {
+ Local,
+ /// Tiny object store chunks so the header spans several chunks
+ Chunked,
+ /// Byte-range partitions that start mid-file
+ Repartitioned,
+ }
+
+ fn write_header_layout_files(dir: &std::path::Path) -> Result<()> {
+ for (name, contents) in HEADER_LAYOUT_FILES {
+ fs::write(dir.join(name), contents)?;
+ }
+ Ok(())
+ }
+
+ fn header_layout_ctx(read: HeaderLayoutRead) -> SessionContext {
+ let ctx = match read {
+ HeaderLayoutRead::Local | HeaderLayoutRead::Chunked => SessionContext::new(),
+ HeaderLayoutRead::Repartitioned => SessionContext::new_with_config(
+ SessionConfig::new()
+ .with_target_partitions(4)
+ .with_repartition_file_min_size(1),
+ ),
+ };
+ if matches!(read, HeaderLayoutRead::Chunked) {
+ ctx.register_object_store(
+ &Url::parse("file://").unwrap(),
+ Arc::new(ChunkedStore::new(Arc::new(LocalFileSystem::new()), 3)),
+ );
+ }
+ ctx
+ }
+
+ async fn header_layout_table(
+ ctx: &SessionContext,
+ location: &str,
+ options: &str,
+ ) -> Result<()> {
+ ctx.sql(&format!(
+ "CREATE EXTERNAL TABLE t (a INT, b TEXT, c INT, d TEXT) \
+ STORED AS CSV LOCATION '{location}' \
+ OPTIONS ('format.has_header' 'true'{options})"
+ ))
+ .await?
+ .collect()
+ .await?;
+ Ok(())
+ }
+
+ #[rstest]
+ #[case::local(HeaderLayoutRead::Local)]
+ #[case::chunked(HeaderLayoutRead::Chunked)]
+ #[case::repartitioned(HeaderLayoutRead::Repartitioned)]
+ #[tokio::test]
+ async fn csv_match_columns_by_header(#[case] read: HeaderLayoutRead) -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ write_header_layout_files(tmp_dir.path())?;
+
+ let ctx = header_layout_ctx(read);
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.match_columns_by_header' 'true'",
+ )
+ .await?;
+
+ let plan = ctx
+ .sql("SELECT * FROM t")
+ .await?
+ .create_physical_plan()
+ .await?;
+ let plan = datafusion_physical_plan::displayable(plan.as_ref())
+ .indent(true)
+ .to_string();
+ assert!(plan.contains("match_columns_by_header=true"), "{plan}");
+ if matches!(read, HeaderLayoutRead::Repartitioned) {
+ assert!(plan.contains("missing_middle.csv:"), "{plan}");
+ }
+
+ let all = ctx
+ .sql("SELECT * FROM t ORDER BY a")
+ .await?
+ .collect()
+ .await?;
+ let projected = ctx
+ .sql("SELECT d, c FROM t ORDER BY d")
+ .await?
+ .collect()
+ .await?;
+ let missing_only = ctx
+ .sql("SELECT count(b), count(*) FROM t")
+ .await?
+ .collect()
+ .await?;
+
+ insta::allow_duplicates! {
+ assert_snapshot!(batches_to_string(&all), @r"
+ +---+---+----+---+
+ | a | b | c | d |
+ +---+---+----+---+
+ | 1 | x | 10 | p |
+ | 2 | | | q |
+ | 3 | | 30 | r |
+ | 4 | y | 40 | s |
+ | 5 | | | t |
+ +---+---+----+---+
+ ");
+ assert_snapshot!(batches_to_string(&projected), @r"
+ +---+----+
+ | d | c |
+ +---+----+
+ | p | 10 |
+ | q | |
+ | r | 30 |
+ | s | 40 |
+ | t | |
+ +---+----+
+ ");
+ assert_snapshot!(batches_to_string(&missing_only), @r"
+ +------------+----------+
+ | count(t.b) | count(*) |
+ +------------+----------+
+ | 2 | 5 |
+ +------------+----------+
+ ");
+ }
+ Ok(())
+ }
+
+ #[rstest]
+ #[case::local(HeaderLayoutRead::Local)]
+ #[case::chunked(HeaderLayoutRead::Chunked)]
+ #[case::repartitioned(HeaderLayoutRead::Repartitioned)]
+ #[tokio::test]
+ async fn csv_match_columns_by_header_with_comments(
+ #[case] read: HeaderLayoutRead,
+ ) -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ fs::write(
+ tmp_dir.path().join("a.csv"),
+ "# generated\n# by test\nd,c,a\nr,30,3\n# trailing\ns,40,4\n",
+ )?;
+
+ let ctx = header_layout_ctx(read);
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.match_columns_by_header' 'true', 'format.comment' '#'",
+ )
+ .await?;
+
+ let batches = ctx
+ .sql("SELECT * FROM t ORDER BY a")
+ .await?
+ .collect()
+ .await?;
+ insta::allow_duplicates! {
+ assert_snapshot!(batches_to_string(&batches), @r"
+ +---+---+----+---+
+ | a | b | c | d |
+ +---+---+----+---+
+ | 3 | | 30 | r |
+ | 4 | | 40 | s |
+ +---+---+----+---+
+ ");
+ }
+ Ok(())
+ }
+
+ /// The header is found the way the CSV parser finds it: after a UTF-8
+ /// BOM, comment lines and blank lines
+ #[rstest]
+ #[case::local(HeaderLayoutRead::Local)]
+ #[case::chunked(HeaderLayoutRead::Chunked)]
+ #[case::repartitioned(HeaderLayoutRead::Repartitioned)]
+ #[tokio::test]
+ async fn csv_match_columns_by_header_with_bom_and_blank_lines(
+ #[case] read: HeaderLayoutRead,
+ ) -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ fs::write(
+ tmp_dir.path().join("a.csv"),
+ "\u{feff}# generated\n\n\r\nd,c,a\nr,30,3\n\ns,40,4\n",
+ )?;
+
+ let ctx = header_layout_ctx(read);
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.match_columns_by_header' 'true', 'format.comment' '#'",
+ )
+ .await?;
+
+ let batches = ctx
+ .sql("SELECT * FROM t ORDER BY a")
+ .await?
+ .collect()
+ .await?;
+ insta::allow_duplicates! {
+ assert_snapshot!(batches_to_string(&batches), @r"
+ +---+---+----+---+
+ | a | b | c | d |
+ +---+---+----+---+
+ | 3 | | 30 | r |
+ | 4 | | 40 | s |
+ +---+---+----+---+
+ ");
+ }
+ Ok(())
+ }
+
+ /// Wherever a file is split into two byte ranges, exactly one of them
+ /// skips the header row
+ #[rstest]
+ #[case::bom("\u{feff}a,b\n1,x\n2,y\n")]
+ #[case::carriage_return_blank_line("\ra,b\n1,x\n2,y\n")]
+ #[case::blank_line("\na,b\n1,x\n2,y\n")]
+ #[case::comment("# c\na,b\n1,x\n2,y\n")]
+ #[case::no_prefix("a,b\n1,x\n2,y\n")]
+ #[tokio::test]
+ async fn csv_match_columns_by_header_with_any_range_split(
+ #[case] contents: &str,
+ ) -> Result<()> {
+ use datafusion_datasource::PartitionedFile;
+ use datafusion_datasource::file_groups::FileGroup;
+
+ let tmp_dir = TempDir::new()?;
+ let path = tmp_dir.path().join("a.csv");
+ fs::write(&path, contents)?;
+ let path = path.to_str().unwrap();
+ let size = contents.len() as u64;
+
+ let table_schema = TableSchema::from(&Arc::new(Schema::new(vec![
+ Field::new("b", DataType::Utf8, true),
+ Field::new("a", DataType::Int32, true),
+ ])));
+ let options = CsvOptions {
+ has_header: Some(true),
+ comment: Some(b'#'),
+ ..Default::default()
+ };
+ let source = Arc::new(
+ CsvSource::new(table_schema)
+ .with_csv_options(options)
+ .with_match_columns_by_header(true),
+ );
+
+ for split in 1..contents.len() as i64 {
+ let file_groups = [(0, split), (split, size as i64)]
+ .into_iter()
+ .map(|(start, end)| {
+ FileGroup::new(vec![
+ PartitionedFile::new(path, size).with_range(start, end),
+ ])
+ })
+ .collect();
+ let config = partitioned_csv_config(file_groups, source.clone())?;
+ let csv = DataSourceExec::from_data_source(config);
+ let batches =
+ datafusion_physical_plan::collect(csv, SessionContext::new().task_ctx())
+ .await?;
+ assert_batches_sorted_eq!(
+ [
+ "+---+---+",
+ "| b | a |",
+ "+---+---+",
+ "| x | 1 |",
+ "| y | 2 |",
+ "+---+---+",
+ ],
+ &batches
+ );
+ }
+ Ok(())
+ }
+
+ /// A terminator inside a quoted header field does not end the header
+ #[rstest]
+ #[case::local(HeaderLayoutRead::Local)]
+ #[case::chunked(HeaderLayoutRead::Chunked)]
+ #[tokio::test]
+ async fn csv_match_columns_by_header_with_quoted_terminator(
+ #[case] read: HeaderLayoutRead,
+ ) -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ fs::write(
+ tmp_dir.path().join("a.csv"),
+ "\"x;y\",c,a;\"p;q\",30,3;s,40,4;",
+ )?;
+
+ let ctx = header_layout_ctx(read);
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.match_columns_by_header' 'true', 'format.terminator' ';'",
+ )
+ .await?;
+
+ let batches = ctx
+ .sql("SELECT * FROM t ORDER BY a")
+ .await?
+ .collect()
+ .await?;
+ insta::allow_duplicates! {
+ assert_snapshot!(batches_to_string(&batches), @r"
+ +---+---+----+---+
+ | a | b | c | d |
+ +---+---+----+---+
+ | 3 | | 30 | |
+ | 4 | | 40 | |
+ +---+---+----+---+
+ ");
+ }
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn csv_match_columns_by_header_with_truncated_rows() -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ fs::write(tmp_dir.path().join("a.csv"), "d,a,c\nr,1,10\ns,2\n")?;
+
+ let ctx = SessionContext::new();
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.match_columns_by_header' 'true', 'format.truncated_rows' 'true'",
+ )
+ .await?;
+
+ let batches = ctx
+ .sql("SELECT * FROM t ORDER BY a")
+ .await?
+ .collect()
+ .await?;
+ assert_snapshot!(batches_to_string(&batches), @r"
+ +---+---+----+---+
+ | a | b | c | d |
+ +---+---+----+---+
+ | 1 | | 10 | r |
+ | 2 | | | s |
+ +---+---+----+---+
+ ");
+ Ok(())
+ }
+
+ /// A quoted empty header names a column that is not in the table, so it is
+ /// not read positionally into `a`
+ #[rstest]
+ #[case::local(HeaderLayoutRead::Local)]
+ #[case::chunked(HeaderLayoutRead::Chunked)]
+ #[tokio::test]
+ async fn csv_match_columns_by_header_quoted_empty_header(
+ #[case] read: HeaderLayoutRead,
+ ) -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ fs::write(tmp_dir.path().join("a.csv"), "\"\"\n42\n")?;
+
+ let ctx = header_layout_ctx(read);
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.match_columns_by_header' 'true'",
+ )
+ .await?;
+
+ let batches = ctx.sql("SELECT * FROM t").await?.collect().await?;
+ insta::allow_duplicates! {
+ assert_snapshot!(batches_to_string(&batches), @r"
+ +---+---+---+---+
+ | a | b | c | d |
+ +---+---+---+---+
+ | | | | |
+ +---+---+---+---+
+ ");
+ }
+ Ok(())
+ }
+
+ #[rstest]
+ #[case::local(HeaderLayoutRead::Local)]
+ #[case::chunked(HeaderLayoutRead::Chunked)]
+ #[tokio::test]
+ async fn csv_match_columns_by_header_comment_only_file(
+ #[case] read: HeaderLayoutRead,
+ ) -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ fs::write(tmp_dir.path().join("a.csv"), "# only\n# comments\n")?;
+
+ let ctx = header_layout_ctx(read);
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.match_columns_by_header' 'true', 'format.comment' '#'",
+ )
+ .await?;
+
+ let batches = ctx.sql("SELECT * FROM t").await?.collect().await?;
+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 0);
+ Ok(())
+ }
+
+ /// The first occurrence of a duplicated header name is read and the
+ /// second is never parsed
+ #[rstest]
+ #[case::local(HeaderLayoutRead::Local)]
+ #[case::chunked(HeaderLayoutRead::Chunked)]
+ #[tokio::test]
+ async fn csv_match_columns_by_header_duplicate_header(
+ #[case] read: HeaderLayoutRead,
+ ) -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ fs::write(tmp_dir.path().join("a.csv"), "a,a\n1,not_an_int\n")?;
+
+ let ctx = header_layout_ctx(read);
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.match_columns_by_header' 'true'",
+ )
+ .await?;
+
+ let batches = ctx.sql("SELECT * FROM t").await?.collect().await?;
+ insta::allow_duplicates! {
+ assert_snapshot!(batches_to_string(&batches), @r"
+ +---+---+---+---+
+ | a | b | c | d |
+ +---+---+---+---+
+ | 1 | | | |
+ +---+---+---+---+
+ ");
+ }
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn csv_columns_mapped_by_position_by_default() -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ fs::write(tmp_dir.path().join("a.csv"), "a,d\n2,q\n")?;
+
+ let ctx = SessionContext::new();
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.truncated_rows' 'true'",
+ )
+ .await?;
+
+ let batches = ctx.sql("SELECT * FROM t").await?.collect().await?;
+ assert_snapshot!(batches_to_string(&batches), @r"
+ +---+---+---+---+
+ | a | b | c | d |
+ +---+---+---+---+
+ | 2 | q | | |
+ +---+---+---+---+
+ ");
+ Ok(())
+ }
+
+ #[cfg(feature = "compression")]
+ #[tokio::test]
+ async fn csv_match_columns_by_header_gzip() -> Result<()> {
+ use flate2::Compression;
+ use flate2::write::GzEncoder;
+
+ let tmp_dir = TempDir::new()?;
+ for (name, contents) in HEADER_LAYOUT_FILES {
+ let file = File::create(tmp_dir.path().join(format!("{name}.gz")))?;
+ let mut encoder = GzEncoder::new(file, Compression::default());
+ encoder.write_all(contents.as_bytes())?;
+ encoder.finish()?;
+ }
+
+ let ctx = SessionContext::new();
+ header_layout_table(
+ &ctx,
+ &format!("{}/", tmp_dir.path().display()),
+ ", 'format.match_columns_by_header' 'true', 'format.compression' 'gzip'",
+ )
+ .await?;
+
+ let batches = ctx
+ .sql("SELECT * FROM t ORDER BY a")
+ .await?
+ .collect()
+ .await?;
+ assert_snapshot!(batches_to_string(&batches), @r"
+ +---+---+----+---+
+ | a | b | c | d |
+ +---+---+----+---+
+ | 1 | x | 10 | p |
+ | 2 | | | q |
+ | 3 | | 30 | r |
+ | 4 | y | 40 | s |
+ | 5 | | | t |
+ +---+---+----+---+
+ ");
+ Ok(())
+ }
}
diff --git a/datafusion/core/tests/datasource/csv.rs b/datafusion/core/tests/datasource/csv.rs
index 93cf4ae6a3..46a00b924b 100644
--- a/datafusion/core/tests/datasource/csv.rs
+++ b/datafusion/core/tests/datasource/csv.rs
@@ -4,6 +4,7 @@
//! Test for CSV schema inference with different column counts (GitHub issue #17516)
+use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::error::Result;
use datafusion::prelude::*;
use datafusion_common::test_util::batches_to_sort_string;
@@ -107,3 +108,40 @@ async fn test_csv_schema_inference_different_column_counts() -> Result<()> {
Ok(())
}
+
+#[tokio::test]
+async fn test_csv_read_options_match_columns_by_header() -> Result<()> {
+ let temp_dir = TempDir::new()?;
+ fs::write(temp_dir.path().join("full.csv"), "a,b,c\n1,x,10\n")?;
+ fs::write(
+ temp_dir.path().join("reordered.csv"),
+ "c,extra,a\n20,ignored,2\n",
+ )?;
+
+ let schema = Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Utf8, true),
+ Field::new("c", DataType::Int32, true),
+ ]);
+ let ctx = SessionContext::new();
+ let batches = ctx
+ .read_csv(
+ temp_dir.path().to_str().unwrap(),
+ CsvReadOptions::new()
+ .schema(&schema)
+ .match_columns_by_header(true),
+ )
+ .await?
+ .collect()
+ .await?;
+
+ assert_snapshot!(batches_to_sort_string(&batches), @r"
+ +---+---+----+
+ | a | b | c |
+ +---+---+----+
+ | 1 | x | 10 |
+ | 2 | | 20 |
+ +---+---+----+
+ ");
+ Ok(())
+}
diff --git a/datafusion/datasource-csv/Cargo.toml b/datafusion/datasource-csv/Cargo.toml
index be178436fb..ea4ffbe5bd 100644
--- a/datafusion/datasource-csv/Cargo.toml
+++ b/datafusion/datasource-csv/Cargo.toml
@@ -29,16 +29,19 @@ proto = [
arrow = { workspace = true }
async-trait = { workspace = true }
bytes = { workspace = true }
+csv-core = "0.1"
datafusion-common = { workspace = true, features = ["object_store"] }
datafusion-common-runtime = { workspace = true }
datafusion-datasource = { workspace = true }
datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
+datafusion-physical-expr-adapter = { workspace = true }
datafusion-physical-expr-common = { workspace = true }
datafusion-physical-plan = { workspace = true }
datafusion-proto-models = { workspace = true, optional = true }
datafusion-session = { workspace = true }
futures = { workspace = true }
+log = { workspace = true }
object_store = { workspace = true }
regex = { workspace = true }
tokio = { workspace = true }
diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs
index 578694c18a..07a75591a3 100644
--- a/datafusion/datasource-csv/src/file_format.rs
+++ b/datafusion/datasource-csv/src/file_format.rs
@@ -293,6 +293,14 @@ impl CsvFormat {
self
}
+ /// Set whether file columns are matched to the table schema by header name
+ /// instead of by position
+ /// - defaults to false
+ pub fn with_match_columns_by_header(mut self, match_by_header: bool) -> Self {
+ self.options.match_columns_by_header = Some(match_by_header);
+ self
+ }
+
/// The delimiter character.
pub fn delimiter(&self) -> u8 {
self.options.delimiter
@@ -998,6 +1006,9 @@ impl From<&CsvFormatFactory> for datafusion_proto_models::protobuf::CsvOptions {
ignore_trailing_whitespace: options
.ignore_trailing_whitespace
.map_or(vec![], |v| vec![v as u8]),
+ match_columns_by_header: options
+ .match_columns_by_header
+ .map_or(vec![], |v| vec![v as u8]),
}
} else {
datafusion_proto_models::protobuf::CsvOptions::default()
diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs
index cbc8df820f..81b43667d7 100644
--- a/datafusion/datasource-csv/src/source.rs
+++ b/datafusion/datasource-csv/src/source.rs
@@ -9,6 +9,7 @@ use datafusion_datasource::projection::{ProjectionOpener, SplitProjection};
use datafusion_physical_plan::projection::ProjectionExprs;
use std::fmt;
use std::io::Read;
+
use std::sync::Arc;
use datafusion_datasource::decoder::{DecoderDeserializer, deserialize_stream};
@@ -18,7 +19,11 @@ use datafusion_datasource::{
FileRange, ListingTableUrl, PartitionedFile, TableSchema, as_file_source,
};
+use arrow::array::RecordBatch;
use arrow::csv;
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use bytes::Bytes;
+use csv_core::ReadRecordResult;
use datafusion_common::config::CsvOptions;
use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, Result, exec_datafusion_err};
@@ -26,15 +31,21 @@ use datafusion_common_runtime::JoinSet;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
use datafusion_execution::TaskContext;
+use datafusion_physical_expr_adapter::{
+ BatchAdapter, BatchAdapterFactory, DefaultPhysicalExprAdapterFactory,
+ PhysicalExprAdapterFactory,
+};
use datafusion_physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet};
use datafusion_physical_plan::{
DisplayFormatType, ExecutionPlan, ExecutionPlanProperties,
};
use crate::file_format::CsvDecoder;
+use futures::stream::BoxStream;
use futures::{StreamExt, TryStreamExt};
use object_store::buffered::BufWriter;
-use object_store::{GetOptions, GetResultPayload, ObjectStore};
+use object_store::path::Path;
+use object_store::{GetOptions, GetResultPayload, ObjectStore, ObjectStoreExt};
use regex::Regex;
use tokio::io::AsyncWriteExt;
@@ -174,28 +185,53 @@ impl CsvSource {
pub fn newlines_in_values(&self) -> bool {
self.options.newlines_in_values.unwrap_or(false)
}
+
+ /// true if file columns are matched to the table schema by header name
+ pub fn match_columns_by_header(&self) -> bool {
+ self.options.match_columns_by_header.unwrap_or(false) && self.has_header()
+ }
+
+ /// Whether to match file columns to the table schema by header name
+ pub fn with_match_columns_by_header(&self, match_by_header: bool) -> Self {
+ let mut conf = self.clone();
+ conf.options.match_columns_by_header = Some(match_by_header);
+ conf
+ }
}
impl CsvSource {
- fn open<R: Read>(&self, reader: R) -> Result<csv::Reader<R>> {
- Ok(self.builder()?.build(reader)?)
+ fn open<R: Read>(
+ &self,
+ reader: R,
+ mapping: Option<&HeaderMapping>,
+ ) -> Result<csv::Reader<R>> {
+ Ok(self.builder(mapping)?.build(reader)?)
}
- fn builder(&self) -> Result<csv::ReaderBuilder> {
- let mut builder =
- csv::ReaderBuilder::new(Arc::clone(self.table_schema.file_schema()))
- .with_delimiter(self.delimiter())
- .with_batch_size(
- self.batch_size
- .expect("Batch size must be set before initializing builder"),
- )
- .with_header(self.has_header())
- .with_quote(self.quote())
- .with_truncated_rows(self.truncate_rows());
+ fn builder(&self, mapping: Option<&HeaderMapping>) -> Result<csv::ReaderBuilder> {
+ let (schema, projection) = match mapping {
+ Some(mapping) => (
+ Arc::clone(&mapping.physical_schema),
+ mapping.physical_projection.clone(),
+ ),
+ None => (
+ Arc::clone(self.table_schema.file_schema()),
+ self.projection.file_indices.clone(),
+ ),
+ };
+ let mut builder = csv::ReaderBuilder::new(schema)
+ .with_delimiter(self.delimiter())
+ .with_batch_size(
+ self.batch_size
+ .expect("Batch size must be set before initializing builder"),
+ )
+ .with_header(self.has_header())
+ .with_quote(self.quote())
+ .with_truncated_rows(self.truncate_rows());
if let Some(terminator) = self.terminator() {
builder = builder.with_terminator(terminator);
}
- builder = builder.with_projection(self.projection.file_indices.clone());
+ builder = builder.with_projection(projection);
if let Some(escape) = self.escape() {
builder = builder.with_escape(escape)
}
@@ -211,6 +247,307 @@ impl CsvSource {
Ok(builder)
}
+
+ /// A parser configured like the one arrow's CSV decoder builds, so the
+ /// header is read with the same quoting, escaping, BOM, blank line and
+ /// comment handling as the rows that follow it.
+ fn header_parser(&self) -> csv_core::Reader {
+ let mut builder = csv_core::ReaderBuilder::new();
+ builder
+ .delimiter(self.delimiter())
+ .quote(self.quote())
+ .escape(self.escape())
+ .comment(self.comment());
+ if let Some(terminator) = self.terminator() {
+ builder.terminator(csv_core::Terminator::Any(terminator));
+ }
+ builder.build()
+ }
+
+ /// Parses the header row at the start of `buf`. Returns `None` when the
+ /// header is not yet complete and more input may follow.
+ fn find_header(&self, buf: &[u8], eof: bool) -> Result<Option<HeaderRow>> {
+ // csv-core treats empty input as the end of the file, including what
+ // is left after it strips a BOM, so it must only see empty input at
+ // EOF.
+ if !eof && buf.strip_prefix(UTF8_BOM).unwrap_or(buf).is_empty() {
+ return Ok(None);
+ }
+ let mut parser = self.header_parser();
+ let mut input = buf;
+ let mut output = vec![0; 1024];
+ let mut ends = vec![0; 32];
+ let (mut output_len, mut ends_len) = (0, 0);
+ loop {
+ if !eof && input.is_empty() {
+ return Ok(None);
+ }
+ let (result, read, written, ends_written) = parser.read_record(
+ input,
+ &mut output[output_len..],
+ &mut ends[ends_len..],
+ );
+ input = &input[read..];
+ output_len += written;
+ ends_len += ends_written;
+ match result {
+ ReadRecordResult::InputEmpty if !eof => return Ok(None),
+ ReadRecordResult::InputEmpty => {}
+ ReadRecordResult::OutputFull => output.resize(output.len() * 2, 0),
+ ReadRecordResult::OutputEndsFull => ends.resize(ends.len() * 2, 0),
+ ReadRecordResult::Record => break,
+ ReadRecordResult::End => {
+ return Ok(Some(HeaderRow {
+ names: vec![],
+ start: buf.len(),
+ }));
+ }
+ }
+ }
+
+ // csv-core reads an unterminated comment at the end of the input as a
+ // record with one empty field, which is indistinguishable from a
+ // quoted empty header by its fields alone
+ let start = self.first_record_start(buf);
+ if start == buf.len() {
+ return Ok(Some(HeaderRow {
+ names: vec![],
+ start,
+ }));
+ }
+
+ let mut field_start = 0;
+ let names = ends[..ends_len]
+ .iter()
+ .map(|&field_end| {
+ let name = std::str::from_utf8(&output[field_start..field_end]).map_err(
+ |e| exec_datafusion_err!("Invalid UTF-8 in CSV header: {e}"),
+ )?;
+ field_start = field_end;
+ Ok(name.to_string())
+ })
+ .collect::<Result<_>>()?;
+ Ok(Some(HeaderRow { names, start }))
+ }
+
+ /// Byte offset of the first record in `buf`, after a leading UTF-8 BOM,
+ /// blank records and comment lines. None of these can contain quotes, so
+ /// a plain byte scan finds the same position the parser starts reading
+ /// the header at. With no explicit terminator both `\r` and `\n` end a
+ /// record.
+ fn first_record_start(&self, buf: &[u8]) -> usize {
+ let terminator = self.terminator();
+ let comment = self.comment();
+ let is_terminator = |b: u8| match terminator {
+ Some(t) => b == t,
+ None => b == b'\n' || b == b'\r',
+ };
+ // csv-core's DFA only ends a comment at `\n` when `\n` is also a
+ // terminator; with any other terminator the comment runs to the end
+ // of the input.
+ let comment_ends_at_newline = matches!(terminator, None | Some(b'\n'));
+
+ let mut start = if buf.starts_with(UTF8_BOM) {
+ UTF8_BOM.len()
+ } else {
+ 0
+ };
+ while let Some(&first) = buf.get(start) {
+ if is_terminator(first) {
+ start += 1;
+ } else if comment == Some(first) {
+ let end = comment_ends_at_newline
+ .then(|| buf[start..].iter().position(|b| *b == b'\n'))
+ .flatten();
+ match end {
+ Some(end) => start += end + 1,
+ None => return buf.len(),
+ }
+ } else {
+ break;
+ }
+ }
+ start
+ }
+
+ /// Builds the mapping for a file with the given header, or `None` if the
+ /// file can be read positionally.
+ fn header_mapping(
+ &self,
+ header: &[String],
+ expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
+ ) -> Result<Option<Arc<HeaderMapping>>> {
+ if header.is_empty() {
+ return Ok(None);
+ }
+ HeaderMapping::try_new(
+ header,
+ self.table_schema.file_schema(),
+ &self.projection.file_indices,
+ expr_adapter_factory,
+ )
+ .map(|mapping| mapping.map(Arc::new))
+ }
+}
+
+/// Maps the columns of a single CSV file onto the table's file schema by
+/// header name.
+///
+/// Only the header columns that are declared in the table are decoded; the
+/// decoded batch is then adapted to the projected file schema with the scan's
+/// [`PhysicalExprAdapterFactory`], which is the same mechanism Parquet uses
+/// to fill missing columns and reorder by name.
+#[derive(Debug)]
+struct HeaderMapping {
+ /// One field per header column, typed from the table schema where the
+ /// name is declared there
+ physical_schema: SchemaRef,
+ /// Header positions to decode, in the order of the table projection
+ physical_projection: Vec<usize>,
+ adapter: BatchAdapter,
+}
+
+impl HeaderMapping {
+ fn try_new(
+ header: &[String],
+ file_schema: &Schema,
+ file_indices: &[usize],
+ expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
+ ) -> Result<Option<Self>> {
+ let matches_positionally = header.len() == file_schema.fields().len()
+ && header
+ .iter()
+ .zip(file_schema.fields())
+ .all(|(name, field)| name == field.name());
+ if matches_positionally {
+ return Ok(None);
+ }
+
+ let physical_fields: Vec<Field> = header
+ .iter()
+ .map(|name| match file_schema.field_with_name(name) {
+ Ok(field) => field.clone(),
+ Err(_) => Field::new(name, DataType::Utf8, true),
+ })
+ .collect();
+
+ let physical_projection: Vec<usize> = file_indices
+ .iter()
+ .filter_map(|&index| {
+ let name = file_schema.field(index).name();
+ header.iter().position(|column| column == name)
+ })
+ .collect();
+
+ let physical_schema = Arc::new(Schema::new(physical_fields));
+ let decoded_schema = Arc::new(physical_schema.project(&physical_projection)?);
+ let target_schema = Arc::new(file_schema.project(file_indices)?);
+ let adapter = BatchAdapterFactory::new(target_schema)
+ .with_adapter_factory(expr_adapter_factory)
+ .make_adapter(&decoded_schema)?;
+
+ Ok(Some(Self {
+ physical_schema,
+ physical_projection,
+ adapter,
+ }))
+ }
+
+ fn remap(&self, batch: &RecordBatch) -> Result<RecordBatch> {
+ self.adapter.adapt_batch(batch)
+ }
+}
+
+const UTF8_BOM: &[u8] = b"\xef\xbb\xbf";
+
+/// The header row of a CSV file
+#[derive(Debug, PartialEq)]
+struct HeaderRow {
+ names: Vec<String>,
+ /// Byte offset where the header record starts
+ start: usize,
+}
+
+/// Reads the header row of `input`, returning it together with a stream that
+/// still yields every byte of `input`.
+async fn peek_header(
+ config: &CsvSource,
+ mut input: BoxStream<'static, Result<Bytes>>,
+) -> Result<(HeaderRow, BoxStream<'static, Result<Bytes>>)> {
+ let mut prefix = Vec::new();
+ let mut buffered = Vec::new();
+ loop {
+ let chunk = input.next().await.transpose()?;
+ let eof = chunk.is_none();
+ if let Some(chunk) = chunk {
+ prefix.extend_from_slice(&chunk);
+ buffered.push(Ok(chunk));
+ }
+ if let Some(header) = config.find_header(&prefix, eof)? {
+ let buffered = futures::stream::iter(buffered);
+ let stream = if eof {
+ buffered.boxed()
+ } else {
+ buffered.chain(input).boxed()
+ };
+ return Ok((header, stream));
+ }
+ }
+}
+
+/// Reads the header row of an uncompressed file, growing the requested range
+/// until it is complete. Also returns the fetched prefix of the file, which
+/// reaches at least to the start of the header.
+async fn fetch_header(
+ config: &CsvSource,
+ store: &dyn ObjectStore,
+ location: &Path,
+ file_size: u64,
+) -> Result<(HeaderRow, Bytes)> {
+ let mut len = file_size.min(64 * 1024);
+ loop {
+ let bytes = store.get_range(location, 0..len).await?;
+ let eof = len >= file_size;
+ if let Some(header) = config.find_header(&bytes, eof)? {
+ return Ok((header, bytes));
+ }
+ len = (len * 2).min(file_size);
+ }
+}
+
+/// Whether the byte-range partition starting at `raw_start` reads the header
+/// row, given a `prefix` of the file that reaches `header_start`.
+///
+/// [`AlignedBoundaryStream`] starts a partition after the first `line_end` at
+/// or after `raw_start - 1`, so a partition starting before the header only
+/// reads it when such a terminator precedes the header. Otherwise the header's
+/// own terminator is the one skipped and the partition starts at the first
+/// data row.
+fn range_owns_header(
+ prefix: &[u8],
+ header_start: usize,
+ raw_start: u64,
+ line_end: u8,
+) -> bool {
+ if raw_start == 0 {
+ return true;
+ }
+ match usize::try_from(raw_start) {
+ Ok(raw_start) if raw_start <= header_start => {
+ prefix[raw_start - 1..header_start].contains(&line_end)
+ }
+ _ => false,
+ }
+}
+
+fn remap_stream(
+ stream: BoxStream<'static, Result<RecordBatch>>,
+ mapping: Option<Arc<HeaderMapping>>,
+) -> BoxStream<'static, Result<RecordBatch>> {
+ match mapping {
+ Some(mapping) => stream.map(move |batch| mapping.remap(&batch?)).boxed(),
+ None => stream,
+ }
}
/// A [`FileOpener`] that opens a CSV file and yields a [`FileOpenFuture`]
@@ -219,6 +556,7 @@ pub struct CsvOpener {
file_compression_type: FileCompressionType,
object_store: Arc<dyn ObjectStore>,
partition_index: usize,
+ expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
}
impl CsvOpener {
@@ -233,8 +571,19 @@ impl CsvOpener {
file_compression_type,
object_store,
partition_index: 0,
+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),
}
}
+
+ /// Set the [`PhysicalExprAdapterFactory`] used to adapt files whose
+ /// columns are matched to the table schema by header name
+ pub fn with_expr_adapter_factory(
+ mut self,
+ expr_adapter_factory: Arc<dyn PhysicalExprAdapterFactory>,
+ ) -> Self {
+ self.expr_adapter_factory = expr_adapter_factory;
+ self
+ }
}
impl From<CsvSource> for Arc<dyn FileSource> {
@@ -250,11 +599,21 @@ impl FileSource for CsvSource {
base_config: &FileScanConfig,
partition_index: usize,
) -> Result<Arc<dyn FileOpener>> {
+ if self.options.match_columns_by_header == Some(true) && !self.has_header() {
+ log::warn!(
+ "CSV option match_columns_by_header is ignored because has_header is false; columns are matched by position"
+ );
+ }
+ let expr_adapter_factory = base_config
+ .expr_adapter_factory
+ .clone()
+ .unwrap_or_else(|| Arc::new(DefaultPhysicalExprAdapterFactory) as _);
let mut opener = Arc::new(CsvOpener {
config: Arc::new(self.clone()),
file_compression_type: base_config.file_compression_type,
object_store,
partition_index,
+ expr_adapter_factory,
}) as Arc<dyn FileOpener>;
opener = ProjectionOpener::try_new(
self.projection.clone(),
@@ -307,7 +666,11 @@ impl FileSource for CsvSource {
fn fmt_extra(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
match t {
DisplayFormatType::Default | DisplayFormatType::Verbose => {
- write!(f, ", has_header={}", self.has_header())
+ write!(f, ", has_header={}", self.has_header())?;
+ if self.match_columns_by_header() {
+ write!(f, ", match_columns_by_header=true")?;
+ }
+ Ok(())
}
DisplayFormatType::TreeRender => Ok(()),
}
@@ -360,6 +723,7 @@ impl FileSource for CsvSource {
newlines_in_values: self.options.newlines_in_values.unwrap_or(false),
truncate_rows: self.truncate_rows(),
terminator: self.terminator().map(|terminator| vec![terminator]),
+ match_columns_by_header: self.match_columns_by_header(),
};
Ok(Some(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::CsvScan(node)),
@@ -395,6 +759,8 @@ impl FileOpener for CsvOpener {
// `self.config.has_header` controls whether to skip reading the 1st line header
// If the .csv file is read in parallel and this `CsvOpener` is only reading some middle
// partition, then don't skip first line
+ let match_by_header = self.config.match_columns_by_header();
+ let expr_adapter_factory = Arc::clone(&self.expr_adapter_factory);
let mut csv_has_header = self.config.has_header();
if let Some(FileRange { start, .. }) = partitioned_file.range
&& start != 0
@@ -416,7 +782,7 @@ impl FileOpener for CsvOpener {
}
let store = Arc::clone(&self.object_store);
- let terminator = self.config.terminator();
+ let line_end = self.config.terminator().unwrap_or(b'\n');
let baseline_metrics =
BaselineMetrics::new(&self.config.metrics, self.partition_index);
@@ -440,18 +806,38 @@ impl FileOpener for CsvOpener {
)
})?;
+ // A partition may start mid-file, so the header is fetched
+ // separately. Every byte-range partition of a file repeats this
+ // small request; cache per file if that becomes costly.
+ let mapping = if match_by_header {
+ let (header, prefix) =
+ fetch_header(&config, store.as_ref(), &location, file_size)
+ .await?;
+ // A partition starting within the lines that precede the
+ // header can own the header row, so it has to skip it.
+ if range_owns_header(&prefix, header.start, raw_start, line_end) {
+ config.options.has_header = Some(true);
+ }
+ config.header_mapping(
+ &header.names,
+ Arc::clone(&expr_adapter_factory),
+ )?
+ } else {
+ None
+ };
+
let aligned_stream = AlignedBoundaryStream::new(
Arc::clone(&store),
location.clone(),
raw_start,
raw_end,
file_size,
- terminator.unwrap_or(b'\n'),
+ line_end,
)
.await?
.map_err(DataFusionError::from);
- let decoder = config.builder()?.build_decoder();
+ let decoder = config.builder(mapping.as_deref())?.build_decoder();
let input = file_compression_type
.convert_stream(aligned_stream.boxed())?
.fuse();
@@ -459,7 +845,7 @@ impl FileOpener for CsvOpener {
input,
DecoderDeserializer::new(CsvDecoder::new(decoder)),
);
- return Ok(stream.map_err(Into::into).boxed());
+ return Ok(remap_stream(stream.map_err(Into::into).boxed(), mapping));
}
// No range specified — read the entire file
@@ -469,8 +855,28 @@ impl FileOpener for CsvOpener {
match result.payload {
#[cfg(not(target_arch = "wasm32"))]
GetResultPayload::File(file, _) => {
+ use std::io::{BufRead, BufReader, Cursor};
+
let decoder = file_compression_type.convert_read(file)?;
- let mut reader = config.open(decoder)?;
+ let (decoder, mapping): (Box<dyn Read + Send>, _) = if match_by_header
+ {
+ let mut decoder = BufReader::new(decoder);
+ let mut prefix = Vec::new();
+ let header = loop {
+ let eof = decoder.read_until(line_end, &mut prefix)? == 0;
+ if let Some(header) = config.find_header(&prefix, eof)? {
+ break header;
+ }
+ };
+ let mapping = config.header_mapping(
+ &header.names,
+ Arc::clone(&expr_adapter_factory),
+ )?;
+ (Box::new(Cursor::new(prefix).chain(decoder)), mapping)
+ } else {
+ (decoder, None)
+ };
+ let mut reader = config.open(decoder, mapping.as_deref())?;
// Use std::iter::from_fn to wrap execution of iterator's next() method.
let iterator = std::iter::from_fn(move || {
@@ -480,20 +886,30 @@ impl FileOpener for CsvOpener {
result
});
- Ok(futures::stream::iter(iterator)
+ let stream = futures::stream::iter(iterator)
.map(|r| r.map_err(Into::into))
- .boxed())
+ .boxed();
+ Ok(remap_stream(stream, mapping))
}
GetResultPayload::Stream(s) => {
- let decoder = config.builder()?.build_decoder();
let s = s.map_err(DataFusionError::from);
- let input = file_compression_type.convert_stream(s.boxed())?.fuse();
-
+ let mut input = file_compression_type.convert_stream(s.boxed())?;
+ let mut mapping = None;
+ if match_by_header {
+ let (header, rest) = peek_header(&config, input).await?;
+ mapping = config.header_mapping(
+ &header.names,
+ Arc::clone(&expr_adapter_factory),
+ )?;
+ input = rest;
+ }
+
+ let decoder = config.builder(mapping.as_deref())?.build_decoder();
let stream = deserialize_stream(
- input,
+ input.fuse(),
DecoderDeserializer::new(CsvDecoder::new(decoder)),
);
- Ok(stream.map_err(Into::into).boxed())
+ Ok(remap_stream(stream.map_err(Into::into).boxed(), mapping))
}
}
}))
@@ -519,7 +935,7 @@ pub async fn plan_to_csv(
let storeref = Arc::clone(&store);
let plan: Arc<dyn ExecutionPlan> = Arc::clone(&plan);
let filename = format!("{}/part-{i}.csv", parsed.prefix());
- let file = object_store::path::Path::parse(filename)?;
+ let file = Path::parse(filename)?;
let mut stream = plan.execute(i, Arc::clone(&task_ctx))?;
join_set.spawn(async move {
@@ -643,6 +1059,7 @@ impl CsvSource {
quote: proto_str_to_byte(&scan.quote, "quote")?,
newlines_in_values: Some(scan.newlines_in_values),
truncated_rows: Some(scan.truncate_rows),
+ match_columns_by_header: Some(scan.match_columns_by_header),
..Default::default()
};
let source = Arc::new(
@@ -685,7 +1102,7 @@ mod tests {
let data = "id,name\n1,alice\n2,N/A\n3,carol\n";
let batch = source
- .open(data.as_bytes())
+ .open(data.as_bytes(), None)
.unwrap()
.next()
.unwrap()
@@ -713,7 +1130,7 @@ mod tests {
let data = "id,value\n1,10\n2,N/A\n3,30\n";
let batch = source
- .open(data.as_bytes())
+ .open(data.as_bytes(), None)
.unwrap()
.next()
.unwrap()
@@ -739,7 +1156,7 @@ mod tests {
let data = "id,name\n1,alice\n2,N/A\n3,carol\n";
let batch = source
- .open(data.as_bytes())
+ .open(data.as_bytes(), None)
.unwrap()
.next()
.unwrap()
@@ -758,10 +1175,333 @@ mod tests {
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, true)]));
let source = csv_source(schema, Some("("));
- let err = source.open(b"id\n1\n".as_slice()).unwrap_err();
+ let err = source.open(b"id\n1\n".as_slice(), None).unwrap_err();
assert!(
err.to_string().contains("Unable to parse CSV null regex"),
"unexpected error: {err}"
);
}
+
+ use arrow::array::{Int32Array, RecordBatchOptions};
+
+ fn table_schema() -> Schema {
+ Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Utf8, true),
+ Field::new("c", DataType::Int32, true),
+ ])
+ }
+
+ fn header(names: &[&str]) -> Vec<String> {
+ names.iter().map(|name| name.to_string()).collect()
+ }
+
+ fn adapter_factory() -> Arc<dyn PhysicalExprAdapterFactory> {
+ Arc::new(DefaultPhysicalExprAdapterFactory)
+ }
+
+ #[test]
+ fn header_mapping_skipped_when_header_matches() -> Result<()> {
+ let mapping = HeaderMapping::try_new(
+ &header(&["a", "b", "c"]),
+ &table_schema(),
+ &[0, 2],
+ adapter_factory(),
+ )?;
+ assert!(mapping.is_none());
+ Ok(())
+ }
+
+ #[test]
+ fn header_mapping_by_name() -> Result<()> {
+ let mapping = HeaderMapping::try_new(
+ &header(&["c", "extra", "a"]),
+ &table_schema(),
+ &[0, 1, 2],
+ adapter_factory(),
+ )?
+ .unwrap();
+
+ let physical_types: Vec<_> = mapping
+ .physical_schema
+ .fields()
+ .iter()
+ .map(|f| f.data_type().clone())
+ .collect();
+ assert_eq!(
+ physical_types,
+ vec![DataType::Int32, DataType::Utf8, DataType::Int32]
+ );
+ assert_eq!(mapping.physical_projection, vec![2, 0]);
+
+ let decoded_schema = mapping.physical_schema.project(&[2, 0])?;
+ let decoded = RecordBatch::try_new(
+ Arc::new(decoded_schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2])),
+ Arc::new(Int32Array::from(vec![10, 20])),
+ ],
+ )?;
+ let remapped = mapping.remap(&decoded)?;
+ assert_eq!(remapped.schema().as_ref(), &table_schema());
+ assert_eq!(remapped.column(1).null_count(), 2);
+ assert_eq!(
+ remapped.column(0).as_ref(),
+ &Int32Array::from(vec![1, 2]) as &dyn Array
+ );
+ assert_eq!(
+ remapped.column(2).as_ref(),
+ &Int32Array::from(vec![10, 20]) as &dyn Array
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn header_mapping_only_missing_columns() -> Result<()> {
+ let mapping = HeaderMapping::try_new(
+ &header(&["a", "c"]),
+ &table_schema(),
+ &[1],
+ adapter_factory(),
+ )?
+ .unwrap();
+ assert!(mapping.physical_projection.is_empty());
+
+ let decoded = RecordBatch::try_new_with_options(
+ Arc::new(Schema::empty()),
+ vec![],
+ &RecordBatchOptions::new().with_row_count(Some(3)),
+ )?;
+ let remapped = mapping.remap(&decoded)?;
+ assert_eq!(remapped.num_rows(), 3);
+ assert_eq!(remapped.column(0).null_count(), 3);
+ Ok(())
+ }
+
+ #[test]
+ fn header_mapping_missing_non_nullable_column() {
+ let schema = Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Utf8, false),
+ ]);
+ let err =
+ HeaderMapping::try_new(&header(&["a"]), &schema, &[0, 1], adapter_factory())
+ .unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("Non-nullable column 'b' is missing from the physical schema"),
+ "{err}"
+ );
+ }
+
+ fn find_header(
+ buf: &[u8],
+ terminator: Option<u8>,
+ comment: Option<u8>,
+ eof: bool,
+ ) -> Option<HeaderRow> {
+ CsvSource::new(Arc::new(table_schema()))
+ .with_terminator(terminator)
+ .with_comment(comment)
+ .find_header(buf, eof)
+ .unwrap()
+ }
+
+ fn row(names: &[&str], start: usize) -> Option<HeaderRow> {
+ Some(HeaderRow {
+ names: header(names),
+ start,
+ })
+ }
+
+ #[test]
+ fn find_header_handles_quotes() {
+ let newline = Some(b'\n');
+ assert_eq!(
+ find_header(b"\"a,1\",b\r", None, None, true),
+ row(&["a,1", "b"], 0)
+ );
+ assert_eq!(
+ find_header(b"\"a;b\",c;1,2;", Some(b';'), None, false),
+ row(&["a;b", "c"], 0)
+ );
+ assert_eq!(
+ find_header(b"\"a\nb\",c\n1,2\n", newline, None, false),
+ row(&["a\nb", "c"], 0)
+ );
+ // A terminator inside an open quote needs more input
+ assert_eq!(find_header(b"\"a\nb", newline, None, false), None);
+ // A comment byte inside a quoted header is data
+ assert_eq!(
+ find_header(b"\"#a\",b\n", newline, Some(b'#'), false),
+ row(&["#a", "b"], 0)
+ );
+ // A quoted empty header is a real column, not an empty file
+ assert_eq!(
+ find_header(b"\"\"\n42\n", newline, Some(b'#'), false),
+ row(&[""], 0)
+ );
+ assert_eq!(
+ find_header(b"\"\"", newline, Some(b'#'), true),
+ row(&[""], 0)
+ );
+ }
+
+ #[test]
+ fn find_header_waits_for_input_before_eof() {
+ let newline = Some(b'\n');
+ assert_eq!(find_header(b"", newline, None, false), None);
+ assert_eq!(find_header(b"\xef\xbb\xbf", newline, None, false), None);
+ // A header longer than the initial output buffer, cut at the end of
+ // the buffer, is incomplete rather than ended
+ let long = "x".repeat(4096);
+ assert_eq!(find_header(long.as_bytes(), newline, None, false), None);
+ assert_eq!(
+ find_header(format!("{long}\n").as_bytes(), newline, None, false),
+ row(&[&long], 0)
+ );
+ }
+
+ #[test]
+ fn find_header_handles_escapes() -> Result<()> {
+ let header = CsvSource::new(Arc::new(table_schema()))
+ .with_escape(Some(b'\\'))
+ .find_header(b"\"a\\\"\nb\",c\n", false)?;
+ assert_eq!(header, row(&["a\"\nb", "c"], 0));
+ Ok(())
+ }
+
+ #[test]
+ fn find_header_rejects_invalid_utf8() {
+ let err = CsvSource::new(Arc::new(table_schema()))
+ .find_header(b"a\xff,b\n", false)
+ .unwrap_err();
+ assert!(
+ err.to_string().contains("Invalid UTF-8 in CSV header"),
+ "{err}"
+ );
+ }
+
+ #[test]
+ fn find_header_skips_comments() {
+ let comment = Some(b'#');
+ let newline = Some(b'\n');
+ assert_eq!(
+ find_header(b"# c1\n#c2\na,b\n1,2\n", newline, comment, false),
+ row(&["a", "b"], 9)
+ );
+ assert_eq!(
+ find_header(b"#c\na,b\n", newline, None, false),
+ row(&["#c"], 0)
+ );
+ assert_eq!(find_header(b"# c1\na,", newline, comment, false), None);
+ assert_eq!(
+ find_header(b"# c1\na,b", newline, comment, true),
+ row(&["a", "b"], 5)
+ );
+ assert_eq!(
+ find_header(b"# c1\n# c2", newline, comment, true),
+ row(&[], 9)
+ );
+ // `\r` does not end a comment
+ assert_eq!(
+ find_header(b"# c1\ra,b\nx,y\n", None, comment, false),
+ row(&["x", "y"], 9)
+ );
+ assert_eq!(find_header(b"# c1\n# c2", newline, comment, false), None);
+ assert_eq!(find_header(b"", newline, comment, true), row(&[], 0));
+ }
+
+ #[test]
+ fn find_header_skips_leading_blank_records() {
+ assert_eq!(
+ find_header(b"\n\na,b\n1,2\n", Some(b'\n'), None, false),
+ row(&["a", "b"], 2)
+ );
+ // Without an explicit terminator both `\r` and `\n` end a record
+ assert_eq!(
+ find_header(b"\r\n\n\ra,b\r\n1,2\r\n", None, None, false),
+ row(&["a", "b"], 4)
+ );
+ // A custom terminator keeps `\n` and `\r` as ordinary bytes
+ assert_eq!(
+ find_header(b";;a,b\n;1,2;", Some(b';'), None, false),
+ row(&["a", "b\n"], 2)
+ );
+ // Whitespace-only lines are records, not blank lines
+ assert_eq!(
+ find_header(b" \na,b\n", Some(b'\n'), None, false),
+ row(&[" "], 0)
+ );
+ assert_eq!(find_header(b"\n\n", Some(b'\n'), None, false), None);
+ assert_eq!(find_header(b"\n\n", Some(b'\n'), None, true), row(&[], 2));
+ }
+
+ #[test]
+ fn find_header_strips_bom() {
+ let comment = Some(b'#');
+ assert_eq!(
+ find_header(b"\xef\xbb\xbfa,b\n1,2\n", Some(b'\n'), comment, false),
+ row(&["a", "b"], 3)
+ );
+ assert_eq!(
+ find_header(b"\xef\xbb\xbf# c1\n\na,b\n", Some(b'\n'), comment, false),
+ row(&["a", "b"], 9)
+ );
+ // An incomplete BOM needs more input before it can be classified
+ assert_eq!(find_header(b"\xef\xbb", Some(b'\n'), comment, false), None);
+ // Only a BOM at the very start of the file is ignored
+ assert_eq!(
+ find_header(b"\n\xef\xbb\xbfa,b\n", Some(b'\n'), comment, false),
+ row(&["\u{feff}a", "b"], 1)
+ );
+ }
+
+ #[test]
+ fn find_header_comment_runs_to_end_with_custom_terminator() {
+ let buf = b"# c1;ignored\na,b;1,2;";
+ assert_eq!(find_header(buf, Some(b';'), Some(b'#'), false), None);
+ assert_eq!(
+ find_header(buf, Some(b';'), Some(b'#'), true),
+ row(&[], buf.len())
+ );
+ assert_eq!(
+ find_header(b"# c1\na,b\n", Some(b'\n'), Some(b'#'), false),
+ row(&["a", "b"], 5)
+ );
+ }
+
+ #[test]
+ fn range_owns_header_follows_stream_alignment() {
+ // No terminator between the BOM and the header
+ let bom = b"\xef\xbb\xbfa,b\n1,x\n";
+ assert!(range_owns_header(bom, 3, 0, b'\n'));
+ for raw_start in 1..=4 {
+ assert!(!range_owns_header(bom, 3, raw_start, b'\n'));
+ }
+
+ // A blank record ended by `\r` is not a boundary the stream aligns to
+ assert!(!range_owns_header(b"\ra,b\n1,x\n", 1, 1, b'\n'));
+ assert!(range_owns_header(b"\na,b\n1,x\n", 1, 1, b'\n'));
+
+ let comment = b"# c\na,b\n1,x\n";
+ for raw_start in 0..=4 {
+ assert!(range_owns_header(comment, 4, raw_start, b'\n'));
+ }
+ assert!(!range_owns_header(comment, 4, 5, b'\n'));
+
+ let custom = b";;a,b;1,x;";
+ assert!(range_owns_header(custom, 2, 1, b';'));
+ assert!(range_owns_header(custom, 2, 2, b';'));
+ assert!(!range_owns_header(custom, 2, 3, b';'));
+ assert!(!range_owns_header(custom, 2, 1, b'\n'));
+ }
+
+ #[test]
+ fn match_columns_by_header_requires_header() {
+ let source = CsvSource::new(Arc::new(table_schema()))
+ .with_csv_options(CsvOptions::default().with_has_header(false))
+ .with_match_columns_by_header(true);
+ assert!(!source.match_columns_by_header());
+ }
}
diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.proto
index 1cd523a38c..0bef47ec7d 100644
--- a/datafusion/proto-common/proto/datafusion_common.proto
+++ b/datafusion/proto-common/proto/datafusion_common.proto
@@ -493,6 +493,8 @@ message CsvOptions {
bytes ignore_leading_whitespace = 21;
// Whether to ignore trailing whitespace in string values
bytes ignore_trailing_whitespace = 22;
+ // Whether to map file columns to the table schema by header name
+ bytes match_columns_by_header = 23;
}
// Options controlling CSV format
diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs
index 5e858f95b2..b380b67e92 100644
--- a/datafusion/proto-common/src/from_proto/mod.rs
+++ b/datafusion/proto-common/src/from_proto/mod.rs
@@ -1163,6 +1163,10 @@ impl TryFrom<&protobuf::CsvOptions> for CsvOptions {
.ignore_trailing_whitespace
.first()
.map(|h| *h != 0),
+ match_columns_by_header: proto_opts
+ .match_columns_by_header
+ .first()
+ .map(|h| *h != 0),
})
}
}
diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rs
index 26f91ef294..9b45d54dcd 100644
--- a/datafusion/proto-common/src/generated/pbjson.rs
+++ b/datafusion/proto-common/src/generated/pbjson.rs
@@ -1755,6 +1755,9 @@ impl serde::Serialize for CsvOptions {
if !self.ignore_trailing_whitespace.is_empty() {
len += 1;
}
+ if !self.match_columns_by_header.is_empty() {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion_common.CsvOptions", len)?;
if !self.has_header.is_empty() {
#[allow(clippy::needless_borrow)]
@@ -1850,6 +1853,11 @@ impl serde::Serialize for CsvOptions {
#[allow(clippy::needless_borrows_for_generic_args)]
struct_ser.serialize_field("ignoreTrailingWhitespace", pbjson::private::base64::encode(&self.ignore_trailing_whitespace).as_str())?;
}
+ if !self.match_columns_by_header.is_empty() {
+ #[allow(clippy::needless_borrow)]
+ #[allow(clippy::needless_borrows_for_generic_args)]
+ struct_ser.serialize_field("matchColumnsByHeader", pbjson::private::base64::encode(&self.match_columns_by_header).as_str())?;
+ }
struct_ser.end()
}
}
@@ -1898,6 +1906,8 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {
"ignoreLeadingWhitespace",
"ignore_trailing_whitespace",
"ignoreTrailingWhitespace",
+ "match_columns_by_header",
+ "matchColumnsByHeader",
];
#[allow(clippy::enum_variant_names)]
@@ -1924,6 +1934,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {
QuoteStyle,
IgnoreLeadingWhitespace,
IgnoreTrailingWhitespace,
+ MatchColumnsByHeader,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -1967,6 +1978,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {
"quoteStyle" | "quote_style" => Ok(GeneratedField::QuoteStyle),
"ignoreLeadingWhitespace" | "ignore_leading_whitespace" => Ok(GeneratedField::IgnoreLeadingWhitespace),
"ignoreTrailingWhitespace" | "ignore_trailing_whitespace" => Ok(GeneratedField::IgnoreTrailingWhitespace),
+ "matchColumnsByHeader" | "match_columns_by_header" => Ok(GeneratedField::MatchColumnsByHeader),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -2008,6 +2020,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {
let mut quote_style__ = None;
let mut ignore_leading_whitespace__ = None;
let mut ignore_trailing_whitespace__ = None;
+ let mut match_columns_by_header__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::HasHeader => {
@@ -2168,6 +2181,14 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {
Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0)
;
}
+ GeneratedField::MatchColumnsByHeader => {
+ if match_columns_by_header__.is_some() {
+ return Err(serde::de::Error::duplicate_field("matchColumnsByHeader"));
+ }
+ match_columns_by_header__ =
+ Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0)
+ ;
+ }
}
}
Ok(CsvOptions {
@@ -2193,6 +2214,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions {
quote_style: quote_style__.unwrap_or_default(),
ignore_leading_whitespace: ignore_leading_whitespace__.unwrap_or_default(),
ignore_trailing_whitespace: ignore_trailing_whitespace__.unwrap_or_default(),
+ match_columns_by_header: match_columns_by_header__.unwrap_or_default(),
})
}
}
diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rs
index 68ccbc6e58..694c37a33d 100644
--- a/datafusion/proto-common/src/generated/prost.rs
+++ b/datafusion/proto-common/src/generated/prost.rs
@@ -711,6 +711,9 @@ pub struct CsvOptions {
/// Whether to ignore trailing whitespace in string values
#[prost(bytes = "vec", tag = "22")]
pub ignore_trailing_whitespace: ::prost::alloc::vec::Vec<u8>,
+ /// Whether to map file columns to the table schema by header name
+ #[prost(bytes = "vec", tag = "23")]
+ pub match_columns_by_header: ::prost::alloc::vec::Vec<u8>,
}
/// Options controlling CSV format
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rs
index e935a712b6..3d37887919 100644
--- a/datafusion/proto-common/src/to_proto/mod.rs
+++ b/datafusion/proto-common/src/to_proto/mod.rs
@@ -1077,6 +1077,9 @@ impl TryFrom<&CsvOptions> for protobuf::CsvOptions {
ignore_trailing_whitespace: opts
.ignore_trailing_whitespace
.map_or_else(Vec::new, |h| vec![h as u8]),
+ match_columns_by_header: opts
+ .match_columns_by_header
+ .map_or_else(Vec::new, |h| vec![h as u8]),
})
}
}
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index 5aaadaaa78..cb28587bc8 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -1404,6 +1404,7 @@ message CsvScanExecNode {
bool truncate_rows = 8;
// Custom one-byte line terminator. Absent means the default newline terminator.
optional bytes terminator = 9;
+ bool match_columns_by_header = 10;
}
message JsonScanExecNode {
diff --git a/datafusion/proto-models/src/from_proto.rs b/datafusion/proto-models/src/from_proto.rs
index ded42ebe5c..2efba9779b 100644
--- a/datafusion/proto-models/src/from_proto.rs
+++ b/datafusion/proto-models/src/from_proto.rs
@@ -283,6 +283,11 @@ impl TryFrom<&CsvOptionsProto> for CsvOptions {
} else {
Some(proto.ignore_trailing_whitespace[0] != 0)
},
+ match_columns_by_header: if proto.match_columns_by_header.is_empty() {
+ None
+ } else {
+ Some(proto.match_columns_by_header[0] != 0)
+ },
})
}
}
diff --git a/datafusion/proto-models/src/generated/datafusion_proto_common.rs b/datafusion/proto-models/src/generated/datafusion_proto_common.rs
index 68ccbc6e58..694c37a33d 100644
--- a/datafusion/proto-models/src/generated/datafusion_proto_common.rs
+++ b/datafusion/proto-models/src/generated/datafusion_proto_common.rs
@@ -711,6 +711,9 @@ pub struct CsvOptions {
/// Whether to ignore trailing whitespace in string values
#[prost(bytes = "vec", tag = "22")]
pub ignore_trailing_whitespace: ::prost::alloc::vec::Vec<u8>,
+ /// Whether to map file columns to the table schema by header name
+ #[prost(bytes = "vec", tag = "23")]
+ pub match_columns_by_header: ::prost::alloc::vec::Vec<u8>,
}
/// Options controlling CSV format
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs
index 1444b37de5..37efc52276 100644
--- a/datafusion/proto-models/src/generated/pbjson.rs
+++ b/datafusion/proto-models/src/generated/pbjson.rs
@@ -5015,6 +5015,9 @@ impl serde::Serialize for CsvScanExecNode {
if self.terminator.is_some() {
len += 1;
}
+ if self.match_columns_by_header {
+ len += 1;
+ }
if self.optional_escape.is_some() {
len += 1;
}
@@ -5045,6 +5048,9 @@ impl serde::Serialize for CsvScanExecNode {
#[allow(clippy::needless_borrows_for_generic_args)]
struct_ser.serialize_field("terminator", pbjson::private::base64::encode(&v).as_str())?;
}
+ if self.match_columns_by_header {
+ struct_ser.serialize_field("matchColumnsByHeader", &self.match_columns_by_header)?;
+ }
if let Some(v) = self.optional_escape.as_ref() {
match v {
csv_scan_exec_node::OptionalEscape::Escape(v) => {
@@ -5080,6 +5086,8 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {
"truncate_rows",
"truncateRows",
"terminator",
+ "match_columns_by_header",
+ "matchColumnsByHeader",
"escape",
"comment",
];
@@ -5093,6 +5101,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {
NewlinesInValues,
TruncateRows,
Terminator,
+ MatchColumnsByHeader,
Escape,
Comment,
}
@@ -5123,6 +5132,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {
"newlinesInValues" | "newlines_in_values" => Ok(GeneratedField::NewlinesInValues),
"truncateRows" | "truncate_rows" => Ok(GeneratedField::TruncateRows),
"terminator" => Ok(GeneratedField::Terminator),
+ "matchColumnsByHeader" | "match_columns_by_header" => Ok(GeneratedField::MatchColumnsByHeader),
"escape" => Ok(GeneratedField::Escape),
"comment" => Ok(GeneratedField::Comment),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
@@ -5151,6 +5161,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {
let mut newlines_in_values__ = None;
let mut truncate_rows__ = None;
let mut terminator__ = None;
+ let mut match_columns_by_header__ = None;
let mut optional_escape__ = None;
let mut optional_comment__ = None;
while let Some(k) = map_.next_key()? {
@@ -5199,6 +5210,12 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {
map_.next_value::<::std::option::Option<::pbjson::private::BytesDeserialize<_>>>()?.map(|x| x.0)
;
}
+ GeneratedField::MatchColumnsByHeader => {
+ if match_columns_by_header__.is_some() {
+ return Err(serde::de::Error::duplicate_field("matchColumnsByHeader"));
+ }
+ match_columns_by_header__ = Some(map_.next_value()?);
+ }
GeneratedField::Escape => {
if optional_escape__.is_some() {
return Err(serde::de::Error::duplicate_field("escape"));
@@ -5221,6 +5238,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {
newlines_in_values: newlines_in_values__.unwrap_or_default(),
truncate_rows: truncate_rows__.unwrap_or_default(),
terminator: terminator__,
+ match_columns_by_header: match_columns_by_header__.unwrap_or_default(),
optional_escape: optional_escape__,
optional_comment: optional_comment__,
})
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index 0df3ba14ca..f64f5277c5 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -2149,6 +2149,8 @@ pub struct CsvScanExecNode {
/// Custom one-byte line terminator. Absent means the default newline terminator.
#[prost(bytes = "vec", optional, tag = "9")]
pub terminator: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,
+ #[prost(bool, tag = "10")]
+ pub match_columns_by_header: bool,
#[prost(oneof = "csv_scan_exec_node::OptionalEscape", tags = "5")]
pub optional_escape: ::core::option::Option<csv_scan_exec_node::OptionalEscape>,
#[prost(oneof = "csv_scan_exec_node::OptionalComment", tags = "6")]
diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rs
index 9ccc35dfc7..b07cba5ffe 100644
--- a/datafusion/proto/tests/cases/plans/sources.rs
+++ b/datafusion/proto/tests/cases/plans/sources.rs
@@ -654,6 +654,44 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {
Ok(())
}
+#[test]
+fn roundtrip_csv_scan_preserves_match_columns_by_header() -> Result<()> {
+ use datafusion::common::config::CsvOptions;
+
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, true)]));
+ let file_source = Arc::new(
+ CsvSource::new(TableSchema::from(&file_schema)).with_csv_options(
+ CsvOptions::default()
+ .with_has_header(true)
+ .with_match_columns_by_header(true),
+ ),
+ );
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.csv".to_string(),
+ 1024,
+ )])])
+ .build();
+
+ let ctx = SessionContext::new();
+ let roundtripped = roundtrip_test_and_return(
+ DataSourceExec::from_data_source(scan_config),
+ &ctx,
+ &DefaultPhysicalExtensionCodec {},
+ &DefaultPhysicalProtoConverter {},
+ )?;
+ let csv_source = roundtripped
+ .downcast_ref::<DataSourceExec>()
+ .and_then(|exec| exec.data_source().downcast_ref::<FileScanConfig>())
+ .and_then(|scan| scan.file_source().downcast_ref::<CsvSource>())
+ .ok_or_else(|| internal_datafusion_err!("Expected CsvSource"))?;
+
+ assert!(csv_source.match_columns_by_header());
+ Ok(())
+}
+
#[tokio::test]
async fn roundtrip_parquet_exec_with_table_partition_cols() -> Result<()> {
let mut file_group =
diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
index 4edd948345..f1712c7e81 100644
--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs
@@ -1069,6 +1069,34 @@ async fn roundtrip_logical_plan_copy_to_csv() -> Result<()> {
Ok(())
}
+/// Unset and `false` must stay distinct so that an unset table option still
+/// falls back to the session default
+#[test]
+fn roundtrip_csv_options_match_columns_by_header() -> Result<()> {
+ let ctx = SessionContext::new();
+ let codec = CsvLogicalExtensionCodec {};
+ for value in [None, Some(false), Some(true)] {
+ let mut options = TableOptions::default().csv;
+ options.match_columns_by_header = value;
+
+ let mut buf = vec![];
+ codec.try_encode_file_format(
+ &mut buf,
+ Arc::new(CsvFormatFactory::new_with_options(options)),
+ )?;
+ let decoded = codec.try_decode_file_format(&buf, &ctx.task_ctx())?;
+ let decoded = decoded
+ .as_ref()
+ .downcast_ref::<CsvFormatFactory>()
+ .unwrap()
+ .options
+ .as_ref()
+ .unwrap();
+ assert_eq!(decoded.match_columns_by_header, value);
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn roundtrip_logical_plan_copy_to_json() -> Result<()> {
let ctx = SessionContext::new();
diff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.slt
index 549a74c4eb..d8ce7e9156 100644
--- a/datafusion/sqllogictest/test_files/csv_files.slt
+++ b/datafusion/sqllogictest/test_files/csv_files.slt
@@ -597,3 +597,43 @@ DROP TABLE stored_trim_both;
statement ok
DROP TABLE whitespace_source;
+
+# Match file columns to the table schema by header name
+query I
+COPY (SELECT 1 AS a, 'x' AS b, 10 AS c)
+TO 'test_files/scratch/csv_files/header_layouts/full.csv'
+STORED AS csv
+OPTIONS ('format.has_header' 'true');
+----
+1
+
+query I
+COPY (SELECT 20 AS c, 'ignored' AS undeclared, 'y' AS b)
+TO 'test_files/scratch/csv_files/header_layouts/reordered.csv'
+STORED AS csv
+OPTIONS ('format.has_header' 'true');
+----
+1
+
+statement ok
+CREATE EXTERNAL TABLE header_layouts (
+ a INT,
+ b TEXT,
+ c INT
+) STORED AS CSV
+LOCATION 'test_files/scratch/csv_files/header_layouts/'
+OPTIONS ('format.has_header' 'true', 'format.match_columns_by_header' 'true');
+
+query ITI
+SELECT a, b, c FROM header_layouts ORDER BY c;
+----
+1 x 10
+NULL y 20
+
+query I
+SELECT c FROM header_layouts WHERE a IS NULL;
+----
+20
+
+statement ok
+DROP TABLE header_layouts;
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index a26e8c9f80..57e701acef 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -1649,3 +1649,26 @@ let return_type: Option<DataType> = definition
`CreateFunction::try_new` takes `Option<FieldRef>` for the return type.
See [issue #18235](https://github.com/apache/datafusion/issues/18235) for details.
+
+### New `match_columns_by_header` field on CSV options
+
+`datafusion_common::config::CsvOptions` and `datafusion::prelude::CsvReadOptions`
+have a new public field, `match_columns_by_header`, which maps CSV file columns
+to the table schema by header name instead of by position. The protobuf
+messages `CsvOptions` and `CsvScanExecNode` gained the same field.
+
+Code that constructs these structs with an exhaustive struct literal must add
+the field or use `..Default::default()`. Builder methods are unaffected:
+
+```rust
+// 55.0.0
+let options = CsvReadOptions {
+ has_header: true,
+ ..Default::default()
+};
+
+// 56.0.0
+let options = CsvReadOptions::new()
+ .has_header(true)
+ .match_columns_by_header(true);
+```
diff --git a/docs/source/user-guide/sql/format_options.md b/docs/source/user-guide/sql/format_options.md
index a4fc32fe8e..50079d8b62 100644
--- a/docs/source/user-guide/sql/format_options.md
+++ b/docs/source/user-guide/sql/format_options.md
@@ -105,25 +105,26 @@ OPTIONS('COMPRESSION' 'gzip');
The following options are available when reading or writing CSV files. Note: If any unsupported option is specified, an error will be raised and the query will fail.
-| Option | Description | Default Value |
-| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ |
-| COMPRESSION | Sets the compression that should be applied to the entire CSV file. Supported values are GZIP, BZIP2, XZ, ZSTD, and UNCOMPRESSED. | UNCOMPRESSED |
-| HAS_HEADER | Sets if the CSV file should include column headers. If not set, uses session or system default. | None |
-| DELIMITER | Sets the character which should be used as the column delimiter within the CSV file. | `,` (comma) |
-| QUOTE | Sets the character which should be used for quoting values within the CSV file. | `"` (double quote) |
-| TERMINATOR | Sets the character which should be used as the line terminator within the CSV file. | None |
-| ESCAPE | Sets the character which should be used for escaping special characters within the CSV file. | None |
-| DOUBLE_QUOTE | Sets if quotes within quoted fields should be escaped by doubling them (e.g., `"aaa""bbb"`). | None |
-| NEWLINES_IN_VALUES | Sets if newlines in quoted values are supported. If not set, uses session or system default. | None |
-| DATE_FORMAT | Sets the format that dates should be encoded in within the CSV file. | None |
-| DATETIME_FORMAT | Sets the format that datetimes should be encoded in within the CSV file. | None |
-| TIMESTAMP_FORMAT | Sets the format that timestamps should be encoded in within the CSV file. | None |
-| TIMESTAMP_TZ_FORMAT | Sets the format that timestamps with timezone should be encoded in within the CSV file. | None |
-| TIME_FORMAT | Sets the format that times should be encoded in within the CSV file. | None |
-| NULL_VALUE | Sets the string which should be used to indicate null values within the CSV file. | None |
-| NULL_REGEX | Sets the regex pattern to match null values when loading CSVs. | None |
-| SCHEMA_INFER_MAX_REC | Sets the maximum number of records to scan to infer the schema. If set to 0, schema inference is disabled and all fields will be inferred as Utf8 (string) type. | None |
-| COMMENT | Sets the character which should be used to indicate comment lines in the CSV file. | None |
+| Option | Description | Default Value |
+| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------ |
+| COMPRESSION | Sets the compression that should be applied to the entire CSV file. Supported values are GZIP, BZIP2, XZ, ZSTD, and UNCOMPRESSED. | UNCOMPRESSED |
+| HAS_HEADER | Sets if the CSV file should include column headers. If not set, uses session or system default. | None |
+| DELIMITER | Sets the character which should be used as the column delimiter within the CSV file. | `,` (comma) |
+| QUOTE | Sets the character which should be used for quoting values within the CSV file. | `"` (double quote) |
+| TERMINATOR | Sets the character which should be used as the line terminator within the CSV file. | None |
+| ESCAPE | Sets the character which should be used for escaping special characters within the CSV file. | None |
+| DOUBLE_QUOTE | Sets if quotes within quoted fields should be escaped by doubling them (e.g., `"aaa""bbb"`). | None |
+| NEWLINES_IN_VALUES | Sets if newlines in quoted values are supported. If not set, uses session or system default. | None |
+| DATE_FORMAT | Sets the format that dates should be encoded in within the CSV file. | None |
+| DATETIME_FORMAT | Sets the format that datetimes should be encoded in within the CSV file. | None |
+| TIMESTAMP_FORMAT | Sets the format that timestamps should be encoded in within the CSV file. | None |
+| TIMESTAMP_TZ_FORMAT | Sets the format that timestamps with timezone should be encoded in within the CSV file. | None |
+| TIME_FORMAT | Sets the format that times should be encoded in within the CSV file. | None |
+| NULL_VALUE | Sets the string which should be used to indicate null values within the CSV file. | None |
+| NULL_REGEX | Sets the regex pattern to match null values when loading CSVs. | None |
+| SCHEMA_INFER_MAX_REC | Sets the maximum number of records to scan to infer the schema. If set to 0, schema inference is disabled and all fields will be inferred as Utf8 (string) type. | None |
+| COMMENT | Sets the character which should be used to indicate comment lines in the CSV file. | None |
+| MATCH_COLUMNS_BY_HEADER | Only applies when `HAS_HEADER` is true. Matches columns to the table schema by header name instead of by position. Missing columns are read as null and undeclared columns are ignored. Names are matched exactly (case-sensitive); duplicate header names resolve to the first occurrence. A leading UTF-8 BOM, blank lines, and comment lines before the header are skipped, as the CSV parser does. | None |
**Example:**