Apache DataFusion SQL Query Engine
fix(proto): preserve CSV/JSON scan options on roundtrip (#24233)
Burak Şen97e1c1c4866c
diff --git a/Cargo.lock b/Cargo.lockindex a09126ce1e..f517fa5593 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2546,11 +2546,13 @@ dependencies = ["datafusion-proto-common","datafusion-proto-models","doc-comment",+ "flate2","object_store","pretty_assertions","prost","recursive","serde_json",+ "tempfile","tokio",]diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rsindex 04533d8c78..0bfa499d7a 100644--- a/datafusion/datasource-csv/src/source.rs+++ b/datafusion/datasource-csv/src/source.rs@@ -356,6 +356,7 @@ impl FileSource for CsvSource {.transpose()?,newlines_in_values: self.newlines_in_values(),truncate_rows: self.truncate_rows(),+ terminator: self.terminator().map(|terminator| vec![terminator]),};Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::CsvScan(node)),@@ -576,20 +577,28 @@ fn proto_str_to_byte(s: &str, description: &str) -> Result<u8> {Ok(s.as_bytes()[0])}+#[cfg(feature = "proto")]+fn proto_bytes_to_byte(bytes: &[u8], description: &str) -> Result<u8> {+ let [byte] = bytes else {+ return datafusion_common::internal_err!(+ "Invalid CSV {description}: expected exactly one byte, got {}",+ bytes.len()+ );+ };+ Ok(*byte)+}+#[cfg(feature = "proto")]impl CsvSource {/// Reconstructs a `DataSourceExec` from a protobuf `CsvScan`.///- /// Custom line terminators are not represented in the wire format.+ /// Payloads without a terminator use the default newline terminator.pub fn try_from_proto(node: &datafusion_proto_models::protobuf::PhysicalPlanNode,ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,) -> Result<Arc<dyn ExecutionPlan>> {use datafusion_common::config::CsvOptions;- use datafusion_datasource::file_compression_type::FileCompressionType;- use datafusion_datasource::file_scan_config::{- FileScanConfig, FileScanConfigBuilder,- };+ use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_datasource::source::DataSourceExec;use datafusion_proto_models::protobuf;@@ -617,6 +626,11 @@ impl CsvSource {}None => None,};+ let terminator = scan+ .terminator+ .as_deref()+ .map(|terminator| proto_bytes_to_byte(terminator, "terminator"))+ .transpose()?;let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;@@ -632,16 +646,11 @@ impl CsvSource {CsvSource::new(table_schema).with_csv_options(csv_options).with_escape(escape)- .with_comment(comment),+ .with_comment(comment)+ .with_terminator(terminator),);- // The compression type is not on the wire; CSV scans always- // deserialize as uncompressed.- let conf = FileScanConfigBuilder::from(FileScanConfig::try_from_proto(- base_conf, ctx, source,- )?)- .with_file_compression_type(FileCompressionType::UNCOMPRESSED)- .build();+ let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;Ok(DataSourceExec::from_data_source(conf))}}diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rsindex 15840223cc..5b576c1166 100644--- a/datafusion/datasource-json/src/source.rs+++ b/datafusion/datasource-json/src/source.rs@@ -159,6 +159,11 @@ impl JsonSource {self.newline_delimited = newline_delimited;self}++ /// Returns whether this source reads newline-delimited JSON.+ pub fn is_newline_delimited(&self) -> bool {+ self.newline_delimited+ }}impl From<JsonSource> for Arc<dyn FileSource> {@@ -254,6 +259,11 @@ impl FileSource for JsonSource {let node = protobuf::JsonScanExecNode {base_conf: Some(base.try_to_proto(ctx)?),+ newline_delimited: if self.newline_delimited {+ None+ } else {+ Some(false)+ },};Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::JsonScan(node)),@@ -265,7 +275,7 @@ impl FileSource for JsonSource {impl JsonSource {/// Reconstructs a `DataSourceExec` from a protobuf `JsonScan`.///- /// Defaults to newline-delimited JSON because protobuf does not encode the mode.+ /// Payloads without a mode default to newline-delimited JSON.pub fn try_from_proto(node: &datafusion_proto_models::protobuf::PhysicalPlanNode,ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,@@ -289,7 +299,10 @@ impl JsonSource {})?;let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;- let source = Arc::new(JsonSource::new(table_schema));+ let source = Arc::new(+ JsonSource::new(table_schema)+ .with_newline_delimited(scan.newline_delimited.unwrap_or(true)),+ );let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;Ok(DataSourceExec::from_data_source(conf))diff --git a/datafusion/datasource/src/file_scan_config/proto.rs b/datafusion/datasource/src/file_scan_config/proto.rsindex d7135173c8..efd17e4f57 100644--- a/datafusion/datasource/src/file_scan_config/proto.rs+++ b/datafusion/datasource/src/file_scan_config/proto.rs@@ -27,8 +27,8 @@//! `FileSource::try_to_proto` hook (CSV, JSON, Arrow, Parquet, Avro) builds its//! `*ScanExecNode` around [`FileScanConfig::try_to_proto`] and decodes with//! [`FileScanConfig::try_from_proto`], keeping a single copy of the shared-//! wire logic. The wire format is byte-for-byte identical to the old central-//! serializer.+//! wire logic. Existing fields remain wire-compatible with the old central+//! serializer; new options use optional fields with legacy defaults.//!//! Child physical expressions (sort orderings, hash/range partitioning, and//! projection expressions) are (de)serialized through `ctx.encode_expr` /@@ -38,6 +38,7 @@use std::sync::Arc;use arrow::datatypes::Schema;+use datafusion_common::parsers::CompressionTypeVariant;use datafusion_common::{DataFusionError, Result, internal_datafusion_err};use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};@@ -46,9 +47,11 @@ use datafusion_physical_expr_common::sort_expr::{sort_exprs_try_from_proto, sort_exprs_try_to_proto,};use datafusion_physical_plan::proto::{ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx};+use datafusion_proto_models::datafusion_common::CompressionTypeVariant as ProtoCompressionTypeVariant;use datafusion_proto_models::protobuf;use crate::file::FileSource;+use crate::file_compression_type::FileCompressionType;use crate::file_scan_config::{FileScanConfig, FileScanConfigBuilder};use crate::table_schema::TableSchema;@@ -57,8 +60,9 @@ impl FileScanConfig {/// [`protobuf::FileScanExecConf`].////// Each concrete [`FileSource::try_to_proto`]- /// wraps the returned value in its own `*ScanExecNode`. Byte-compatible with- /// the former `serialize_file_scan_config` in `datafusion-proto`.+ /// wraps the returned value in its own `*ScanExecNode`. Existing fields are+ /// byte-compatible with the former `serialize_file_scan_config` in+ /// `datafusion-proto`.pub fn try_to_proto(&self,ctx: &ExecutionPlanEncodeCtx<'_>,@@ -114,6 +118,13 @@ impl FileScanConfig {}).transpose()?;+ let file_compression_type =+ self.file_compression_type.is_compressed().then(|| {+ let compression: ProtoCompressionTypeVariant =+ (*self.file_compression_type.get_variant()).into();+ compression as i32+ });+Ok(protobuf::FileScanExecConf {file_groups,statistics: Some((&self.statistics()).into()),@@ -131,6 +142,7 @@ impl FileScanConfig {batch_size: self.batch_size.map(|s| s as u64),projection_exprs,output_partitioning,+ file_compression_type,})}@@ -138,7 +150,8 @@ impl FileScanConfig {/// and a `file_source` the caller has already rebuilt (typically from the/// table schema via [`FileScanConfig::parse_table_schema_from_proto`]).///- /// Byte-compatible with the former `parse_protobuf_file_scan_config`.+ /// Existing fields are byte-compatible with the former+ /// `parse_protobuf_file_scan_config`.pub fn try_from_proto(conf: &protobuf::FileScanExecConf,ctx: &ExecutionPlanDecodeCtx<'_>,@@ -194,6 +207,19 @@ impl FileScanConfig {.transpose()?.flatten();+ let file_compression_type = conf+ .file_compression_type+ .map(|value| {+ let compression =+ ProtoCompressionTypeVariant::try_from(value).map_err(|_| {+ internal_datafusion_err!("Unknown file compression type: {value}")+ })?;+ let compression: CompressionTypeVariant = compression.into();+ Ok::<_, DataFusionError>(FileCompressionType::from(compression))+ })+ .transpose()?+ .unwrap_or(FileCompressionType::UNCOMPRESSED);+// Parse projection expressions if present and apply to the file source.let file_source = if let Some(proto_projection_exprs) = &conf.projection_exprs {let projection_exprs: Vec<ProjectionExpr> = proto_projection_exprs@@ -226,7 +252,8 @@ impl FileScanConfig {.with_limit(conf.limit.as_ref().map(|sl| sl.limit as usize)).with_output_ordering(output_ordering).with_output_partitioning(output_partitioning)- .with_batch_size(conf.batch_size.map(|s| s as usize));+ .with_batch_size(conf.batch_size.map(|s| s as usize))+ .with_file_compression_type(file_compression_type);Ok(config_builder.build())}diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex 7b8f8e3985..ce5d11425c 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1265,6 +1265,9 @@ message FileScanExecConf {reserved 14;reserved "partitioned_by_file_group";optional Partitioning output_partitioning = 15;+ // Compression used by formats such as CSV and JSON. Absent means uncompressed+ // for compatibility with payloads written before this field existed.+ optional datafusion_common.CompressionTypeVariant file_compression_type = 16;}message ParquetScanExecNode {@@ -1291,10 +1294,14 @@ message CsvScanExecNode {}bool newlines_in_values = 7;bool truncate_rows = 8;+ // Custom one-byte line terminator. Absent means the default newline terminator.+ optional bytes terminator = 9;}message JsonScanExecNode {FileScanExecConf base_conf = 1;+ // Absent means newline-delimited JSON for compatibility with older payloads.+ optional bool newline_delimited = 2;}message AvroScanExecNode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex bc88f5c9c2..0f622de3fa 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -4471,6 +4471,9 @@ impl serde::Serialize for CsvScanExecNode {if self.truncate_rows {len += 1;}+ if self.terminator.is_some() {+ len += 1;+ }if self.optional_escape.is_some() {len += 1;}@@ -4496,6 +4499,11 @@ impl serde::Serialize for CsvScanExecNode {if self.truncate_rows {struct_ser.serialize_field("truncateRows", &self.truncate_rows)?;}+ if let Some(v) = self.terminator.as_ref() {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("terminator", pbjson::private::base64::encode(&v).as_str())?;+ }if let Some(v) = self.optional_escape.as_ref() {match v {csv_scan_exec_node::OptionalEscape::Escape(v) => {@@ -4530,6 +4538,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {"newlinesInValues","truncate_rows","truncateRows",+ "terminator","escape","comment",];@@ -4542,6 +4551,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {Quote,NewlinesInValues,TruncateRows,+ Terminator,Escape,Comment,}@@ -4571,6 +4581,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {"quote" => Ok(GeneratedField::Quote),"newlinesInValues" | "newlines_in_values" => Ok(GeneratedField::NewlinesInValues),"truncateRows" | "truncate_rows" => Ok(GeneratedField::TruncateRows),+ "terminator" => Ok(GeneratedField::Terminator),"escape" => Ok(GeneratedField::Escape),"comment" => Ok(GeneratedField::Comment),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),@@ -4598,6 +4609,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {let mut quote__ = None;let mut newlines_in_values__ = None;let mut truncate_rows__ = None;+ let mut terminator__ = None;let mut optional_escape__ = None;let mut optional_comment__ = None;while let Some(k) = map_.next_key()? {@@ -4638,6 +4650,14 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {}truncate_rows__ = Some(map_.next_value()?);}+ GeneratedField::Terminator => {+ if terminator__.is_some() {+ return Err(serde::de::Error::duplicate_field("terminator"));+ }+ terminator__ =+ map_.next_value::<::std::option::Option<::pbjson::private::BytesDeserialize<_>>>()?.map(|x| x.0)+ ;+ }GeneratedField::Escape => {if optional_escape__.is_some() {return Err(serde::de::Error::duplicate_field("escape"));@@ -4659,6 +4679,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode {quote: quote__.unwrap_or_default(),newlines_in_values: newlines_in_values__.unwrap_or_default(),truncate_rows: truncate_rows__.unwrap_or_default(),+ terminator: terminator__,optional_escape: optional_escape__,optional_comment: optional_comment__,})@@ -7109,6 +7130,9 @@ impl serde::Serialize for FileScanExecConf {if self.output_partitioning.is_some() {len += 1;}+ if self.file_compression_type.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.FileScanExecConf", len)?;if !self.file_groups.is_empty() {struct_ser.serialize_field("fileGroups", &self.file_groups)?;@@ -7148,6 +7172,11 @@ impl serde::Serialize for FileScanExecConf {if let Some(v) = self.output_partitioning.as_ref() {struct_ser.serialize_field("outputPartitioning", v)?;}+ if let Some(v) = self.file_compression_type.as_ref() {+ let v = super::datafusion_common::CompressionTypeVariant::try_from(*v)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", *v)))?;+ struct_ser.serialize_field("fileCompressionType", &v)?;+ }struct_ser.end()}}@@ -7177,6 +7206,8 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {"projectionExprs","output_partitioning","outputPartitioning",+ "file_compression_type",+ "fileCompressionType",];#[allow(clippy::enum_variant_names)]@@ -7193,6 +7224,7 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {BatchSize,ProjectionExprs,OutputPartitioning,+ FileCompressionType,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -7226,6 +7258,7 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {"batchSize" | "batch_size" => Ok(GeneratedField::BatchSize),"projectionExprs" | "projection_exprs" => Ok(GeneratedField::ProjectionExprs),"outputPartitioning" | "output_partitioning" => Ok(GeneratedField::OutputPartitioning),+ "fileCompressionType" | "file_compression_type" => Ok(GeneratedField::FileCompressionType),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -7257,6 +7290,7 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {let mut batch_size__ = None;let mut projection_exprs__ = None;let mut output_partitioning__ = None;+ let mut file_compression_type__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::FileGroups => {@@ -7336,6 +7370,12 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {}output_partitioning__ = map_.next_value()?;}+ GeneratedField::FileCompressionType => {+ if file_compression_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("fileCompressionType"));+ }+ file_compression_type__ = map_.next_value::<::std::option::Option<super::datafusion_common::CompressionTypeVariant>>()?.map(|x| x as i32);+ }}}Ok(FileScanExecConf {@@ -7351,6 +7391,7 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {batch_size: batch_size__,projection_exprs: projection_exprs__,output_partitioning: output_partitioning__,+ file_compression_type: file_compression_type__,})}}@@ -11282,10 +11323,16 @@ impl serde::Serialize for JsonScanExecNode {if self.base_conf.is_some() {len += 1;}+ if self.newline_delimited.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.JsonScanExecNode", len)?;if let Some(v) = self.base_conf.as_ref() {struct_ser.serialize_field("baseConf", v)?;}+ if let Some(v) = self.newline_delimited.as_ref() {+ struct_ser.serialize_field("newlineDelimited", v)?;+ }struct_ser.end()}}@@ -11298,11 +11345,14 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {const FIELDS: &[&str] = &["base_conf","baseConf",+ "newline_delimited",+ "newlineDelimited",];#[allow(clippy::enum_variant_names)]enum GeneratedField {BaseConf,+ NewlineDelimited,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -11325,6 +11375,7 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {{match value {"baseConf" | "base_conf" => Ok(GeneratedField::BaseConf),+ "newlineDelimited" | "newline_delimited" => Ok(GeneratedField::NewlineDelimited),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -11345,6 +11396,7 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {V: serde::de::MapAccess<'de>,{let mut base_conf__ = None;+ let mut newline_delimited__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::BaseConf => {@@ -11353,10 +11405,17 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {}base_conf__ = map_.next_value()?;}+ GeneratedField::NewlineDelimited => {+ if newline_delimited__.is_some() {+ return Err(serde::de::Error::duplicate_field("newlineDelimited"));+ }+ newline_delimited__ = map_.next_value()?;+ }}}Ok(JsonScanExecNode {base_conf: base_conf__,+ newline_delimited: newline_delimited__,})}}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 5bc46dfd7c..7527c58895 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -1947,6 +1947,14 @@ pub struct FileScanExecConf {pub projection_exprs: ::core::option::Option<ProjectionExprs>,#[prost(message, optional, tag = "15")]pub output_partitioning: ::core::option::Option<Partitioning>,+ /// Compression used by formats such as CSV and JSON. Absent means uncompressed+ /// for compatibility with payloads written before this field existed.+ #[prost(+ enumeration = "super::datafusion_common::CompressionTypeVariant",+ optional,+ tag = "16"+ )]+ pub file_compression_type: ::core::option::Option<i32>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct ParquetScanExecNode {@@ -1973,6 +1981,9 @@ pub struct CsvScanExecNode {pub newlines_in_values: bool,#[prost(bool, tag = "8")]pub truncate_rows: bool,+ /// Custom one-byte line terminator. Absent means the default newline terminator.+ #[prost(bytes = "vec", optional, tag = "9")]+ pub terminator: ::core::option::Option<::prost::alloc::vec::Vec<u8>>,#[prost(oneof = "csv_scan_exec_node::OptionalEscape", tags = "5")]pub optional_escape: ::core::option::Option<csv_scan_exec_node::OptionalEscape>,#[prost(oneof = "csv_scan_exec_node::OptionalComment", tags = "6")]@@ -1995,6 +2006,9 @@ pub mod csv_scan_exec_node {pub struct JsonScanExecNode {#[prost(message, optional, tag = "1")]pub base_conf: ::core::option::Option<FileScanExecConf>,+ /// Absent means newline-delimited JSON for compatibility with older payloads.+ #[prost(bool, optional, tag = "2")]+ pub newline_delimited: ::core::option::Option<bool>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct AvroScanExecNode {diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex b6e9d25868..4b11bea011 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -83,6 +83,7 @@ serde_json = { workspace = true, optional = true }async-trait = { workspace = true }datafusion = { workspace = true, default-features = false, features = ["sql",+ "compression","datetime_expressions","nested_expressions","unicode_expressions",@@ -91,5 +92,7 @@ datafusion-functions = { workspace = true, default-features = true }datafusion-functions-aggregate = { workspace = true }datafusion-functions-window-common = { workspace = true }doc-comment = { workspace = true }+flate2 = { workspace = true }pretty_assertions = "1.4"+tempfile = { workspace = true }tokio = { workspace = true, features = ["rt-multi-thread"] }diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 222901aff5..5f534ef633 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -106,6 +106,7 @@ mod file_scan_config_serde {use arrow::datatypes::{DataType, Field};use datafusion_common::{Constraint, Constraints, ScalarValue, Statistics};use datafusion_datasource::file::FileSource;+ use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder,@@ -257,6 +258,7 @@ mod file_scan_config_serde {.with_statistics(table_statistics).with_limit(Some(17)).with_batch_size(Some(256))+ .with_file_compression_type(FileCompressionType::GZIP).with_output_ordering(vec![ordering]).with_output_partitioning(output_partitioning).build()@@ -365,10 +367,27 @@ mod file_scan_config_serde {assert_eq!(decoded.file_groups[1].len(), 1);assert!(decoded.file_groups[0].files()[0].arrow_schema.is_some());assert!(decoded.file_groups[0].files()[1].arrow_schema.is_none());+ assert_eq!(decoded.file_compression_type, FileCompressionType::GZIP);Ok(())}+ #[test]+ fn new_file_scan_config_decode_without_compression_uses_legacy_default() -> Result<()>+ {+ let serde = FileScanSerdeHarness::new();+ let mut encoded = serde.encode(&test_config(None))?;+ assert!(encoded.file_compression_type.is_some());++ encoded.file_compression_type = None;+ let decoded = serde.decode(&encoded)?;+ assert_eq!(+ decoded.file_compression_type,+ FileCompressionType::UNCOMPRESSED+ );+ Ok(())+ }+#[test]fn new_file_scan_config_serde_preserves_projection_presence() -> Result<()> {let serde = FileScanSerdeHarness::new();@@ -449,6 +468,16 @@ mod file_scan_config_serde {"unexpected error: {err}");+ let mut unknown_compression = valid;+ unknown_compression.file_compression_type = Some(i32::MAX);+ let err = serde+ .decode(&unknown_compression)+ .expect_err("unknown compression type must fail");+ assert!(+ err.to_string().contains("Unknown file compression type"),+ "unexpected error: {err}"+ );+Ok(())}diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rsindex ca8dca74d7..4392ef5238 100644--- a/datafusion/proto/tests/cases/plans/sources.rs+++ b/datafusion/proto/tests/cases/plans/sources.rs@@ -53,6 +53,7 @@ use datafusion::scalar::ScalarValue;use datafusion_common::config::TableParquetOptions;use datafusion_common::stats::Precision;use datafusion_common::{DataFusionError, Result, internal_datafusion_err, internal_err};+use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::{TableSchema, TableSchemaBuilder};use datafusion_expr::ColumnarValue;use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;@@ -280,18 +281,119 @@ fn arrow_scan_without_format_field_decodes_as_file_format() -> Result<()> {}#[test]-fn roundtrip_json_scan() -> Result<()> {+fn roundtrip_json_scan_preserves_format_options() -> Result<()> {let file_schema =Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));- let file_source = Arc::new(JsonSource::new(TableSchema::from(&file_schema)));+ let file_source = Arc::new(+ JsonSource::new(TableSchema::from(&file_schema)).with_newline_delimited(false),+ );let scan_config =FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source).with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.json".to_string(),+ "/path/to/file.json.gz".to_string(),1024,)])])+ .with_file_compression_type(FileCompressionType::GZIP).build();- roundtrip_test(DataSourceExec::from_data_source(scan_config))++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);+ let roundtripped = roundtrip_test_and_return(+ Arc::clone(&plan),+ &ctx,+ &codec,+ &DefaultPhysicalProtoConverter {},+ )?;+ let file_scan = roundtripped+ .downcast_ref::<DataSourceExec>()+ .and_then(|exec| exec.data_source().downcast_ref::<FileScanConfig>())+ .ok_or_else(|| internal_datafusion_err!("Expected FileScanConfig"))?;+ let json_source = file_scan+ .file_source()+ .downcast_ref::<JsonSource>()+ .ok_or_else(|| internal_datafusion_err!("Expected JsonSource"))?;+ assert!(!json_source.is_newline_delimited());+ assert_eq!(file_scan.file_compression_type, FileCompressionType::GZIP);++ // Payloads written before these fields existed must keep their historical+ // defaults: newline-delimited JSON without compression.+ let mut node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;+ match node.physical_plan_type.as_mut() {+ Some(protobuf::physical_plan_node::PhysicalPlanType::JsonScan(scan)) => {+ scan.newline_delimited = None;+ scan.base_conf+ .as_mut()+ .expect("JSON scan has a base config")+ .file_compression_type = None;+ }+ other => return internal_err!("Expected JsonScan node, got {other:?}"),+ }+ let decoded = node.try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)?;+ let file_scan = decoded+ .downcast_ref::<DataSourceExec>()+ .and_then(|exec| exec.data_source().downcast_ref::<FileScanConfig>())+ .ok_or_else(|| internal_datafusion_err!("Expected FileScanConfig"))?;+ let json_source = file_scan+ .file_source()+ .downcast_ref::<JsonSource>()+ .ok_or_else(|| internal_datafusion_err!("Expected JsonSource"))?;+ assert!(json_source.is_newline_delimited());+ assert_eq!(+ file_scan.file_compression_type,+ FileCompressionType::UNCOMPRESSED+ );+ Ok(())+}++#[tokio::test]+async fn roundtrip_compressed_json_array_scan_executes() -> Result<()> {+ use datafusion::prelude::JsonReadOptions;+ use flate2::Compression;+ use flate2::write::GzEncoder;+ use std::io::Write;++ let tmp_dir = tempfile::TempDir::new()?;+ let path = tmp_dir.path().join("array.json.gz");+ let file = std::fs::File::create(&path)?;+ let mut encoder = GzEncoder::new(file, Compression::default());+ encoder.write_all(br#"[{"a": 1, "b": "hello"}, {"a": 2, "b": "world"}]"#)?;+ encoder.finish()?;++ let ctx = SessionContext::new();+ let options = JsonReadOptions::default()+ .newline_delimited(false)+ .file_compression_type(FileCompressionType::GZIP)+ .file_extension(".json.gz");+ ctx.register_json("test_table", path.to_string_lossy(), options)+ .await?;++ let initial_plan = ctx+ .sql("SELECT a, b FROM test_table ORDER BY a")+ .await?+ .create_physical_plan()+ .await?;+ let roundtripped = roundtrip_test_and_return(+ initial_plan,+ &ctx,+ &DefaultPhysicalExtensionCodec {},+ &DefaultPhysicalProtoConverter {},+ )?;+ let batches =+ datafusion::physical_plan::collect(roundtripped, ctx.task_ctx()).await?;++ datafusion::assert_batches_eq!(+ &[+ "+---+-------+",+ "| a | b |",+ "+---+-------+",+ "| 1 | hello |",+ "| 2 | world |",+ "+---+-------+",+ ],+ &batches+ );+ Ok(())}#[cfg(feature = "avro")]@@ -326,6 +428,7 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {quote: b'\'',escape: Some(b'\\'),comment: Some(b'#'),+ terminator: Some(0xff),newlines_in_values: Some(true),truncated_rows: Some(true),..Default::default()@@ -334,16 +437,19 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {let scan_config =FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source).with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.csv".to_string(),+ "/path/to/file.csv.gz".to_string(),1024,)])])+ .with_file_compression_type(FileCompressionType::GZIP).build();let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);let roundtripped = roundtrip_test_and_return(- DataSourceExec::from_data_source(scan_config),+ Arc::clone(&plan),&ctx,- &DefaultPhysicalExtensionCodec {},+ &codec,&DefaultPhysicalProtoConverter {},)?;let data_source = roundtripped@@ -363,8 +469,28 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {assert_eq!(csv_source.quote(), b'\'');assert_eq!(csv_source.escape(), Some(b'\\'));assert_eq!(csv_source.comment(), Some(b'#'));+ assert_eq!(csv_source.terminator(), Some(0xff));assert!(csv_source.newlines_in_values());assert!(csv_source.truncate_rows());+ assert_eq!(file_scan.file_compression_type, FileCompressionType::GZIP);++ for invalid_terminator in [vec![], vec![b'\r', b'\n']] {+ let mut node =+ PhysicalPlanNode::try_from_physical_plan(Arc::clone(&plan), &codec)?;+ match node.physical_plan_type.as_mut() {+ Some(protobuf::physical_plan_node::PhysicalPlanType::CsvScan(scan)) => {+ scan.terminator = Some(invalid_terminator);+ }+ other => return internal_err!("Expected CsvScan node, got {other:?}"),+ }+ let err = node+ .try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)+ .expect_err("invalid terminator length must fail");+ assert!(+ err.to_string().contains("expected exactly one byte"),+ "unexpected error: {err}"+ );+ }Ok(())}