Apache DataFusion SQL Query Engine
Deprecate CsvSource::newlines_in_values and preserve proto coverage (#26179)
kosiew75ddead2b765
diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rsindex 0bfa499d7a..b62589a3e8 100644--- a/datafusion/datasource-csv/src/source.rs+++ b/datafusion/datasource-csv/src/source.rs@@ -174,6 +174,10 @@ impl CsvSource {}/// Whether values may contain newline characters+ #[deprecated(+ since = "56.0.0",+ note = "No replacement getter exists. Retain CsvOptions and read its newlines_in_values field with unwrap_or(false)."+ )]pub fn newlines_in_values(&self) -> bool {self.options.newlines_in_values.unwrap_or(false)}@@ -354,7 +358,7 @@ impl FileSource for CsvSource {)}).transpose()?,- newlines_in_values: self.newlines_in_values(),+ newlines_in_values: self.options.newlines_in_values.unwrap_or(false),truncate_rows: self.truncate_rows(),terminator: self.terminator().map(|terminator| vec![terminator]),};diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rsindex 04d002e1e6..a215a01e84 100644--- a/datafusion/proto/tests/cases/plans/sources.rs+++ b/datafusion/proto/tests/cases/plans/sources.rs@@ -535,76 +535,89 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> {let file_schema =Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));- let table_schema = TableSchema::from(&file_schema);- let file_source =- Arc::new(CsvSource::new(table_schema).with_csv_options(CsvOptions {- has_header: Some(false),- delimiter: b'|',- quote: b'\'',- escape: Some(b'\\'),- comment: Some(b'#'),- terminator: Some(0xff),- newlines_in_values: Some(true),- truncated_rows: Some(true),- ..Default::default()- }));-- let scan_config =- FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)- .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(- "/path/to/file.csv.gz".to_string(),- 1024,- )])])- .with_file_compression_type(FileCompressionType::GZIP)- .build();-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);- let roundtripped = roundtrip_test_and_return(- Arc::clone(&plan),- &ctx,- &codec,- &DefaultPhysicalProtoConverter {},- )?;- let data_source = roundtripped- .downcast_ref::<DataSourceExec>()- .ok_or_else(|| internal_datafusion_err!("Expected DataSourceExec"))?;- let file_scan = data_source- .data_source()- .downcast_ref::<FileScanConfig>()- .ok_or_else(|| internal_datafusion_err!("Expected FileScanConfig"))?;- let csv_source = file_scan- .file_source()- .downcast_ref::<CsvSource>()- .ok_or_else(|| internal_datafusion_err!("Expected CsvSource"))?;-- assert!(!csv_source.has_header());- assert_eq!(csv_source.delimiter(), b'|');- assert_eq!(csv_source.quote(), b'\'');- assert_eq!(csv_source.escape(), Some(b'\\'));- assert_eq!(csv_source.comment(), Some(b'#'));- assert_eq!(csv_source.terminator(), Some(0xff));- assert!(csv_source.newlines_in_values());- assert!(csv_source.truncate_rows());- assert_eq!(file_scan.file_compression_type, FileCompressionType::GZIP);+ for newlines_in_values in [Some(true), Some(false), None] {+ let table_schema = TableSchema::from(&file_schema);+ let file_source =+ Arc::new(CsvSource::new(table_schema).with_csv_options(CsvOptions {+ has_header: Some(false),+ delimiter: b'|',+ quote: b'\'',+ escape: Some(b'\\'),+ comment: Some(b'#'),+ terminator: Some(0xff),+ newlines_in_values,+ truncated_rows: Some(true),+ ..Default::default()+ }));++ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.csv.gz".to_string(),+ 1024,+ )])])+ .with_file_compression_type(FileCompressionType::GZIP)+ .build();- for invalid_terminator in [vec![], vec![b'\r', b'\n']] {- let mut node =- PhysicalPlanNode::try_from_physical_plan(Arc::clone(&plan), &codec)?;- match node.physical_plan_type.as_mut() {+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);+ let roundtripped = roundtrip_test_and_return(+ Arc::clone(&plan),+ &ctx,+ &codec,+ &DefaultPhysicalProtoConverter {},+ )?;+ let data_source = roundtripped+ .downcast_ref::<DataSourceExec>()+ .ok_or_else(|| internal_datafusion_err!("Expected DataSourceExec"))?;+ let file_scan = data_source+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .ok_or_else(|| internal_datafusion_err!("Expected FileScanConfig"))?;+ let csv_source = file_scan+ .file_source()+ .downcast_ref::<CsvSource>()+ .ok_or_else(|| internal_datafusion_err!("Expected CsvSource"))?;++ assert!(!csv_source.has_header());+ assert_eq!(csv_source.delimiter(), b'|');+ assert_eq!(csv_source.quote(), b'\'');+ assert_eq!(csv_source.escape(), Some(b'\\'));+ assert_eq!(csv_source.comment(), Some(b'#'));+ assert_eq!(csv_source.terminator(), Some(0xff));+ let node =+ PhysicalPlanNode::try_from_physical_plan(Arc::clone(&roundtripped), &codec)?;+ match node.physical_plan_type {Some(protobuf::physical_plan_node::PhysicalPlanType::CsvScan(scan)) => {- scan.terminator = Some(invalid_terminator);+ assert_eq!(scan.newlines_in_values, newlines_in_values.unwrap_or(false));}other => return internal_err!("Expected CsvScan node, got {other:?}"),}- let err = node- .try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)- .expect_err("invalid terminator length must fail");- assert!(- err.to_string().contains("expected exactly one byte"),- "unexpected error: {err}"- );+ assert!(csv_source.truncate_rows());+ assert_eq!(file_scan.file_compression_type, FileCompressionType::GZIP);++ // Terminator validation is independent of the newline setting.+ if newlines_in_values != Some(true) {+ continue;+ }+ for invalid_terminator in [vec![], vec![b'\r', b'\n']] {+ let mut node =+ PhysicalPlanNode::try_from_physical_plan(Arc::clone(&plan), &codec)?;+ match node.physical_plan_type.as_mut() {+ Some(protobuf::physical_plan_node::PhysicalPlanType::CsvScan(scan)) => {+ scan.terminator = Some(invalid_terminator);+ }+ other => return internal_err!("Expected CsvScan node, got {other:?}"),+ }+ let err = node+ .try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)+ .expect_err("invalid terminator length must fail");+ assert!(+ err.to_string().contains("expected exactly one byte"),+ "unexpected error: {err}"+ );+ }}Ok(())}diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 11a63a5aa5..4bc75d462d 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -115,6 +115,20 @@ change. Use `FieldMetadata::to_hashmap` where a `HashMap` is still required.[`metadata`]: https://docs.rs/arrow/60.0.0/arrow/datatypes/struct.Metadata.html+### `CsvSource::newlines_in_values` is deprecated++`CsvSource::newlines_in_values()` is deprecated starting in 56.0.0 but remains+callable with unchanged behavior. There is no replacement getter, and+`CsvSource` does not expose a `csv_options()` method.++**Migration guide:**++Retain the `CsvOptions` used to configure the source before passing them to+`CsvSource::with_csv_options`. Read `options.newlines_in_values.unwrap_or(false)`+to obtain the same boolean value: `None` and `Some(false)` mean false, while+`Some(true)` means true. CSV reading, repartitioning, and serialization behavior+are unchanged.+### `ParquetAccessPlan::into_overall_row_selection` is deprecated`ParquetAccessPlan::into_overall_row_selection` is deprecated starting in 56.0.0.