Apache DataFusion SQL Query Engine
feat: add coerce_primitive in json (#26561)
Ion Koutsouris6c429176e589
diff --git a/Cargo.lock b/Cargo.lockindex c55c3e5647..6003cba5d3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2074,7 +2074,9 @@ dependencies = ["datafusion-session","futures","object_store",+ "rstest","serde_json",+ "tempfile","tokio","tokio-stream",]diff --git a/datafusion/core/src/datasource/physical_plan/json.rs b/datafusion/core/src/datasource/physical_plan/json.rsindex 2590f3a8ba..6bebcd0d8e 100644--- a/datafusion/core/src/datasource/physical_plan/json.rs+++ b/datafusion/core/src/datasource/physical_plan/json.rs@@ -600,4 +600,58 @@ mod tests {);Ok(())}++ #[tokio::test]+ async fn json_exec_coerce_primitive() -> Result<()> {+ use crate::physical_plan::collect;+ use crate::test::object_store::local_unpartitioned_file;+ use arrow::datatypes::{DataType, Schema};+ use datafusion_common::assert_contains;+ use datafusion_datasource::PartitionedFile;++ let tmp_dir = TempDir::new()?;+ let path = tmp_dir.path().join("coerce.json");+ fs::write(+ &path,+ "{\"a\": false}\n{\"a\": 3.5}\n{\"a\": \"text\"}\n{\"a\": null}\n{}\n",+ )?;+ let file = PartitionedFile::from(local_unpartitioned_file(&path));+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, true)]));++ let read = |coerce| {+ let source =+ JsonSource::new(Arc::clone(&schema)).with_coerce_primitive(coerce);+ let conf = FileScanConfigBuilder::new(+ ObjectStoreUrl::local_filesystem(),+ Arc::new(source),+ )+ .with_file(file.clone())+ .build();+ collect(+ DataSourceExec::from_data_source(conf),+ SessionContext::new().task_ctx(),+ )+ };++ let batches = read(true).await?;+ assert_snapshot!(batches_to_string(&batches), @r"+ +-------++ | a |+ +-------++ | false |+ | 3.5 |+ | text |+ | |+ | |+ +-------++ ");+ // The snapshot shows NULL as an empty cell, the same as an empty string.+ let nulls: usize = batches.iter().map(|b| b.column(0).null_count()).sum();+ assert_eq!(nulls, 2);+ assert_contains!(+ read(false).await.unwrap_err().to_string(),+ "expected string got false"+ );+ Ok(())+ }}diff --git a/datafusion/datasource-json/Cargo.toml b/datafusion/datasource-json/Cargo.tomlindex b1c7cc006d..3ed08635bf 100644--- a/datafusion/datasource-json/Cargo.toml+++ b/datafusion/datasource-json/Cargo.toml@@ -50,7 +50,10 @@ tokio-stream = { workspace = true, features = ["sync"] }workspace = true[dev-dependencies]+datafusion-datasource = { workspace = true, features = ["compression"] }+rstest = { workspace = true }serde_json = { workspace = true }+tempfile = { workspace = true }[lib]name = "datafusion_datasource_json"diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rsindex 3b326e3972..c3ca46a170 100644--- a/datafusion/datasource-json/src/source.rs+++ b/datafusion/datasource-json/src/source.rs@@ -92,6 +92,9 @@ pub struct JsonOpener {/// When `true` (default), expects newline-delimited JSON (NDJSON)./// When `false`, expects JSON array format `[{...}, {...}]`.newline_delimited: bool,+ /// When `true`, a JSON number or boolean read into a string column is kept as its text.+ /// When `false` (default), it is an error.+ coerce_primitive: bool,}impl JsonOpener {@@ -109,8 +112,18 @@ impl JsonOpener {file_compression_type,object_store,newline_delimited,+ coerce_primitive: false,}}++ /// Set whether a JSON number or boolean read into a string column is kept as its text.+ ///+ /// When `false` (default), it is an error. See+ /// [`ReaderBuilder::with_coerce_primitive`].+ pub fn with_coerce_primitive(mut self, coerce_primitive: bool) -> Self {+ self.coerce_primitive = coerce_primitive;+ self+ }}/// JsonSource holds the extra configuration that is necessary for [`JsonOpener`]@@ -123,6 +136,9 @@ pub struct JsonSource {/// When `true` (default), expects newline-delimited JSON (NDJSON)./// When `false`, expects JSON array format `[{...}, {...}]`.newline_delimited: bool,+ /// When `true`, a JSON number or boolean read into a string column is kept as its text.+ /// When `false` (default), it is an error.+ coerce_primitive: bool,}impl JsonSource {@@ -135,9 +151,24 @@ impl JsonSource {batch_size: None,metrics: ExecutionPlanMetricsSet::new(),newline_delimited: true,+ coerce_primitive: false,}}+ /// Set whether a JSON number or boolean read into a string column is kept as its text.+ ///+ /// For example, `{"a": false}` read with `a: Utf8` gives `"false"`. When `false`+ /// (default), it is an error. See [`ReaderBuilder::with_coerce_primitive`].+ pub fn with_coerce_primitive(mut self, coerce_primitive: bool) -> Self {+ self.coerce_primitive = coerce_primitive;+ self+ }++ /// Returns whether a JSON number or boolean read into a string column is kept as its text.+ pub fn coerce_primitive(&self) -> bool {+ self.coerce_primitive+ }+/// Set whether to read as newline-delimited JSON.////// When `true` (default), expects newline-delimited format.@@ -179,6 +210,7 @@ impl FileSource for JsonSource {file_compression_type: base_config.file_compression_type,object_store,newline_delimited: self.newline_delimited,+ coerce_primitive: self.coerce_primitive,}) as Arc<dyn FileOpener>;// Wrap with ProjectionOpener@@ -257,6 +289,7 @@ impl FileSource for JsonSource {// Serialized in `base` and reapplied on decode.projection: _,newline_delimited,+ coerce_primitive,} = self;let node = protobuf::JsonScanExecNode {@@ -266,6 +299,7 @@ impl FileSource for JsonSource {} else {Some(false)},+ coerce_primitive: coerce_primitive.then_some(true),};Ok(Some(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::JsonScan(node)),@@ -277,7 +311,8 @@ impl FileSource for JsonSource {impl JsonSource {/// Reconstructs a `DataSourceExec` from a protobuf `JsonScan`.///- /// Payloads without a mode default to newline-delimited JSON.+ /// Payloads without a mode default to newline-delimited JSON, and payloads without+ /// `coerce_primitive` default to `false`.pub fn try_from_proto(node: &datafusion_proto_models::protobuf::PhysicalPlanNode,ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,@@ -299,6 +334,7 @@ impl JsonSource {let protobuf::JsonScanExecNode {base_conf,newline_delimited,+ coerce_primitive,} = scan;let base_conf = base_conf.as_ref().ok_or_else(|| {@@ -310,7 +346,8 @@ impl JsonSource {let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;let source = Arc::new(JsonSource::new(table_schema)- .with_newline_delimited(newline_delimited.unwrap_or(true)),+ .with_newline_delimited(newline_delimited.unwrap_or(true))+ .with_coerce_primitive(coerce_primitive.unwrap_or(false)),);let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;@@ -336,6 +373,7 @@ impl FileOpener for JsonOpener {let batch_size = self.batch_size;let file_compression_type = self.file_compression_type.to_owned();let newline_delimited = self.newline_delimited;+ let coerce_primitive = self.coerce_primitive;// JSON array format requires reading the complete fileif !newline_delimited && partitioned_file.range.is_some() {@@ -377,6 +415,7 @@ impl FileOpener for JsonOpener {let decoder = ReaderBuilder::new(schema).with_batch_size(batch_size)+ .with_coerce_primitive(coerce_primitive).build_decoder()?;let input = file_compression_type.convert_stream(aligned_stream.boxed())?@@ -402,6 +441,7 @@ impl FileOpener for JsonOpener {let reader = BufReader::new(bytes);let arrow_reader = ReaderBuilder::new(schema).with_batch_size(batch_size)+ .with_coerce_primitive(coerce_primitive).build(reader)?;Ok(futures::stream::iter(arrow_reader)@@ -415,6 +455,7 @@ impl FileOpener for JsonOpener {);let arrow_reader = ReaderBuilder::new(schema).with_batch_size(batch_size)+ .with_coerce_primitive(coerce_primitive).build(ndjson_reader)?;Ok(futures::stream::iter(arrow_reader)@@ -428,6 +469,7 @@ impl FileOpener for JsonOpener {let s = s.map_err(DataFusionError::from);let decoder = ReaderBuilder::new(schema).with_batch_size(batch_size)+ .with_coerce_primitive(coerce_primitive).build_decoder()?;let input =file_compression_type.convert_stream(s.boxed())?.fuse();@@ -504,6 +546,7 @@ impl FileOpener for JsonOpener {match ReaderBuilder::new(schema).with_batch_size(batch_size)+ .with_coerce_primitive(coerce_primitive).build(&mut ndjson_reader){Ok(arrow_reader) => {@@ -602,15 +645,17 @@ pub async fn plan_to_json(mod tests {use super::*;use crate::test_utils::{CHUNK_SIZES, make_chunked_store};- use arrow::array::{Int64Array, StringArray};+ use arrow::array::{AsArray, Int64Array, StringArray};use arrow::compute;use arrow::datatypes::{DataType, Field, Schema};use arrow::record_batch::RecordBatch;use bytes::Bytes;+ use datafusion_common::assert_contains;use datafusion_datasource::FileRange;use object_store::memory::InMemory;use object_store::path::Path;use object_store::{ObjectStoreExt, PutPayload};+ use rstest::rstest;use std::fmt::Write as _;/// Helper to create a test schema@@ -819,6 +864,117 @@ mod tests {Ok(())}+ /// Reads the `name` column of `file` with `coerce_primitive` set to `coerce`.+ async fn read_names(+ store: &Arc<dyn ObjectStore>,+ file: PartitionedFile,+ newline_delimited: bool,+ coerce: bool,+ compression: FileCompressionType,+ ) -> Result<Vec<String>> {+ let opener = JsonOpener::new(+ 1024,+ test_schema(),+ compression,+ Arc::clone(store),+ newline_delimited,+ )+ .with_coerce_primitive(coerce);+ let batches: Vec<RecordBatch> = opener.open(file)?.await?.try_collect().await?;+ Ok(batches+ .iter()+ .flat_map(|batch| batch.column(1).as_string::<i32>().iter().flatten())+ .map(str::to_string)+ .collect())+ }++ /// Two rows with a boolean and a number in the string column `name`.+ fn primitive_names_json(newline_delimited: bool) -> &'static str {+ if newline_delimited {+ "{\"id\": 1, \"name\": false}\n{\"id\": 2, \"name\": 3.5}\n"+ } else {+ r#"[{"id": 1, "name": false}, {"id": 2, "name": 3.5}]"#+ }+ }++ /// With `coerce_primitive`, a JSON number or boolean in a string column is kept as its+ /// text; without it, reading fails. Covers each read path: a byte range, a local file,+ /// and a stream, for both newline-delimited JSON and JSON arrays.+ #[rstest]+ #[case::ndjson(true, false)]+ #[case::array(false, false)]+ // Only newline-delimited JSON can be read by byte range.+ #[case::range(true, true)]+ #[tokio::test]+ async fn test_coerce_primitive(+ #[case] newline_delimited: bool,+ #[case] byte_range: bool,+ // `InMemory` returns a stream payload and `LocalFileSystem` a file payload.+ #[values(false, true)] local_file: bool,+ ) -> Result<()> {+ let data = primitive_names_json(newline_delimited);+ let dir = tempfile::tempdir()?;+ let store: Arc<dyn ObjectStore> = if local_file {+ Arc::new(object_store::local::LocalFileSystem::new_with_prefix(+ dir.path(),+ )?)+ } else {+ Arc::new(InMemory::new())+ };+ let mut file = PartitionedFile::new("a.json", data.len() as u64);+ let payload = PutPayload::from_static(data.as_bytes());+ store.put(&file.object_meta.location, payload).await?;+ if byte_range {+ file = file.with_range(0, data.len() as i64);+ }++ let read = |coerce| {+ read_names(+ &store,+ file.clone(),+ newline_delimited,+ coerce,+ FileCompressionType::UNCOMPRESSED,+ )+ };+ assert_eq!(read(true).await?, ["false", "3.5"]);+ assert_contains!(+ read(false).await.unwrap_err().to_string(),+ "expected string got false"+ );+ Ok(())+ }++ /// One-byte chunks split every JSON token, such as `false` and `3.5`, across stream+ /// chunks. For gzip, they split the compressed bytes: the decoded chunks can be larger.+ #[rstest]+ #[tokio::test]+ async fn test_coerce_primitive_chunked_stream(+ #[values(FileCompressionType::UNCOMPRESSED, FileCompressionType::GZIP)]+ compression: FileCompressionType,+ #[values(true, false)] newline_delimited: bool,+ ) -> Result<()> {+ let data = primitive_names_json(newline_delimited);+ let plain = futures::stream::iter([Ok(Bytes::from(data))]).boxed();+ let chunks: Vec<Bytes> = compression+ .convert_to_compress_stream(plain)?+ .try_collect()+ .await?;+ let bytes = chunks.concat();+ let (store, path) = make_chunked_store(&bytes, 1).await;+ let file = PartitionedFile::new(path.to_string(), bytes.len() as u64);++ let read = |coerce| {+ read_names(&store, file.clone(), newline_delimited, coerce, compression)+ };+ assert_eq!(read(true).await?, ["false", "3.5"]);+ assert_contains!(+ read(false).await.unwrap_err().to_string(),+ "expected string got false"+ );+ Ok(())+ }+#[tokio::test]async fn test_json_array_large_file() -> Result<()> {// Test with a larger JSON array to verify streaming worksdiff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex cb28587bc8..64e4be982e 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1411,6 +1411,8 @@ message JsonScanExecNode {FileScanExecConf base_conf = 1;// Absent means newline-delimited JSON for compatibility with older payloads.optional bool newline_delimited = 2;+ // Absent means false: a JSON number or boolean in a string column is an error.+ optional bool coerce_primitive = 3;}message AvroScanExecNode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex 37efc52276..f7814a2780 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -12002,6 +12002,9 @@ impl serde::Serialize for JsonScanExecNode {if self.newline_delimited.is_some() {len += 1;}+ if self.coerce_primitive.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.JsonScanExecNode", len)?;if let Some(v) = self.base_conf.as_ref() {struct_ser.serialize_field("baseConf", v)?;@@ -12009,6 +12012,9 @@ impl serde::Serialize for JsonScanExecNode {if let Some(v) = self.newline_delimited.as_ref() {struct_ser.serialize_field("newlineDelimited", v)?;}+ if let Some(v) = self.coerce_primitive.as_ref() {+ struct_ser.serialize_field("coercePrimitive", v)?;+ }struct_ser.end()}}@@ -12023,12 +12029,15 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {"baseConf","newline_delimited","newlineDelimited",+ "coerce_primitive",+ "coercePrimitive",];#[allow(clippy::enum_variant_names)]enum GeneratedField {BaseConf,NewlineDelimited,+ CoercePrimitive,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -12052,6 +12061,7 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {match value {"baseConf" | "base_conf" => Ok(GeneratedField::BaseConf),"newlineDelimited" | "newline_delimited" => Ok(GeneratedField::NewlineDelimited),+ "coercePrimitive" | "coerce_primitive" => Ok(GeneratedField::CoercePrimitive),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -12073,6 +12083,7 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {{let mut base_conf__ = None;let mut newline_delimited__ = None;+ let mut coerce_primitive__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::BaseConf => {@@ -12087,11 +12098,18 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {}newline_delimited__ = map_.next_value()?;}+ GeneratedField::CoercePrimitive => {+ if coerce_primitive__.is_some() {+ return Err(serde::de::Error::duplicate_field("coercePrimitive"));+ }+ coerce_primitive__ = map_.next_value()?;+ }}}Ok(JsonScanExecNode {base_conf: base_conf__,newline_delimited: newline_delimited__,+ coerce_primitive: coerce_primitive__,})}}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex f64f5277c5..66b28437ee 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -2176,6 +2176,9 @@ pub struct JsonScanExecNode {/// Absent means newline-delimited JSON for compatibility with older payloads.#[prost(bool, optional, tag = "2")]pub newline_delimited: ::core::option::Option<bool>,+ /// Absent means false: a JSON number or boolean in a string column is an error.+ #[prost(bool, optional, tag = "3")]+ pub coerce_primitive: ::core::option::Option<bool>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct AvroScanExecNode {diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rsindex b07cba5ffe..0b9ddad080 100644--- a/datafusion/proto/tests/cases/plans/sources.rs+++ b/datafusion/proto/tests/cases/plans/sources.rs@@ -432,7 +432,9 @@ fn roundtrip_json_scan_preserves_format_options() -> Result<()> {let file_schema =Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));let file_source = Arc::new(- JsonSource::new(TableSchema::from(&file_schema)).with_newline_delimited(false),+ JsonSource::new(TableSchema::from(&file_schema))+ .with_newline_delimited(false)+ .with_coerce_primitive(true),);let scan_config =FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)@@ -461,14 +463,16 @@ fn roundtrip_json_scan_preserves_format_options() -> Result<()> {.downcast_ref::<JsonSource>().ok_or_else(|| internal_datafusion_err!("Expected JsonSource"))?;assert!(!json_source.is_newline_delimited());+ assert!(json_source.coerce_primitive());assert_eq!(file_scan.file_compression_type, FileCompressionType::GZIP);// Payloads written before these fields existed must keep their historical- // defaults: newline-delimited JSON without compression.+ // defaults: newline-delimited JSON without compression or primitive coercion.let mut node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;match node.physical_plan_type.as_mut() {Some(protobuf::physical_plan_node::PhysicalPlanType::JsonScan(scan)) => {scan.newline_delimited = None;+ scan.coerce_primitive = None;scan.base_conf.as_mut().expect("JSON scan has a base config")@@ -486,6 +490,7 @@ fn roundtrip_json_scan_preserves_format_options() -> Result<()> {.downcast_ref::<JsonSource>().ok_or_else(|| internal_datafusion_err!("Expected JsonSource"))?;assert!(json_source.is_newline_delimited());+ assert!(!json_source.coerce_primitive());assert_eq!(file_scan.file_compression_type,FileCompressionType::UNCOMPRESSED