Skip to content

Apache DataFusion SQL Query Engine

feat: add coerce_primitive in json (#26561)

Ion Koutsouris6c429176e589
diff --git a/Cargo.lock b/Cargo.lock
index c55c3e5647..6003cba5d3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2074,7 +2074,9 @@ dependencies = [
"datafusion-session",
"futures",
"object_store",
+ "rstest",
"serde_json",
+ "tempfile",
"tokio",
"tokio-stream",
]
diff --git a/datafusion/core/src/datasource/physical_plan/json.rs b/datafusion/core/src/datasource/physical_plan/json.rs
index 2590f3a8ba..6bebcd0d8e 100644
--- a/datafusion/core/src/datasource/physical_plan/json.rs
+++ b/datafusion/core/src/datasource/physical_plan/json.rs
@@ -600,4 +600,58 @@ mod tests {
);
Ok(())
}
+
+ #[tokio::test]
+ async fn json_exec_coerce_primitive() -> Result<()> {
+ use crate::physical_plan::collect;
+ use crate::test::object_store::local_unpartitioned_file;
+ use arrow::datatypes::{DataType, Schema};
+ use datafusion_common::assert_contains;
+ use datafusion_datasource::PartitionedFile;
+
+ let tmp_dir = TempDir::new()?;
+ let path = tmp_dir.path().join("coerce.json");
+ fs::write(
+ &path,
+ "{\"a\": false}\n{\"a\": 3.5}\n{\"a\": \"text\"}\n{\"a\": null}\n{}\n",
+ )?;
+ let file = PartitionedFile::from(local_unpartitioned_file(&path));
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, true)]));
+
+ let read = |coerce| {
+ let source =
+ JsonSource::new(Arc::clone(&schema)).with_coerce_primitive(coerce);
+ let conf = FileScanConfigBuilder::new(
+ ObjectStoreUrl::local_filesystem(),
+ Arc::new(source),
+ )
+ .with_file(file.clone())
+ .build();
+ collect(
+ DataSourceExec::from_data_source(conf),
+ SessionContext::new().task_ctx(),
+ )
+ };
+
+ let batches = read(true).await?;
+ assert_snapshot!(batches_to_string(&batches), @r"
+ +-------+
+ | a |
+ +-------+
+ | false |
+ | 3.5 |
+ | text |
+ | |
+ | |
+ +-------+
+ ");
+ // The snapshot shows NULL as an empty cell, the same as an empty string.
+ let nulls: usize = batches.iter().map(|b| b.column(0).null_count()).sum();
+ assert_eq!(nulls, 2);
+ assert_contains!(
+ read(false).await.unwrap_err().to_string(),
+ "expected string got false"
+ );
+ Ok(())
+ }
}
diff --git a/datafusion/datasource-json/Cargo.toml b/datafusion/datasource-json/Cargo.toml
index b1c7cc006d..3ed08635bf 100644
--- a/datafusion/datasource-json/Cargo.toml
+++ b/datafusion/datasource-json/Cargo.toml
@@ -50,7 +50,10 @@ tokio-stream = { workspace = true, features = ["sync"] }
workspace = true
[dev-dependencies]
+datafusion-datasource = { workspace = true, features = ["compression"] }
+rstest = { workspace = true }
serde_json = { workspace = true }
+tempfile = { workspace = true }
[lib]
name = "datafusion_datasource_json"
diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rs
index 3b326e3972..c3ca46a170 100644
--- a/datafusion/datasource-json/src/source.rs
+++ b/datafusion/datasource-json/src/source.rs
@@ -92,6 +92,9 @@ pub struct JsonOpener {
/// When `true` (default), expects newline-delimited JSON (NDJSON).
/// When `false`, expects JSON array format `[{...}, {...}]`.
newline_delimited: bool,
+ /// When `true`, a JSON number or boolean read into a string column is kept as its text.
+ /// When `false` (default), it is an error.
+ coerce_primitive: bool,
}
impl JsonOpener {
@@ -109,8 +112,18 @@ impl JsonOpener {
file_compression_type,
object_store,
newline_delimited,
+ coerce_primitive: false,
}
}
+
+ /// Set whether a JSON number or boolean read into a string column is kept as its text.
+ ///
+ /// When `false` (default), it is an error. See
+ /// [`ReaderBuilder::with_coerce_primitive`].
+ pub fn with_coerce_primitive(mut self, coerce_primitive: bool) -> Self {
+ self.coerce_primitive = coerce_primitive;
+ self
+ }
}
/// JsonSource holds the extra configuration that is necessary for [`JsonOpener`]
@@ -123,6 +136,9 @@ pub struct JsonSource {
/// When `true` (default), expects newline-delimited JSON (NDJSON).
/// When `false`, expects JSON array format `[{...}, {...}]`.
newline_delimited: bool,
+ /// When `true`, a JSON number or boolean read into a string column is kept as its text.
+ /// When `false` (default), it is an error.
+ coerce_primitive: bool,
}
impl JsonSource {
@@ -135,9 +151,24 @@ impl JsonSource {
batch_size: None,
metrics: ExecutionPlanMetricsSet::new(),
newline_delimited: true,
+ coerce_primitive: false,
}
}
+ /// Set whether a JSON number or boolean read into a string column is kept as its text.
+ ///
+ /// For example, `{"a": false}` read with `a: Utf8` gives `"false"`. When `false`
+ /// (default), it is an error. See [`ReaderBuilder::with_coerce_primitive`].
+ pub fn with_coerce_primitive(mut self, coerce_primitive: bool) -> Self {
+ self.coerce_primitive = coerce_primitive;
+ self
+ }
+
+ /// Returns whether a JSON number or boolean read into a string column is kept as its text.
+ pub fn coerce_primitive(&self) -> bool {
+ self.coerce_primitive
+ }
+
/// Set whether to read as newline-delimited JSON.
///
/// When `true` (default), expects newline-delimited format.
@@ -179,6 +210,7 @@ impl FileSource for JsonSource {
file_compression_type: base_config.file_compression_type,
object_store,
newline_delimited: self.newline_delimited,
+ coerce_primitive: self.coerce_primitive,
}) as Arc<dyn FileOpener>;
// Wrap with ProjectionOpener
@@ -257,6 +289,7 @@ impl FileSource for JsonSource {
// Serialized in `base` and reapplied on decode.
projection: _,
newline_delimited,
+ coerce_primitive,
} = self;
let node = protobuf::JsonScanExecNode {
@@ -266,6 +299,7 @@ impl FileSource for JsonSource {
} else {
Some(false)
},
+ coerce_primitive: coerce_primitive.then_some(true),
};
Ok(Some(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::JsonScan(node)),
@@ -277,7 +311,8 @@ impl FileSource for JsonSource {
impl JsonSource {
/// Reconstructs a `DataSourceExec` from a protobuf `JsonScan`.
///
- /// Payloads without a mode default to newline-delimited JSON.
+ /// Payloads without a mode default to newline-delimited JSON, and payloads without
+ /// `coerce_primitive` default to `false`.
pub fn try_from_proto(
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
@@ -299,6 +334,7 @@ impl JsonSource {
let protobuf::JsonScanExecNode {
base_conf,
newline_delimited,
+ coerce_primitive,
} = scan;
let base_conf = base_conf.as_ref().ok_or_else(|| {
@@ -310,7 +346,8 @@ impl JsonSource {
let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?;
let source = Arc::new(
JsonSource::new(table_schema)
- .with_newline_delimited(newline_delimited.unwrap_or(true)),
+ .with_newline_delimited(newline_delimited.unwrap_or(true))
+ .with_coerce_primitive(coerce_primitive.unwrap_or(false)),
);
let conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?;
@@ -336,6 +373,7 @@ impl FileOpener for JsonOpener {
let batch_size = self.batch_size;
let file_compression_type = self.file_compression_type.to_owned();
let newline_delimited = self.newline_delimited;
+ let coerce_primitive = self.coerce_primitive;
// JSON array format requires reading the complete file
if !newline_delimited && partitioned_file.range.is_some() {
@@ -377,6 +415,7 @@ impl FileOpener for JsonOpener {
let decoder = ReaderBuilder::new(schema)
.with_batch_size(batch_size)
+ .with_coerce_primitive(coerce_primitive)
.build_decoder()?;
let input = file_compression_type
.convert_stream(aligned_stream.boxed())?
@@ -402,6 +441,7 @@ impl FileOpener for JsonOpener {
let reader = BufReader::new(bytes);
let arrow_reader = ReaderBuilder::new(schema)
.with_batch_size(batch_size)
+ .with_coerce_primitive(coerce_primitive)
.build(reader)?;
Ok(futures::stream::iter(arrow_reader)
@@ -415,6 +455,7 @@ impl FileOpener for JsonOpener {
);
let arrow_reader = ReaderBuilder::new(schema)
.with_batch_size(batch_size)
+ .with_coerce_primitive(coerce_primitive)
.build(ndjson_reader)?;
Ok(futures::stream::iter(arrow_reader)
@@ -428,6 +469,7 @@ impl FileOpener for JsonOpener {
let s = s.map_err(DataFusionError::from);
let decoder = ReaderBuilder::new(schema)
.with_batch_size(batch_size)
+ .with_coerce_primitive(coerce_primitive)
.build_decoder()?;
let input =
file_compression_type.convert_stream(s.boxed())?.fuse();
@@ -504,6 +546,7 @@ impl FileOpener for JsonOpener {
match ReaderBuilder::new(schema)
.with_batch_size(batch_size)
+ .with_coerce_primitive(coerce_primitive)
.build(&mut ndjson_reader)
{
Ok(arrow_reader) => {
@@ -602,15 +645,17 @@ pub async fn plan_to_json(
mod tests {
use super::*;
use crate::test_utils::{CHUNK_SIZES, make_chunked_store};
- use arrow::array::{Int64Array, StringArray};
+ use arrow::array::{AsArray, Int64Array, StringArray};
use arrow::compute;
use arrow::datatypes::{DataType, Field, Schema};
use arrow::record_batch::RecordBatch;
use bytes::Bytes;
+ use datafusion_common::assert_contains;
use datafusion_datasource::FileRange;
use object_store::memory::InMemory;
use object_store::path::Path;
use object_store::{ObjectStoreExt, PutPayload};
+ use rstest::rstest;
use std::fmt::Write as _;
/// Helper to create a test schema
@@ -819,6 +864,117 @@ mod tests {
Ok(())
}
+ /// Reads the `name` column of `file` with `coerce_primitive` set to `coerce`.
+ async fn read_names(
+ store: &Arc<dyn ObjectStore>,
+ file: PartitionedFile,
+ newline_delimited: bool,
+ coerce: bool,
+ compression: FileCompressionType,
+ ) -> Result<Vec<String>> {
+ let opener = JsonOpener::new(
+ 1024,
+ test_schema(),
+ compression,
+ Arc::clone(store),
+ newline_delimited,
+ )
+ .with_coerce_primitive(coerce);
+ let batches: Vec<RecordBatch> = opener.open(file)?.await?.try_collect().await?;
+ Ok(batches
+ .iter()
+ .flat_map(|batch| batch.column(1).as_string::<i32>().iter().flatten())
+ .map(str::to_string)
+ .collect())
+ }
+
+ /// Two rows with a boolean and a number in the string column `name`.
+ fn primitive_names_json(newline_delimited: bool) -> &'static str {
+ if newline_delimited {
+ "{\"id\": 1, \"name\": false}\n{\"id\": 2, \"name\": 3.5}\n"
+ } else {
+ r#"[{"id": 1, "name": false}, {"id": 2, "name": 3.5}]"#
+ }
+ }
+
+ /// With `coerce_primitive`, a JSON number or boolean in a string column is kept as its
+ /// text; without it, reading fails. Covers each read path: a byte range, a local file,
+ /// and a stream, for both newline-delimited JSON and JSON arrays.
+ #[rstest]
+ #[case::ndjson(true, false)]
+ #[case::array(false, false)]
+ // Only newline-delimited JSON can be read by byte range.
+ #[case::range(true, true)]
+ #[tokio::test]
+ async fn test_coerce_primitive(
+ #[case] newline_delimited: bool,
+ #[case] byte_range: bool,
+ // `InMemory` returns a stream payload and `LocalFileSystem` a file payload.
+ #[values(false, true)] local_file: bool,
+ ) -> Result<()> {
+ let data = primitive_names_json(newline_delimited);
+ let dir = tempfile::tempdir()?;
+ let store: Arc<dyn ObjectStore> = if local_file {
+ Arc::new(object_store::local::LocalFileSystem::new_with_prefix(
+ dir.path(),
+ )?)
+ } else {
+ Arc::new(InMemory::new())
+ };
+ let mut file = PartitionedFile::new("a.json", data.len() as u64);
+ let payload = PutPayload::from_static(data.as_bytes());
+ store.put(&file.object_meta.location, payload).await?;
+ if byte_range {
+ file = file.with_range(0, data.len() as i64);
+ }
+
+ let read = |coerce| {
+ read_names(
+ &store,
+ file.clone(),
+ newline_delimited,
+ coerce,
+ FileCompressionType::UNCOMPRESSED,
+ )
+ };
+ assert_eq!(read(true).await?, ["false", "3.5"]);
+ assert_contains!(
+ read(false).await.unwrap_err().to_string(),
+ "expected string got false"
+ );
+ Ok(())
+ }
+
+ /// One-byte chunks split every JSON token, such as `false` and `3.5`, across stream
+ /// chunks. For gzip, they split the compressed bytes: the decoded chunks can be larger.
+ #[rstest]
+ #[tokio::test]
+ async fn test_coerce_primitive_chunked_stream(
+ #[values(FileCompressionType::UNCOMPRESSED, FileCompressionType::GZIP)]
+ compression: FileCompressionType,
+ #[values(true, false)] newline_delimited: bool,
+ ) -> Result<()> {
+ let data = primitive_names_json(newline_delimited);
+ let plain = futures::stream::iter([Ok(Bytes::from(data))]).boxed();
+ let chunks: Vec<Bytes> = compression
+ .convert_to_compress_stream(plain)?
+ .try_collect()
+ .await?;
+ let bytes = chunks.concat();
+ let (store, path) = make_chunked_store(&bytes, 1).await;
+ let file = PartitionedFile::new(path.to_string(), bytes.len() as u64);
+
+ let read = |coerce| {
+ read_names(&store, file.clone(), newline_delimited, coerce, compression)
+ };
+ assert_eq!(read(true).await?, ["false", "3.5"]);
+ assert_contains!(
+ read(false).await.unwrap_err().to_string(),
+ "expected string got false"
+ );
+ Ok(())
+ }
+
#[tokio::test]
async fn test_json_array_large_file() -> Result<()> {
// Test with a larger JSON array to verify streaming works
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index cb28587bc8..64e4be982e 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -1411,6 +1411,8 @@ message JsonScanExecNode {
FileScanExecConf base_conf = 1;
// Absent means newline-delimited JSON for compatibility with older payloads.
optional bool newline_delimited = 2;
+ // Absent means false: a JSON number or boolean in a string column is an error.
+ optional bool coerce_primitive = 3;
}
message AvroScanExecNode {
diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs
index 37efc52276..f7814a2780 100644
--- a/datafusion/proto-models/src/generated/pbjson.rs
+++ b/datafusion/proto-models/src/generated/pbjson.rs
@@ -12002,6 +12002,9 @@ impl serde::Serialize for JsonScanExecNode {
if self.newline_delimited.is_some() {
len += 1;
}
+ if self.coerce_primitive.is_some() {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.JsonScanExecNode", len)?;
if let Some(v) = self.base_conf.as_ref() {
struct_ser.serialize_field("baseConf", v)?;
@@ -12009,6 +12012,9 @@ impl serde::Serialize for JsonScanExecNode {
if let Some(v) = self.newline_delimited.as_ref() {
struct_ser.serialize_field("newlineDelimited", v)?;
}
+ if let Some(v) = self.coerce_primitive.as_ref() {
+ struct_ser.serialize_field("coercePrimitive", v)?;
+ }
struct_ser.end()
}
}
@@ -12023,12 +12029,15 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {
"baseConf",
"newline_delimited",
"newlineDelimited",
+ "coerce_primitive",
+ "coercePrimitive",
];
#[allow(clippy::enum_variant_names)]
enum GeneratedField {
BaseConf,
NewlineDelimited,
+ CoercePrimitive,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -12052,6 +12061,7 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {
match value {
"baseConf" | "base_conf" => Ok(GeneratedField::BaseConf),
"newlineDelimited" | "newline_delimited" => Ok(GeneratedField::NewlineDelimited),
+ "coercePrimitive" | "coerce_primitive" => Ok(GeneratedField::CoercePrimitive),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -12073,6 +12083,7 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {
{
let mut base_conf__ = None;
let mut newline_delimited__ = None;
+ let mut coerce_primitive__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::BaseConf => {
@@ -12087,11 +12098,18 @@ impl<'de> serde::Deserialize<'de> for JsonScanExecNode {
}
newline_delimited__ = map_.next_value()?;
}
+ GeneratedField::CoercePrimitive => {
+ if coerce_primitive__.is_some() {
+ return Err(serde::de::Error::duplicate_field("coercePrimitive"));
+ }
+ coerce_primitive__ = map_.next_value()?;
+ }
}
}
Ok(JsonScanExecNode {
base_conf: base_conf__,
newline_delimited: newline_delimited__,
+ coerce_primitive: coerce_primitive__,
})
}
}
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index f64f5277c5..66b28437ee 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -2176,6 +2176,9 @@ pub struct JsonScanExecNode {
/// Absent means newline-delimited JSON for compatibility with older payloads.
#[prost(bool, optional, tag = "2")]
pub newline_delimited: ::core::option::Option<bool>,
+ /// Absent means false: a JSON number or boolean in a string column is an error.
+ #[prost(bool, optional, tag = "3")]
+ pub coerce_primitive: ::core::option::Option<bool>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct AvroScanExecNode {
diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rs
index b07cba5ffe..0b9ddad080 100644
--- a/datafusion/proto/tests/cases/plans/sources.rs
+++ b/datafusion/proto/tests/cases/plans/sources.rs
@@ -432,7 +432,9 @@ fn roundtrip_json_scan_preserves_format_options() -> Result<()> {
let file_schema =
Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));
let file_source = Arc::new(
- JsonSource::new(TableSchema::from(&file_schema)).with_newline_delimited(false),
+ JsonSource::new(TableSchema::from(&file_schema))
+ .with_newline_delimited(false)
+ .with_coerce_primitive(true),
);
let scan_config =
FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
@@ -461,14 +463,16 @@ fn roundtrip_json_scan_preserves_format_options() -> Result<()> {
.downcast_ref::<JsonSource>()
.ok_or_else(|| internal_datafusion_err!("Expected JsonSource"))?;
assert!(!json_source.is_newline_delimited());
+ assert!(json_source.coerce_primitive());
assert_eq!(file_scan.file_compression_type, FileCompressionType::GZIP);
// Payloads written before these fields existed must keep their historical
- // defaults: newline-delimited JSON without compression.
+ // defaults: newline-delimited JSON without compression or primitive coercion.
let mut node = PhysicalPlanNode::try_from_physical_plan(plan, &codec)?;
match node.physical_plan_type.as_mut() {
Some(protobuf::physical_plan_node::PhysicalPlanType::JsonScan(scan)) => {
scan.newline_delimited = None;
+ scan.coerce_primitive = None;
scan.base_conf
.as_mut()
.expect("JSON scan has a base config")
@@ -486,6 +490,7 @@ fn roundtrip_json_scan_preserves_format_options() -> Result<()> {
.downcast_ref::<JsonSource>()
.ok_or_else(|| internal_datafusion_err!("Expected JsonSource"))?;
assert!(json_source.is_newline_delimited());
+ assert!(!json_source.coerce_primitive());
assert_eq!(
file_scan.file_compression_type,
FileCompressionType::UNCOMPRESSED