Apache DataFusion SQL Query Engine
Support JSON arrays reader/parse for datafusion (#19924)
Qi Zhu69d0f44cc408
diff --git a/Cargo.lock b/Cargo.lockindex 472b3e84d6..7757f86a6a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2042,7 +2042,9 @@ dependencies = ["datafusion-session","futures","object_store",+ "serde_json","tokio",+ "tokio-stream",][[package]]@@ -6167,6 +6169,7 @@ dependencies = ["futures-core","pin-project-lite","tokio",+ "tokio-util",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex 157dd68b9c..1731f07ea2 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -188,6 +188,8 @@ strum_macros = "0.27.2"tempfile = "3"testcontainers-modules = { version = "0.14" }tokio = { version = "1.48", features = ["macros", "rt", "sync"] }+tokio-stream = "0.1"+tokio-util = "0.7"url = "2.5.7"uuid = "1.20"zstd = { version = "0.13", default-features = false }diff --git a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs b/datafusion-examples/examples/custom_data_source/csv_json_opener.rsindex 347f1a0464..fc1130313e 100644--- a/datafusion-examples/examples/custom_data_source/csv_json_opener.rs+++ b/datafusion-examples/examples/custom_data_source/csv_json_opener.rs@@ -125,6 +125,7 @@ async fn json_opener() -> Result<()> {projected,FileCompressionType::UNCOMPRESSED,Arc::new(object_store),+ true,);let scan_config = FileScanConfigBuilder::new(diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex a18861aa3a..0ba587bbc6 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -3065,6 +3065,22 @@ config_namespace! {/// If not specified, the default level for the compression algorithm is used.pub compression_level: Option<u32>, default = Nonepub schema_infer_max_rec: Option<usize>, default = None+ /// The JSON format to use when reading files.+ ///+ /// When `true` (default), expects newline-delimited JSON (NDJSON):+ /// ```text+ /// {"key1": 1, "key2": "val"}+ /// {"key1": 2, "key2": "vals"}+ /// ```+ ///+ /// When `false`, expects JSON array format:+ /// ```text+ /// [+ /// {"key1": 1, "key2": "val"},+ /// {"key1": 2, "key2": "vals"}+ /// ]+ /// ```+ pub newline_delimited: bool, default = true}}diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex fadc6ad792..2292f5855b 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -512,7 +512,7 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_json("tests/data/unnest.json", NdJsonReadOptions::default()).await?;+ /// let df = ctx.read_json("tests/data/unnest.json", JsonReadOptions::default()).await?;/// // expand into multiple columns if it's json array, flatten field name if it's nested structure/// let df = df.unnest_columns(&["b","c","d"])?;/// let expected = vec![diff --git a/datafusion/core/src/datasource/file_format/json.rs b/datafusion/core/src/datasource/file_format/json.rsindex cb2e9d787e..5b3e227056 100644--- a/datafusion/core/src/datasource/file_format/json.rs+++ b/datafusion/core/src/datasource/file_format/json.rs@@ -25,7 +25,7 @@ mod tests {use super::*;use crate::datasource::file_format::test_util::scan_format;- use crate::prelude::{NdJsonReadOptions, SessionConfig, SessionContext};+ use crate::prelude::{SessionConfig, SessionContext};use crate::test::object_store::local_unpartitioned_file;use arrow::array::RecordBatch;use arrow_schema::Schema;@@ -46,12 +46,54 @@ mod tests {use datafusion_common::internal_err;use datafusion_common::stats::Precision;+ use crate::execution::options::JsonReadOptions;use datafusion_common::Result;+ use datafusion_datasource::file_compression_type::FileCompressionType;use futures::StreamExt;use insta::assert_snapshot;use object_store::local::LocalFileSystem;use regex::Regex;use rstest::rstest;+ // ==================== Test Helpers ====================++ /// Create a temporary JSON file and return (TempDir, path)+ fn create_temp_json(content: &str) -> (tempfile::TempDir, String) {+ let tmp_dir = tempfile::TempDir::new().unwrap();+ let path = tmp_dir.path().join("test.json");+ std::fs::write(&path, content).unwrap();+ (tmp_dir, path.to_string_lossy().to_string())+ }++ /// Infer schema from JSON array format file+ async fn infer_json_array_schema(+ content: &str,+ ) -> Result<arrow::datatypes::SchemaRef> {+ let (_tmp_dir, path) = create_temp_json(content);+ let session = SessionContext::new();+ let ctx = session.state();+ let store = Arc::new(LocalFileSystem::new()) as _;+ let format = JsonFormat::default().with_newline_delimited(false);+ format+ .infer_schema(&ctx, &store, &[local_unpartitioned_file(&path)])+ .await+ }++ /// Register a JSON array table and run a query+ async fn query_json_array(content: &str, query: &str) -> Result<Vec<RecordBatch>> {+ let (_tmp_dir, path) = create_temp_json(content);+ let ctx = SessionContext::new();+ let options = JsonReadOptions::default().newline_delimited(false);+ ctx.register_json("test_table", &path, options).await?;+ ctx.sql(query).await?.collect().await+ }++ /// Register a JSON array table and run a query, return formatted string+ async fn query_json_array_str(content: &str, query: &str) -> Result<String> {+ let result = query_json_array(content, query).await?;+ Ok(batches_to_string(&result))+ }++ // ==================== Existing Tests ====================#[tokio::test]async fn read_small_batches() -> Result<()> {@@ -208,7 +250,7 @@ mod tests {let ctx = SessionContext::new_with_config(config);let table_path = "tests/data/1.json";- let options = NdJsonReadOptions::default();+ let options = JsonReadOptions::default();ctx.register_json("json_parallel", table_path, options).await?;@@ -240,7 +282,7 @@ mod tests {let ctx = SessionContext::new_with_config(config);let table_path = "tests/data/empty.json";- let options = NdJsonReadOptions::default();+ let options = JsonReadOptions::default();ctx.register_json("json_parallel_empty", table_path, options).await?;@@ -314,7 +356,6 @@ mod tests {.digest(r#"{ "c1": 11, "c2": 12, "c3": 13, "c4": 14, "c5": 15 }"#.into());let mut all_batches = RecordBatch::new_empty(schema.clone());- // We get RequiresMoreData after 2 batches because of how json::Decoder worksfor _ in 0..2 {let output = deserializer.next()?;let DeserializerOutput::RecordBatch(batch) = output else {@@ -354,11 +395,11 @@ mod tests {async fn test_write_empty_json_from_sql() -> Result<()> {let ctx = SessionContext::new();let tmp_dir = tempfile::TempDir::new()?;- let path = format!("{}/empty_sql.json", tmp_dir.path().to_string_lossy());+ let path = tmp_dir.path().join("empty_sql.json");+ let path = path.to_string_lossy().to_string();let df = ctx.sql("SELECT CAST(1 AS BIGINT) AS id LIMIT 0").await?;df.write_json(&path, crate::dataframe::DataFrameWriteOptions::new(), None).await?;- // Expected the file to exist and be emptyassert!(std::path::Path::new(&path).exists());let metadata = std::fs::metadata(&path)?;assert_eq!(metadata.len(), 0);@@ -381,14 +422,216 @@ mod tests {)?;let tmp_dir = tempfile::TempDir::new()?;- let path = format!("{}/empty_batch.json", tmp_dir.path().to_string_lossy());+ let path = tmp_dir.path().join("empty_batch.json");+ let path = path.to_string_lossy().to_string();let df = ctx.read_batch(empty_batch.clone())?;df.write_json(&path, crate::dataframe::DataFrameWriteOptions::new(), None).await?;- // Expected the file to exist and be emptyassert!(std::path::Path::new(&path).exists());let metadata = std::fs::metadata(&path)?;assert_eq!(metadata.len(), 0);Ok(())}++ // ==================== JSON Array Format Tests ====================++ #[tokio::test]+ async fn test_json_array_schema_inference() -> Result<()> {+ let schema = infer_json_array_schema(+ r#"[{"a": 1, "b": 2.0, "c": true}, {"a": 2, "b": 3.5, "c": false}]"#,+ )+ .await?;++ let fields: Vec<_> = schema+ .fields()+ .iter()+ .map(|f| format!("{}: {:?}", f.name(), f.data_type()))+ .collect();+ assert_eq!(vec!["a: Int64", "b: Float64", "c: Boolean"], fields);+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_empty() -> Result<()> {+ let schema = infer_json_array_schema("[]").await?;+ assert_eq!(schema.fields().len(), 0);+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_nested_struct() -> Result<()> {+ let schema = infer_json_array_schema(+ r#"[{"id": 1, "info": {"name": "Alice", "age": 30}}]"#,+ )+ .await?;++ let info_field = schema.field_with_name("info").unwrap();+ assert!(matches!(info_field.data_type(), DataType::Struct(_)));+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_list_type() -> Result<()> {+ let schema =+ infer_json_array_schema(r#"[{"id": 1, "tags": ["a", "b", "c"]}]"#).await?;++ let tags_field = schema.field_with_name("tags").unwrap();+ assert!(matches!(tags_field.data_type(), DataType::List(_)));+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_basic_query() -> Result<()> {+ let result = query_json_array_str(+ r#"[{"a": 1, "b": "hello"}, {"a": 2, "b": "world"}, {"a": 3, "b": "test"}]"#,+ "SELECT a, b FROM test_table ORDER BY a",+ )+ .await?;++ assert_snapshot!(result, @r"+ +---+-------++ | a | b |+ +---+-------++ | 1 | hello |+ | 2 | world |+ | 3 | test |+ +---+-------++ ");+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_with_nulls() -> Result<()> {+ let result = query_json_array_str(+ r#"[{"id": 1, "name": "Alice"}, {"id": 2, "name": null}, {"id": 3, "name": "Charlie"}]"#,+ "SELECT id, name FROM test_table ORDER BY id",+ )+ .await?;++ assert_snapshot!(result, @r"+ +----+---------++ | id | name |+ +----+---------++ | 1 | Alice |+ | 2 | |+ | 3 | Charlie |+ +----+---------++ ");+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_unnest() -> Result<()> {+ let result = query_json_array_str(+ r#"[{"id": 1, "values": [10, 20, 30]}, {"id": 2, "values": [40, 50]}]"#,+ "SELECT id, unnest(values) as value FROM test_table ORDER BY id, value",+ )+ .await?;++ assert_snapshot!(result, @r"+ +----+-------++ | id | value |+ +----+-------++ | 1 | 10 |+ | 1 | 20 |+ | 1 | 30 |+ | 2 | 40 |+ | 2 | 50 |+ +----+-------++ ");+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_unnest_struct() -> Result<()> {+ let result = query_json_array_str(+ r#"[{"id": 1, "orders": [{"product": "A", "qty": 2}, {"product": "B", "qty": 3}]}, {"id": 2, "orders": [{"product": "C", "qty": 1}]}]"#,+ "SELECT id, unnest(orders)['product'] as product, unnest(orders)['qty'] as qty FROM test_table ORDER BY id, product",+ )+ .await?;++ assert_snapshot!(result, @r"+ +----+---------+-----++ | id | product | qty |+ +----+---------+-----++ | 1 | A | 2 |+ | 1 | B | 3 |+ | 2 | C | 1 |+ +----+---------+-----++ ");+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_nested_struct_access() -> Result<()> {+ let result = query_json_array_str(+ r#"[{"id": 1, "dept": {"name": "Engineering", "head": "Alice"}}, {"id": 2, "dept": {"name": "Sales", "head": "Bob"}}]"#,+ "SELECT id, dept['name'] as dept_name, dept['head'] as head FROM test_table ORDER BY id",+ )+ .await?;++ assert_snapshot!(result, @r"+ +----+-------------+-------++ | id | dept_name | head |+ +----+-------------+-------++ | 1 | Engineering | Alice |+ | 2 | Sales | Bob |+ +----+-------------+-------++ ");+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_with_compression() -> Result<()> {+ use flate2::Compression;+ use flate2::write::GzEncoder;+ use std::io::Write;++ let tmp_dir = tempfile::TempDir::new()?;+ let path = tmp_dir.path().join("array.json.gz");+ let path = path.to_string_lossy().to_string();++ let file = std::fs::File::create(&path)?;+ let mut encoder = GzEncoder::new(file, Compression::default());+ encoder.write_all(+ r#"[{"a": 1, "b": "hello"}, {"a": 2, "b": "world"}]"#.as_bytes(),+ )?;+ encoder.finish()?;++ let ctx = SessionContext::new();+ let options = JsonReadOptions::default()+ .newline_delimited(false)+ .file_compression_type(FileCompressionType::GZIP)+ .file_extension(".json.gz");++ ctx.register_json("test_table", &path, options).await?;+ let result = ctx+ .sql("SELECT a, b FROM test_table ORDER BY a")+ .await?+ .collect()+ .await?;++ assert_snapshot!(batches_to_string(&result), @r"+ +---+-------++ | a | b |+ +---+-------++ | 1 | hello |+ | 2 | world |+ +---+-------++ ");+ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_list_of_structs() -> Result<()> {+ let batches = query_json_array(+ r#"[{"id": 1, "items": [{"name": "x", "price": 10.5}]}, {"id": 2, "items": []}]"#,+ "SELECT id, items FROM test_table ORDER BY id",+ )+ .await?;++ assert_eq!(1, batches.len());+ assert_eq!(2, batches[0].num_rows());+ Ok(())+ }}diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rsindex 146c5f6f5f..bd0ac36087 100644--- a/datafusion/core/src/datasource/file_format/options.rs+++ b/datafusion/core/src/datasource/file_format/options.rs@@ -442,14 +442,23 @@ impl<'a> AvroReadOptions<'a> {}}-/// Options that control the reading of Line-delimited JSON files (NDJson)+#[deprecated(+ since = "53.0.0",+ note = "Use `JsonReadOptions` instead. This alias will be removed in a future version."+)]+#[doc = "Deprecated: Use [`JsonReadOptions`] instead."]+pub type NdJsonReadOptions<'a> = JsonReadOptions<'a>;++/// Options that control the reading of JSON files.+///+/// Supports both newline-delimited JSON (NDJSON) and JSON array formats.////// Note this structure is supplied when a datasource is created and-/// can not not vary from statement to statement. For settings that+/// can not vary from statement to statement. For settings that/// can vary statement to statement see/// [`ConfigOptions`](crate::config::ConfigOptions).#[derive(Clone)]-pub struct NdJsonReadOptions<'a> {+pub struct JsonReadOptions<'a> {/// The data source schema.pub schema: Option<&'a Schema>,/// Max number of rows to read from JSON files for schema inference if needed. Defaults to `DEFAULT_SCHEMA_INFER_MAX_RECORD`.@@ -465,9 +474,25 @@ pub struct NdJsonReadOptions<'a> {pub infinite: bool,/// Indicates how the file is sortedpub file_sort_order: Vec<Vec<SortExpr>>,+ /// Whether to read as newline-delimited JSON (default: true).+ ///+ /// When `true` (default), expects newline-delimited JSON (NDJSON):+ /// ```text+ /// {"key1": 1, "key2": "val"}+ /// {"key1": 2, "key2": "vals"}+ /// ```+ ///+ /// When `false`, expects JSON array format:+ /// ```text+ /// [+ /// {"key1": 1, "key2": "val"},+ /// {"key1": 2, "key2": "vals"}+ /// ]+ /// ```+ pub newline_delimited: bool,}-impl Default for NdJsonReadOptions<'_> {+impl Default for JsonReadOptions<'_> {fn default() -> Self {Self {schema: None,@@ -477,11 +502,12 @@ impl Default for NdJsonReadOptions<'_> {file_compression_type: FileCompressionType::UNCOMPRESSED,infinite: false,file_sort_order: vec![],+ newline_delimited: true,}}}-impl<'a> NdJsonReadOptions<'a> {+impl<'a> JsonReadOptions<'a> {/// Specify table_partition_cols for partition pruningpub fn table_partition_cols(mut self,@@ -529,6 +555,26 @@ impl<'a> NdJsonReadOptions<'a> {self.schema_infer_max_records = schema_infer_max_records;self}++ /// Set whether to read as newline-delimited JSON.+ ///+ /// When `true` (default), expects newline-delimited JSON (NDJSON):+ /// ```text+ /// {"key1": 1, "key2": "val"}+ /// {"key1": 2, "key2": "vals"}+ /// ```+ ///+ /// When `false`, expects JSON array format:+ /// ```text+ /// [+ /// {"key1": 1, "key2": "val"},+ /// {"key1": 2, "key2": "vals"}+ /// ]+ /// ```+ pub fn newline_delimited(mut self, newline_delimited: bool) -> Self {+ self.newline_delimited = newline_delimited;+ self+ }}#[async_trait]@@ -654,7 +700,7 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> {}#[async_trait]-impl ReadOptions<'_> for NdJsonReadOptions<'_> {+impl ReadOptions<'_> for JsonReadOptions<'_> {fn to_listing_options(&self,config: &SessionConfig,@@ -663,7 +709,8 @@ impl ReadOptions<'_> for NdJsonReadOptions<'_> {let file_format = JsonFormat::default().with_options(table_options.json).with_schema_infer_max_rec(self.schema_infer_max_records)- .with_file_compression_type(self.file_compression_type.to_owned());+ .with_file_compression_type(self.file_compression_type.to_owned())+ .with_newline_delimited(self.newline_delimited);ListingOptions::new(Arc::new(file_format)).with_file_extension(self.file_extension)diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex 4e33f3cad5..5dd11739c1 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -110,6 +110,7 @@ mod tests {#[cfg(feature = "parquet")]use crate::datasource::file_format::parquet::ParquetFormat;use crate::datasource::listing::table::ListingTableConfigExt;+ use crate::execution::options::JsonReadOptions;use crate::prelude::*;use crate::{datasource::{@@ -808,7 +809,7 @@ mod tests {.register_json("t",tmp_dir.path().to_str().unwrap(),- NdJsonReadOptions::default()+ JsonReadOptions::default().schema(schema.as_ref()).file_compression_type(file_compression_type),)diff --git a/datafusion/core/src/datasource/physical_plan/json.rs b/datafusion/core/src/datasource/physical_plan/json.rsindex 8de6a60258..b70791c7b2 100644--- a/datafusion/core/src/datasource/physical_plan/json.rs+++ b/datafusion/core/src/datasource/physical_plan/json.rs@@ -32,7 +32,7 @@ mod tests {use crate::dataframe::DataFrameWriteOptions;use crate::execution::SessionState;- use crate::prelude::{CsvReadOptions, NdJsonReadOptions, SessionContext};+ use crate::prelude::{CsvReadOptions, JsonReadOptions, SessionContext};use crate::test::partitioned_file_groups;use datafusion_common::Result;use datafusion_common::cast::{as_int32_array, as_int64_array, as_string_array};@@ -136,7 +136,7 @@ mod tests {.get_ext_with_compression(&file_compression_type).unwrap();- let read_options = NdJsonReadOptions::default()+ let read_options = JsonReadOptions::default().file_extension(ext.as_str()).file_compression_type(file_compression_type.to_owned());let frame = ctx.read_json(path, read_options).await.unwrap();@@ -389,7 +389,7 @@ mod tests {let path = format!("{TEST_DATA_BASE}/1.json");// register json file with the execution context- ctx.register_json("test", path.as_str(), NdJsonReadOptions::default())+ ctx.register_json("test", path.as_str(), JsonReadOptions::default()).await?;// register a local file system object store for /tmp directory@@ -431,7 +431,7 @@ mod tests {}// register each partition as well as the top level dir- let json_read_option = NdJsonReadOptions::default();+ let json_read_option = JsonReadOptions::default();ctx.register_json("part0",&format!("{out_dir}/{part_0_name}"),@@ -511,7 +511,7 @@ mod tests {async fn read_test_data(schema_infer_max_records: usize) -> Result<SchemaRef> {let ctx = SessionContext::new();- let options = NdJsonReadOptions {+ let options = JsonReadOptions {schema_infer_max_records,..Default::default()};@@ -587,7 +587,7 @@ mod tests {.get_ext_with_compression(&file_compression_type).unwrap();- let read_option = NdJsonReadOptions::default()+ let read_option = JsonReadOptions::default().file_compression_type(file_compression_type).file_extension(ext.as_str());diff --git a/datafusion/core/src/execution/context/json.rs b/datafusion/core/src/execution/context/json.rsindex e9d7994008..f7df2ad7a1 100644--- a/datafusion/core/src/execution/context/json.rs+++ b/datafusion/core/src/execution/context/json.rs@@ -15,13 +15,13 @@// specific language governing permissions and limitations// under the License.+use super::super::options::ReadOptions;+use super::{DataFilePaths, DataFrame, ExecutionPlan, Result, SessionContext};+use crate::execution::options::JsonReadOptions;use datafusion_common::TableReference;use datafusion_datasource_json::source::plan_to_json;use std::sync::Arc;-use super::super::options::{NdJsonReadOptions, ReadOptions};-use super::{DataFilePaths, DataFrame, ExecutionPlan, Result, SessionContext};-impl SessionContext {/// Creates a [`DataFrame`] for reading an JSON data source.///@@ -32,7 +32,7 @@ impl SessionContext {pub async fn read_json<P: DataFilePaths>(&self,table_paths: P,- options: NdJsonReadOptions<'_>,+ options: JsonReadOptions<'_>,) -> Result<DataFrame> {self._read_type(table_paths, options).await}@@ -43,7 +43,7 @@ impl SessionContext {&self,table_ref: impl Into<TableReference>,table_path: impl AsRef<str>,- options: NdJsonReadOptions<'_>,+ options: JsonReadOptions<'_>,) -> Result<()> {let listing_options = options.to_listing_options(&self.copied_config(), self.copied_table_options());diff --git a/datafusion/core/src/prelude.rs b/datafusion/core/src/prelude.rsindex 50e4a2649c..31d9d7eb47 100644--- a/datafusion/core/src/prelude.rs+++ b/datafusion/core/src/prelude.rs@@ -29,7 +29,7 @@ pub use crate::dataframe;pub use crate::dataframe::DataFrame;pub use crate::execution::context::{SQLOptions, SessionConfig, SessionContext};pub use crate::execution::options::{- AvroReadOptions, CsvReadOptions, NdJsonReadOptions, ParquetReadOptions,+ AvroReadOptions, CsvReadOptions, JsonReadOptions, ParquetReadOptions,};pub use datafusion_common::Column;diff --git a/datafusion/core/tests/data/json_array.json b/datafusion/core/tests/data/json_array.jsonnew file mode 100644index 0000000000..1a8716dbf4--- /dev/null+++ b/datafusion/core/tests/data/json_array.json@@ -0,0 +1,5 @@+[+ {"a": 1, "b": "hello"},+ {"a": 2, "b": "world"},+ {"a": 3, "b": "test"}+]diff --git a/datafusion/core/tests/data/json_empty_array.json b/datafusion/core/tests/data/json_empty_array.jsonnew file mode 100644index 0000000000..fe51488c70--- /dev/null+++ b/datafusion/core/tests/data/json_empty_array.json@@ -0,0 +1 @@+[]diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex bab00ced1c..6c0452a99b 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -57,9 +57,7 @@ use datafusion::error::Result;use datafusion::execution::context::SessionContext;use datafusion::execution::session_state::SessionStateBuilder;use datafusion::logical_expr::{ColumnarValue, Volatility};-use datafusion::prelude::{- CsvReadOptions, JoinType, NdJsonReadOptions, ParquetReadOptions,-};+use datafusion::prelude::{CsvReadOptions, JoinType, ParquetReadOptions};use datafusion::test_util::{parquet_test_data, populate_csv_partitions, register_aggregate_csv, test_table,test_table_with_cache_factory, test_table_with_name,@@ -94,6 +92,7 @@ use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties, displayable};use datafusion::error::Result as DataFusionResult;+use datafusion::execution::options::JsonReadOptions;use datafusion_functions_window::expr_fn::lag;// Get string representation of the plan@@ -2896,7 +2895,7 @@ async fn write_json_with_order() -> Result<()> {ctx.register_json("data",test_path.to_str().unwrap(),- NdJsonReadOptions::default().schema(&schema),+ JsonReadOptions::default().schema(&schema),).await?;@@ -6322,7 +6321,7 @@ async fn register_non_json_file() {.register_json("data","tests/data/test_binary.parquet",- NdJsonReadOptions::default(),+ JsonReadOptions::default(),).await;assert_contains!(diff --git a/datafusion/datasource-json/Cargo.toml b/datafusion/datasource-json/Cargo.tomlindex 37fa8d43a0..bd0cead8d2 100644--- a/datafusion/datasource-json/Cargo.toml+++ b/datafusion/datasource-json/Cargo.toml@@ -44,7 +44,9 @@ datafusion-physical-plan = { workspace = true }datafusion-session = { workspace = true }futures = { workspace = true }object_store = { workspace = true }+serde_json = { workspace = true }tokio = { workspace = true }+tokio-stream = { workspace = true, features = ["sync"] }# Note: add additional linter rules in lib.rs.# Rust does not support workspace + new linter rules in subcrates yetdiff --git a/datafusion/datasource-json/src/file_format.rs b/datafusion/datasource-json/src/file_format.rsindex a14458b5ac..881e5f3d87 100644--- a/datafusion/datasource-json/src/file_format.rs+++ b/datafusion/datasource-json/src/file_format.rs@@ -15,13 +15,13 @@// specific language governing permissions and limitations// under the License.-//! [`JsonFormat`]: Line delimited JSON [`FileFormat`] abstractions+//! [`JsonFormat`]: Line delimited and array JSON [`FileFormat`] abstractionsuse std::any::Any;use std::collections::HashMap;use std::fmt;use std::fmt::Debug;-use std::io::BufReader;+use std::io::{BufReader, Read};use std::sync::Arc;use crate::source::JsonSource;@@ -31,6 +31,7 @@ use arrow::datatypes::{Schema, SchemaRef};use arrow::error::ArrowError;use arrow::json;use arrow::json::reader::{ValueIter, infer_json_schema_from_iterator};+use bytes::{Buf, Bytes};use datafusion_common::config::{ConfigField, ConfigFileType, JsonOptions};use datafusion_common::file_options::json_writer::JsonWriterOptions;use datafusion_common::{@@ -48,6 +49,7 @@ use datafusion_datasource::file_format::{use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};use datafusion_datasource::file_sink_config::{FileSink, FileSinkConfig};use datafusion_datasource::sink::{DataSink, DataSinkExec};+use datafusion_datasource::source::DataSourceExec;use datafusion_datasource::write::BatchSerializer;use datafusion_datasource::write::demux::DemuxedStreamReceiver;use datafusion_datasource::write::orchestration::spawn_writer_tasks_and_join;@@ -57,9 +59,8 @@ use datafusion_physical_expr_common::sort_expr::LexRequirement;use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan};use datafusion_session::Session;+use crate::utils::JsonArrayToNdjsonReader;use async_trait::async_trait;-use bytes::{Buf, Bytes};-use datafusion_datasource::source::DataSourceExec;use object_store::{GetResultPayload, ObjectMeta, ObjectStore};#[derive(Default)]@@ -132,7 +133,26 @@ impl Debug for JsonFormatFactory {}}-/// New line delimited JSON `FileFormat` implementation.+/// JSON `FileFormat` implementation supporting both line-delimited and array formats.+///+/// # Supported Formats+///+/// ## Line-Delimited JSON (default, `newline_delimited = true`)+/// ```text+/// {"key1": 1, "key2": "val"}+/// {"key1": 2, "key2": "vals"}+/// ```+///+/// ## JSON Array Format (`newline_delimited = false`)+/// ```text+/// [+/// {"key1": 1, "key2": "val"},+/// {"key1": 2, "key2": "vals"}+/// ]+/// ```+///+/// Note: JSON array format is processed using streaming conversion,+/// which is memory-efficient even for large files.#[derive(Debug, Default)]pub struct JsonFormat {options: JsonOptions,@@ -166,6 +186,57 @@ impl JsonFormat {self.options.compression = file_compression_type.into();self}++ /// Set whether to read as newline-delimited JSON (NDJSON).+ ///+ /// When `true` (default), expects newline-delimited format:+ /// ```text+ /// {"a": 1}+ /// {"a": 2}+ /// ```+ ///+ /// When `false`, expects JSON array format:+ /// ```text+ /// [{"a": 1}, {"a": 2}]+ /// ```+ pub fn with_newline_delimited(mut self, newline_delimited: bool) -> Self {+ self.options.newline_delimited = newline_delimited;+ self+ }++ /// Returns whether this format expects newline-delimited JSON.+ pub fn is_newline_delimited(&self) -> bool {+ self.options.newline_delimited+ }+}++/// Infer schema from JSON array format using streaming conversion.+///+/// This function converts JSON array format to NDJSON on-the-fly and uses+/// arrow-json's schema inference. It properly tracks the number of records+/// processed for correct `records_to_read` management.+///+/// # Returns+/// A tuple of (Schema, records_consumed) where records_consumed is the+/// number of records that were processed for schema inference.+fn infer_schema_from_json_array<R: Read>(+ reader: R,+ max_records: usize,+) -> Result<(Schema, usize)> {+ let ndjson_reader = JsonArrayToNdjsonReader::new(reader);++ let iter = ValueIter::new(ndjson_reader, None);+ let mut count = 0;++ let schema = infer_json_schema_from_iterator(iter.take_while(|_| {+ let should_take = count < max_records;+ if should_take {+ count += 1;+ }+ should_take+ }))?;++ Ok((schema, count))}#[async_trait]@@ -202,37 +273,67 @@ impl FileFormat for JsonFormat {.schema_infer_max_rec.unwrap_or(DEFAULT_SCHEMA_INFER_MAX_RECORD);let file_compression_type = FileCompressionType::from(self.options.compression);+ let newline_delimited = self.options.newline_delimited;+for object in objects {- let mut take_while = || {- let should_take = records_to_read > 0;- if should_take {- records_to_read -= 1;- }- should_take- };+ // Early exit if we've read enough records+ if records_to_read == 0 {+ break;+ }let r = store.as_ref().get(&object.location).await?;- let schema = match r.payload {++ let (schema, records_consumed) = match r.payload {#[cfg(not(target_arch = "wasm32"))]GetResultPayload::File(file, _) => {let decoder = file_compression_type.convert_read(file)?;- let mut reader = BufReader::new(decoder);- let iter = ValueIter::new(&mut reader, None);- infer_json_schema_from_iterator(iter.take_while(|_| take_while()))?+ let reader = BufReader::new(decoder);++ if newline_delimited {+ // NDJSON: use ValueIter directly+ let iter = ValueIter::new(reader, None);+ let mut count = 0;+ let schema =+ infer_json_schema_from_iterator(iter.take_while(|_| {+ let should_take = count < records_to_read;+ if should_take {+ count += 1;+ }+ should_take+ }))?;+ (schema, count)+ } else {+ // JSON array format: use streaming converter+ infer_schema_from_json_array(reader, records_to_read)?+ }}GetResultPayload::Stream(_) => {let data = r.bytes().await?;let decoder = file_compression_type.convert_read(data.reader())?;- let mut reader = BufReader::new(decoder);- let iter = ValueIter::new(&mut reader, None);- infer_json_schema_from_iterator(iter.take_while(|_| take_while()))?+ let reader = BufReader::new(decoder);++ if newline_delimited {+ let iter = ValueIter::new(reader, None);+ let mut count = 0;+ let schema =+ infer_json_schema_from_iterator(iter.take_while(|_| {+ let should_take = count < records_to_read;+ if should_take {+ count += 1;+ }+ should_take+ }))?;+ (schema, count)+ } else {+ // JSON array format: use streaming converter+ infer_schema_from_json_array(reader, records_to_read)?+ }}};schemas.push(schema);- if records_to_read == 0 {- break;- }+ // Correctly decrement records_to_read+ records_to_read = records_to_read.saturating_sub(records_consumed);}let schema = Schema::try_merge(schemas)?;@@ -281,7 +382,10 @@ impl FileFormat for JsonFormat {}fn file_source(&self, table_schema: TableSchema) -> Arc<dyn FileSource> {- Arc::new(JsonSource::new(table_schema))+ Arc::new(+ JsonSource::new(table_schema)+ .with_newline_delimited(self.options.newline_delimited),+ )}}diff --git a/datafusion/datasource-json/src/mod.rs b/datafusion/datasource-json/src/mod.rsindex c39ee2cd93..7dc0a0c7ba 100644--- a/datafusion/datasource-json/src/mod.rs+++ b/datafusion/datasource-json/src/mod.rs@@ -22,5 +22,6 @@pub mod file_format;pub mod source;+pub mod utils;pub use file_format::*;diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rsindex 5797054f11..867cfe0e98 100644--- a/datafusion/datasource-json/src/source.rs+++ b/datafusion/datasource-json/src/source.rs@@ -15,17 +15,19 @@// specific language governing permissions and limitations// under the License.-//! Execution plan for reading line-delimited JSON files+//! Execution plan for reading JSON files (line-delimited and array formats)use std::any::Any;use std::io::{BufReader, Read, Seek, SeekFrom};+use std::pin::Pin;use std::sync::Arc;-use std::task::Poll;+use std::task::{Context, Poll};use crate::file_format::JsonDecoder;+use crate::utils::{ChannelReader, JsonArrayToNdjsonReader};use datafusion_common::error::{DataFusionError, Result};-use datafusion_common_runtime::JoinSet;+use datafusion_common_runtime::{JoinSet, SpawnedTask};use datafusion_datasource::decoder::{DecoderDeserializer, deserialize_stream};use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_stream::{FileOpenFuture, FileOpener};@@ -36,6 +38,7 @@ use datafusion_datasource::{use datafusion_physical_plan::projection::ProjectionExprs;use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};+use arrow::array::RecordBatch;use arrow::json::ReaderBuilder;use arrow::{datatypes::SchemaRef, json};use datafusion_datasource::file::FileSource;@@ -43,10 +46,55 @@ use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_execution::TaskContext;use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;-use futures::{StreamExt, TryStreamExt};+use futures::{Stream, StreamExt, TryStreamExt};use object_store::buffered::BufWriter;use object_store::{GetOptions, GetResultPayload, ObjectStore};use tokio::io::AsyncWriteExt;+use tokio_stream::wrappers::ReceiverStream;++/// Channel buffer size for streaming JSON array processing.+/// With ~128KB average chunk size, 128 chunks ≈ 16MB buffer.+const CHANNEL_BUFFER_SIZE: usize = 128;++/// Buffer size for JsonArrayToNdjsonReader (2MB each, 4MB total for input+output)+const JSON_CONVERTER_BUFFER_SIZE: usize = 2 * 1024 * 1024;++// ============================================================================+// JsonArrayStream - Custom stream wrapper to hold SpawnedTask handles+// ============================================================================++/// A stream wrapper that holds SpawnedTask handles to keep them alive+/// until the stream is fully consumed or dropped.+///+/// This ensures cancel-safety: when the stream is dropped, the tasks+/// are properly aborted via SpawnedTask's Drop implementation.+struct JsonArrayStream {+ inner: ReceiverStream<std::result::Result<RecordBatch, arrow::error::ArrowError>>,+ /// Task that reads from object store and sends bytes to channel.+ /// Kept alive until stream is consumed or dropped.+ _read_task: SpawnedTask<()>,+ /// Task that parses JSON and sends RecordBatches.+ /// Kept alive until stream is consumed or dropped.+ _parse_task: SpawnedTask<()>,+}++impl Stream for JsonArrayStream {+ type Item = std::result::Result<RecordBatch, arrow::error::ArrowError>;++ fn poll_next(+ mut self: Pin<&mut Self>,+ cx: &mut Context<'_>,+ ) -> Poll<Option<Self::Item>> {+ Pin::new(&mut self.inner).poll_next(cx)+ }++ fn size_hint(&self) -> (usize, Option<usize>) {+ self.inner.size_hint()+ }+}+// ============================================================================+// JsonOpener and JsonSource+// ============================================================================/// A [`FileOpener`] that opens a JSON file and yields a [`FileOpenFuture`]pub struct JsonOpener {@@ -54,21 +102,26 @@ pub struct JsonOpener {projected_schema: SchemaRef,file_compression_type: FileCompressionType,object_store: Arc<dyn ObjectStore>,+ /// When `true` (default), expects newline-delimited JSON (NDJSON).+ /// When `false`, expects JSON array format `[{...}, {...}]`.+ newline_delimited: bool,}impl JsonOpener {- /// Returns a [`JsonOpener`]+ /// Returns a [`JsonOpener`]pub fn new(batch_size: usize,projected_schema: SchemaRef,file_compression_type: FileCompressionType,object_store: Arc<dyn ObjectStore>,+ newline_delimited: bool,) -> Self {Self {batch_size,projected_schema,file_compression_type,object_store,+ newline_delimited,}}}@@ -80,6 +133,9 @@ pub struct JsonSource {batch_size: Option<usize>,metrics: ExecutionPlanMetricsSet,projection: SplitProjection,+ /// When `true` (default), expects newline-delimited JSON (NDJSON).+ /// When `false`, expects JSON array format `[{...}, {...}]`.+ newline_delimited: bool,}impl JsonSource {@@ -91,8 +147,18 @@ impl JsonSource {table_schema,batch_size: None,metrics: ExecutionPlanMetricsSet::new(),+ newline_delimited: true,}}++ /// Set whether to read as newline-delimited JSON.+ ///+ /// When `true` (default), expects newline-delimited format.+ /// When `false`, expects JSON array format `[{...}, {...}]`.+ pub fn with_newline_delimited(mut self, newline_delimited: bool) -> Self {+ self.newline_delimited = newline_delimited;+ self+ }}impl From<JsonSource> for Arc<dyn FileSource> {@@ -120,6 +186,7 @@ impl FileSource for JsonSource {projected_schema,file_compression_type: base_config.file_compression_type,object_store,+ newline_delimited: self.newline_delimited,}) as Arc<dyn FileOpener>;// Wrap with ProjectionOpener@@ -172,7 +239,7 @@ impl FileSource for JsonSource {}impl FileOpener for JsonOpener {- /// Open a partitioned NDJSON file.+ /// Open a partitioned JSON file.////// If `file_meta.range` is `None`, the entire file is opened./// Else `file_meta.range` is `Some(FileRange{start, end})`, which corresponds to the byte range [start, end) within the file.@@ -181,11 +248,23 @@ impl FileOpener for JsonOpener {/// are applied to determine which lines to read:/// 1. The first line of the partition is the line in which the index of the first character >= `start`./// 2. The last line of the partition is the line in which the byte at position `end - 1` resides.+ ///+ /// Note: JSON array format does not support range-based scanning.fn open(&self, partitioned_file: PartitionedFile) -> Result<FileOpenFuture> {let store = Arc::clone(&self.object_store);let schema = Arc::clone(&self.projected_schema);let batch_size = self.batch_size;let file_compression_type = self.file_compression_type.to_owned();+ let newline_delimited = self.newline_delimited;++ // JSON array format requires reading the complete file+ if !newline_delimited && partitioned_file.range.is_some() {+ return Err(DataFusionError::NotImplemented(+ "JSON array format does not support range-based file scanning. \+ Disable repartition_file_scans or use newline-delimited JSON format."+ .to_string(),+ ));+ }Ok(Box::pin(async move {let calculated_range =@@ -218,31 +297,150 @@ impl FileOpener for JsonOpener {Some(_) => {file.seek(SeekFrom::Start(result.range.start as _))?;let limit = result.range.end - result.range.start;- file_compression_type.convert_read(file.take(limit as u64))?+ file_compression_type.convert_read(file.take(limit))?}};- let reader = ReaderBuilder::new(schema)- .with_batch_size(batch_size)- .build(BufReader::new(bytes))?;-- Ok(futures::stream::iter(reader)- .map(|r| r.map_err(Into::into))- .boxed())+ if newline_delimited {+ // NDJSON: use BufReader directly+ let reader = BufReader::new(bytes);+ let arrow_reader = ReaderBuilder::new(schema)+ .with_batch_size(batch_size)+ .build(reader)?;++ Ok(futures::stream::iter(arrow_reader)+ .map(|r| r.map_err(Into::into))+ .boxed())+ } else {+ // JSON array format: wrap with streaming converter+ let ndjson_reader = JsonArrayToNdjsonReader::with_capacity(+ bytes,+ JSON_CONVERTER_BUFFER_SIZE,+ );+ let arrow_reader = ReaderBuilder::new(schema)+ .with_batch_size(batch_size)+ .build(ndjson_reader)?;++ Ok(futures::stream::iter(arrow_reader)+ .map(|r| r.map_err(Into::into))+ .boxed())+ }}GetResultPayload::Stream(s) => {- let s = s.map_err(DataFusionError::from);-- let decoder = ReaderBuilder::new(schema)- .with_batch_size(batch_size)- .build_decoder()?;- let input = file_compression_type.convert_stream(s.boxed())?.fuse();-- let stream = deserialize_stream(- input,- DecoderDeserializer::new(JsonDecoder::new(decoder)),- );- Ok(stream.map_err(Into::into).boxed())+ if newline_delimited {+ // Newline-delimited JSON (NDJSON) streaming reader+ let s = s.map_err(DataFusionError::from);+ let decoder = ReaderBuilder::new(schema)+ .with_batch_size(batch_size)+ .build_decoder()?;+ let input =+ file_compression_type.convert_stream(s.boxed())?.fuse();+ let stream = deserialize_stream(+ input,+ DecoderDeserializer::new(JsonDecoder::new(decoder)),+ );+ Ok(stream.map_err(Into::into).boxed())+ } else {+ // JSON array format: streaming conversion with channel-based byte transfer+ //+ // Architecture:+ // 1. Async task reads from object store stream, decompresses, sends to channel+ // 2. Blocking task receives bytes, converts JSON array to NDJSON, parses to Arrow+ // 3. RecordBatches are sent back via another channel+ //+ // Memory budget (~32MB):+ // - sync_channel: CHANNEL_BUFFER_SIZE chunks (~16MB)+ // - JsonArrayToNdjsonReader: 2 × JSON_CONVERTER_BUFFER_SIZE (~4MB)+ // - Arrow JsonReader internal buffer (~8MB)+ // - Miscellaneous (~4MB)++ let s = s.map_err(DataFusionError::from);+ let decompressed_stream =+ file_compression_type.convert_stream(s.boxed())?;++ // Channel for bytes: async producer -> blocking consumer+ // Uses tokio::sync::mpsc so the async send never blocks a+ // tokio worker thread; the consumer calls blocking_recv()+ // inside spawn_blocking.+ let (byte_tx, byte_rx) = tokio::sync::mpsc::channel::<bytes::Bytes>(+ CHANNEL_BUFFER_SIZE,+ );++ // Channel for results: sync producer -> async consumer+ let (result_tx, result_rx) = tokio::sync::mpsc::channel(2);+ let error_tx = result_tx.clone();++ // Async task: read from object store stream and send bytes to channel+ // Store the SpawnedTask to keep it alive until stream is dropped+ let read_task = SpawnedTask::spawn(async move {+ tokio::pin!(decompressed_stream);+ while let Some(chunk) = decompressed_stream.next().await {+ match chunk {+ Ok(bytes) => {+ if byte_tx.send(bytes).await.is_err() {+ break; // Consumer dropped+ }+ }+ Err(e) => {+ let _ = error_tx+ .send(Err(+ arrow::error::ArrowError::ExternalError(+ Box::new(e),+ ),+ ))+ .await;+ break;+ }+ }+ }+ // byte_tx dropped here, signals EOF to ChannelReader+ });++ // Blocking task: receive bytes from channel and parse JSON+ // Store the SpawnedTask to keep it alive until stream is dropped+ let parse_task = SpawnedTask::spawn_blocking(move || {+ let channel_reader = ChannelReader::new(byte_rx);+ let mut ndjson_reader =+ JsonArrayToNdjsonReader::with_capacity(+ channel_reader,+ JSON_CONVERTER_BUFFER_SIZE,+ );++ match ReaderBuilder::new(schema)+ .with_batch_size(batch_size)+ .build(&mut ndjson_reader)+ {+ Ok(arrow_reader) => {+ for batch_result in arrow_reader {+ if result_tx.blocking_send(batch_result).is_err()+ {+ break; // Receiver dropped+ }+ }+ }+ Err(e) => {+ let _ = result_tx.blocking_send(Err(e));+ }+ }++ // Validate the JSON array was properly formed+ if let Err(e) = ndjson_reader.validate_complete() {+ let _ = result_tx.blocking_send(Err(+ arrow::error::ArrowError::JsonError(e.to_string()),+ ));+ }+ // result_tx dropped here, closes the stream+ });++ // Wrap in JsonArrayStream to keep tasks alive until stream is consumed+ let stream = JsonArrayStream {+ inner: ReceiverStream::new(result_rx),+ _read_task: read_task,+ _parse_task: parse_task,+ };++ Ok(stream.map(|r| r.map_err(Into::into)).boxed())+ }}}}))@@ -303,3 +501,307 @@ pub async fn plan_to_json(Ok(())}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::datatypes::{DataType, Field, Schema};+ use bytes::Bytes;+ use datafusion_datasource::FileRange;+ use futures::TryStreamExt;+ use object_store::PutPayload;+ use object_store::memory::InMemory;+ use object_store::path::Path;++ /// Helper to create a test schema+ fn test_schema() -> SchemaRef {+ Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int64, true),+ Field::new("name", DataType::Utf8, true),+ ]))+ }++ #[tokio::test]+ async fn test_json_array_from_file() -> Result<()> {+ // Test reading JSON array format from a file+ let json_data = r#"[{"id": 1, "name": "alice"}, {"id": 2, "name": "bob"}]"#;++ let store = Arc::new(InMemory::new());+ let path = Path::from("test.json");+ store+ .put(&path, PutPayload::from_static(json_data.as_bytes()))+ .await?;++ let opener = JsonOpener::new(+ 1024,+ test_schema(),+ FileCompressionType::UNCOMPRESSED,+ store.clone(),+ false, // JSON array format+ );++ let meta = store.head(&path).await?;+ let file = PartitionedFile::new(path.to_string(), meta.size);++ let stream = opener.open(file)?.await?;+ let batches: Vec<_> = stream.try_collect().await?;++ assert_eq!(batches.len(), 1);+ assert_eq!(batches[0].num_rows(), 2);++ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_from_stream() -> Result<()> {+ // Test reading JSON array format from object store stream (simulates S3)+ let json_data = r#"[{"id": 1, "name": "alice"}, {"id": 2, "name": "bob"}, {"id": 3, "name": "charlie"}]"#;++ // Use InMemory store which returns Stream payload+ let store = Arc::new(InMemory::new());+ let path = Path::from("test_stream.json");+ store+ .put(&path, PutPayload::from_static(json_data.as_bytes()))+ .await?;++ let opener = JsonOpener::new(+ 2, // small batch size to test multiple batches+ test_schema(),+ FileCompressionType::UNCOMPRESSED,+ store.clone(),+ false, // JSON array format+ );++ let meta = store.head(&path).await?;+ let file = PartitionedFile::new(path.to_string(), meta.size);++ let stream = opener.open(file)?.await?;+ let batches: Vec<_> = stream.try_collect().await?;++ let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(total_rows, 3);++ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_nested_objects() -> Result<()> {+ // Test JSON array with nested objects and arrays+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int64, true),+ Field::new("data", DataType::Utf8, true),+ ]));++ let json_data = r#"[+ {"id": 1, "data": "{\"nested\": true}"},+ {"id": 2, "data": "[1, 2, 3]"}+ ]"#;++ let store = Arc::new(InMemory::new());+ let path = Path::from("nested.json");+ store+ .put(&path, PutPayload::from_static(json_data.as_bytes()))+ .await?;++ let opener = JsonOpener::new(+ 1024,+ schema,+ FileCompressionType::UNCOMPRESSED,+ store.clone(),+ false,+ );++ let meta = store.head(&path).await?;+ let file = PartitionedFile::new(path.to_string(), meta.size);++ let stream = opener.open(file)?.await?;+ let batches: Vec<_> = stream.try_collect().await?;++ assert_eq!(batches[0].num_rows(), 2);++ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_empty() -> Result<()> {+ // Test empty JSON array+ let json_data = "[]";++ let store = Arc::new(InMemory::new());+ let path = Path::from("empty.json");+ store+ .put(&path, PutPayload::from_static(json_data.as_bytes()))+ .await?;++ let opener = JsonOpener::new(+ 1024,+ test_schema(),+ FileCompressionType::UNCOMPRESSED,+ store.clone(),+ false,+ );++ let meta = store.head(&path).await?;+ let file = PartitionedFile::new(path.to_string(), meta.size);++ let stream = opener.open(file)?.await?;+ let batches: Vec<_> = stream.try_collect().await?;++ let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(total_rows, 0);++ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_range_not_supported() {+ // Test that range-based scanning returns error for JSON array format+ let store = Arc::new(InMemory::new());+ let path = Path::from("test.json");+ store+ .put(&path, PutPayload::from_static(b"[]"))+ .await+ .unwrap();++ let opener = JsonOpener::new(+ 1024,+ test_schema(),+ FileCompressionType::UNCOMPRESSED,+ store.clone(),+ false, // JSON array format+ );++ let meta = store.head(&path).await.unwrap();+ let mut file = PartitionedFile::new(path.to_string(), meta.size);+ file.range = Some(FileRange { start: 0, end: 10 });++ let result = opener.open(file);+ match result {+ Ok(_) => panic!("Expected error for range-based JSON array scanning"),+ Err(e) => {+ assert!(+ e.to_string().contains("does not support range-based"),+ "Unexpected error message: {e}"+ );+ }+ }+ }++ #[tokio::test]+ async fn test_ndjson_still_works() -> Result<()> {+ // Ensure NDJSON format still works correctly+ let json_data =+ "{\"id\": 1, \"name\": \"alice\"}\n{\"id\": 2, \"name\": \"bob\"}\n";++ let store = Arc::new(InMemory::new());+ let path = Path::from("test.ndjson");+ store+ .put(&path, PutPayload::from_static(json_data.as_bytes()))+ .await?;++ let opener = JsonOpener::new(+ 1024,+ test_schema(),+ FileCompressionType::UNCOMPRESSED,+ store.clone(),+ true, // NDJSON format+ );++ let meta = store.head(&path).await?;+ let file = PartitionedFile::new(path.to_string(), meta.size);++ let stream = opener.open(file)?.await?;+ let batches: Vec<_> = stream.try_collect().await?;++ assert_eq!(batches.len(), 1);+ assert_eq!(batches[0].num_rows(), 2);++ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_large_file() -> Result<()> {+ // Test with a larger JSON array to verify streaming works+ let mut json_data = String::from("[");+ for i in 0..1000 {+ if i > 0 {+ json_data.push(',');+ }+ json_data.push_str(&format!(r#"{{"id": {i}, "name": "user{i}"}}"#));+ }+ json_data.push(']');++ let store = Arc::new(InMemory::new());+ let path = Path::from("large.json");+ store+ .put(&path, PutPayload::from(Bytes::from(json_data)))+ .await?;++ let opener = JsonOpener::new(+ 100, // batch size of 100+ test_schema(),+ FileCompressionType::UNCOMPRESSED,+ store.clone(),+ false,+ );++ let meta = store.head(&path).await?;+ let file = PartitionedFile::new(path.to_string(), meta.size);++ let stream = opener.open(file)?.await?;+ let batches: Vec<_> = stream.try_collect().await?;++ let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();+ assert_eq!(total_rows, 1000);++ // Should have multiple batches due to batch_size=100+ assert!(batches.len() >= 10);++ Ok(())+ }++ #[tokio::test]+ async fn test_json_array_stream_cancellation() -> Result<()> {+ // Test that cancellation works correctly (tasks are aborted when stream is dropped)+ let mut json_data = String::from("[");+ for i in 0..10000 {+ if i > 0 {+ json_data.push(',');+ }+ json_data.push_str(&format!(r#"{{"id": {i}, "name": "user{i}"}}"#));+ }+ json_data.push(']');++ let store = Arc::new(InMemory::new());+ let path = Path::from("cancel_test.json");+ store+ .put(&path, PutPayload::from(Bytes::from(json_data)))+ .await?;++ let opener = JsonOpener::new(+ 10, // small batch size+ test_schema(),+ FileCompressionType::UNCOMPRESSED,+ store.clone(),+ false,+ );++ let meta = store.head(&path).await?;+ let file = PartitionedFile::new(path.to_string(), meta.size);++ let mut stream = opener.open(file)?.await?;++ // Read only first batch, then drop the stream (simulating cancellation)+ let first_batch = stream.next().await;+ assert!(first_batch.is_some());++ // Drop the stream - this should abort the spawned tasks via SpawnedTask's Drop+ drop(stream);++ // Give tasks time to be aborted+ tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;++ // If we reach here without hanging, cancellation worked+ Ok(())+ }+}diff --git a/datafusion/datasource-json/src/utils.rs b/datafusion/datasource-json/src/utils.rsnew file mode 100644index 0000000000..bc75799edf--- /dev/null+++ b/datafusion/datasource-json/src/utils.rs@@ -0,0 +1,778 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Utility types for JSON processing++use std::io::{BufRead, Read};++use bytes::Bytes;++// ============================================================================+// JsonArrayToNdjsonReader - Streaming JSON Array to NDJSON Converter+// ============================================================================+//+// Architecture:+//+// ```text+// ┌─────────────────────────────────────────────────────────────┐+// │ JSON Array File (potentially very large, e.g. 33GB) │+// │ [{"a":1}, {"a":2}, {"a":3}, ...... {"a":1000000}] │+// └─────────────────────────────────────────────────────────────┘+// │+// ▼ read chunks via ChannelReader+// ┌───────────────────┐+// │ JsonArrayToNdjson │ ← character substitution only:+// │ Reader │ '[' skip, ',' → '\n', ']' stop+// └───────────────────┘+// │+// ▼ outputs NDJSON format+// ┌───────────────────┐+// │ Arrow Reader │ ← internal buffer, batch parsing+// │ batch_size=8192 │+// └───────────────────┘+// │+// ▼ outputs RecordBatch+// ┌───────────────────┐+// │ RecordBatch │+// └───────────────────┘+// ```+//+// Memory Efficiency:+//+// | Approach | Memory for 33GB file | Parse count |+// |---------------------------------------|----------------------|-------------|+// | Load entire file + serde_json | ~100GB+ | 3x |+// | Streaming with JsonArrayToNdjsonReader| ~32MB (configurable) | 1x |+//+// Design Note:+//+// This implementation uses `inner: R` directly (not `BufReader<R>`) and manages+// its own input buffer. This is critical for compatibility with `SyncIoBridge`+// and `ChannelReader` in `spawn_blocking` contexts.+//++/// Default buffer size for JsonArrayToNdjsonReader (2MB for better throughput)+const DEFAULT_BUF_SIZE: usize = 2 * 1024 * 1024;++/// Parser state for JSON array streaming+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum JsonArrayState {+ /// Initial state, looking for opening '['+ Start,+ /// Inside the JSON array, processing objects+ InArray,+ /// Reached the closing ']', finished+ Done,+}++/// A streaming reader that converts JSON array format to NDJSON format.+///+/// This reader wraps an underlying reader containing JSON array data+/// `[{...}, {...}, ...]` and transforms it on-the-fly to newline-delimited+/// JSON format that Arrow's JSON reader can process.+///+/// Implements both `Read` and `BufRead` traits for compatibility with Arrow's+/// `ReaderBuilder::build()` which requires `BufRead`.+///+/// # Transformation Rules+///+/// - Skip leading `[` and whitespace before it+/// - Convert top-level `,` (between objects) to `\n`+/// - Skip whitespace at top level (between objects)+/// - Stop at trailing `]`+/// - Preserve everything inside objects (including nested `[`, `]`, `,`)+/// - Properly handle strings (ignore special chars inside quotes)+///+/// # Example+///+/// ```text+/// Input: [{"a":1}, {"b":[1,2]}, {"c":"x,y"}]+/// Output: {"a":1}+/// {"b":[1,2]}+/// {"c":"x,y"}+/// ```+pub struct JsonArrayToNdjsonReader<R: Read> {+ /// Inner reader - we use R directly (not `BufReader<R>`) for SyncIoBridge compatibility+ inner: R,+ state: JsonArrayState,+ /// Tracks nesting depth of `{` and `[` to identify top-level commas+ depth: i32,+ /// Whether we're currently inside a JSON string+ in_string: bool,+ /// Whether the next character is escaped (after `\`)+ escape_next: bool,+ /// Input buffer - stores raw bytes read from inner reader+ input_buffer: Vec<u8>,+ /// Current read position in input buffer+ input_pos: usize,+ /// Number of valid bytes in input buffer+ input_filled: usize,+ /// Output buffer - stores transformed NDJSON bytes+ output_buffer: Vec<u8>,+ /// Current read position in output buffer+ output_pos: usize,+ /// Number of valid bytes in output buffer+ output_filled: usize,+ /// Whether trailing non-whitespace content was detected after ']'+ has_trailing_content: bool,+ /// Whether leading non-whitespace content was detected before '['+ has_leading_content: bool,+}++impl<R: Read> JsonArrayToNdjsonReader<R> {+ /// Create a new streaming reader that converts JSON array to NDJSON.+ pub fn new(reader: R) -> Self {+ Self::with_capacity(reader, DEFAULT_BUF_SIZE)+ }++ /// Create a new streaming reader with custom buffer size.+ ///+ /// Larger buffers improve throughput but use more memory.+ /// Total memory usage is approximately 2 * capacity (input + output buffers).+ pub fn with_capacity(reader: R, capacity: usize) -> Self {+ Self {+ inner: reader,+ state: JsonArrayState::Start,+ depth: 0,+ in_string: false,+ escape_next: false,+ input_buffer: vec![0; capacity],+ input_pos: 0,+ input_filled: 0,+ output_buffer: vec![0; capacity],+ output_pos: 0,+ output_filled: 0,+ has_trailing_content: false,+ has_leading_content: false,+ }+ }++ /// Check if the JSON array was properly terminated.+ ///+ /// This should be called after all data has been read.+ ///+ /// Returns an error if:+ /// - Unbalanced braces/brackets (depth != 0)+ /// - Unterminated string+ /// - Missing closing `]`+ /// - Unexpected trailing content after `]`+ pub fn validate_complete(&self) -> std::io::Result<()> {+ if self.has_leading_content {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidData,+ "Malformed JSON: unexpected leading content before '['",+ ));+ }+ if self.depth != 0 {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidData,+ "Malformed JSON array: unbalanced braces or brackets",+ ));+ }+ if self.in_string {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidData,+ "Malformed JSON array: unterminated string",+ ));+ }+ if self.state != JsonArrayState::Done {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidData,+ "Incomplete JSON array: expected closing bracket ']'",+ ));+ }+ if self.has_trailing_content {+ return Err(std::io::Error::new(+ std::io::ErrorKind::InvalidData,+ "Malformed JSON: unexpected trailing content after ']'",+ ));+ }+ Ok(())+ }++ /// Process a single byte and return the transformed byte (if any)+ #[inline]+ fn process_byte(&mut self, byte: u8) -> Option<u8> {+ match self.state {+ JsonArrayState::Start => {+ // Looking for the opening '[', skip whitespace+ if byte == b'[' {+ self.state = JsonArrayState::InArray;+ } else if !byte.is_ascii_whitespace() {+ self.has_leading_content = true;+ }+ None+ }+ JsonArrayState::InArray => {+ // Handle escape sequences in strings+ if self.escape_next {+ self.escape_next = false;+ return Some(byte);+ }++ if self.in_string {+ // Inside a string: handle escape and closing quote+ match byte {+ b'\\' => self.escape_next = true,+ b'"' => self.in_string = false,+ _ => {}+ }+ Some(byte)+ } else {+ // Outside strings: track depth and transform+ match byte {+ b'"' => {+ self.in_string = true;+ Some(byte)+ }+ b'{' | b'[' => {+ self.depth += 1;+ Some(byte)+ }+ b'}' => {+ self.depth -= 1;+ Some(byte)+ }+ b']' => {+ if self.depth == 0 {+ // Top-level ']' means end of array+ self.state = JsonArrayState::Done;+ None+ } else {+ // Nested ']' inside an object+ self.depth -= 1;+ Some(byte)+ }+ }+ b',' if self.depth == 0 => {+ // Top-level comma between objects → newline+ Some(b'\n')+ }+ _ => {+ // At depth 0, skip whitespace between objects+ if self.depth == 0 && byte.is_ascii_whitespace() {+ None+ } else {+ Some(byte)+ }+ }+ }+ }+ }+ JsonArrayState::Done => {+ // After ']', check for non-whitespace trailing content+ if !byte.is_ascii_whitespace() {+ self.has_trailing_content = true;+ }+ None+ }+ }+ }++ /// Refill input buffer from inner reader if needed.+ /// Returns true if there's data available, false on EOF.+ fn refill_input_if_needed(&mut self) -> std::io::Result<bool> {+ if self.input_pos >= self.input_filled {+ // Input buffer exhausted, read more from inner+ let bytes_read = self.inner.read(&mut self.input_buffer)?;+ if bytes_read == 0 {+ return Ok(false); // EOF+ }+ self.input_pos = 0;+ self.input_filled = bytes_read;+ }+ Ok(true)+ }++ /// Fill the output buffer with transformed data.+ ///+ /// This method manages its own input buffer, reading from the inner reader+ /// as needed. When the output buffer is full, we stop processing but preserve+ /// the current position in the input buffer for the next call.+ fn fill_output_buffer(&mut self) -> std::io::Result<()> {+ let mut write_pos = 0;++ while write_pos < self.output_buffer.len() {+ // Refill input buffer if exhausted+ if !self.refill_input_if_needed()? {+ break; // EOF+ }++ // Process bytes from input buffer+ while self.input_pos < self.input_filled+ && write_pos < self.output_buffer.len()+ {+ let byte = self.input_buffer[self.input_pos];+ self.input_pos += 1;++ if let Some(transformed) = self.process_byte(byte) {+ self.output_buffer[write_pos] = transformed;+ write_pos += 1;+ }+ }+ }++ self.output_pos = 0;+ self.output_filled = write_pos;+ Ok(())+ }+}++impl<R: Read> Read for JsonArrayToNdjsonReader<R> {+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {+ // If output buffer is empty, fill it+ if self.output_pos >= self.output_filled {+ self.fill_output_buffer()?;+ if self.output_filled == 0 {+ return Ok(0); // EOF+ }+ }++ // Copy from output buffer to caller's buffer+ let available = self.output_filled - self.output_pos;+ let to_copy = std::cmp::min(available, buf.len());+ buf[..to_copy].copy_from_slice(+ &self.output_buffer[self.output_pos..self.output_pos + to_copy],+ );+ self.output_pos += to_copy;+ Ok(to_copy)+ }+}++impl<R: Read> BufRead for JsonArrayToNdjsonReader<R> {+ fn fill_buf(&mut self) -> std::io::Result<&[u8]> {+ if self.output_pos >= self.output_filled {+ self.fill_output_buffer()?;+ }+ Ok(&self.output_buffer[self.output_pos..self.output_filled])+ }++ fn consume(&mut self, amt: usize) {+ self.output_pos = std::cmp::min(self.output_pos + amt, self.output_filled);+ }+}++// ============================================================================+// ChannelReader - Sync reader that receives bytes from async channel+// ============================================================================+//+// Architecture:+//+// ```text+// ┌─────────────────────────────────────────────────────────────────────────┐+// │ S3 / MinIO (async) │+// │ (33GB JSON Array File) │+// └─────────────────────────────────────────────────────────────────────────┘+// │+// ▼ async stream (Bytes chunks)+// ┌─────────────────────────────────────────────────────────────────────────┐+// │ Async Task (tokio runtime) │+// │ while let Some(chunk) = stream.next().await │+// │ byte_tx.send(chunk) │+// └─────────────────────────────────────────────────────────────────────────┘+// │+// ▼ tokio::sync::mpsc::channel<Bytes>+// │ (bounded, ~32MB buffer)+// ▼+// ┌─────────────────────────────────────────────────────────────────────────┐+// │ Blocking Task (spawn_blocking) │+// │ ┌──────────────┐ ┌────────────────────────┐ ┌──────────────────┐ │+// │ │ChannelReader │ → │JsonArrayToNdjsonReader │ → │ Arrow JsonReader │ │+// │ │ (Read) │ │ [{},...] → {}\n{} │ │ (RecordBatch) │ │+// │ └──────────────┘ └────────────────────────┘ └──────────────────┘ │+// └─────────────────────────────────────────────────────────────────────────┘+// │+// ▼ tokio::sync::mpsc::channel<RecordBatch>+// ┌─────────────────────────────────────────────────────────────────────────┐+// │ ReceiverStream (async) │+// │ → DataFusion execution engine │+// └─────────────────────────────────────────────────────────────────────────┘+// ```+//+// Memory Budget (~32MB total):+// - sync_channel buffer: 128 chunks × ~128KB = ~16MB+// - JsonArrayToNdjsonReader: 2 × 2MB = 4MB+// - Arrow JsonReader internal: ~8MB+// - Miscellaneous: ~4MB+//++/// A synchronous `Read` implementation that receives bytes from an async channel.+///+/// This enables true streaming between async and sync contexts without+/// loading the entire file into memory. Uses `tokio::sync::mpsc::Receiver`+/// with `blocking_recv()` so the async producer never blocks a tokio worker+/// thread, while the sync consumer (running in `spawn_blocking`) safely blocks.+pub struct ChannelReader {+ rx: tokio::sync::mpsc::Receiver<Bytes>,+ current: Option<Bytes>,+ pos: usize,+}++impl ChannelReader {+ /// Create a new ChannelReader from a tokio mpsc receiver.+ pub fn new(rx: tokio::sync::mpsc::Receiver<Bytes>) -> Self {+ Self {+ rx,+ current: None,+ pos: 0,+ }+ }+}++impl Read for ChannelReader {+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {+ loop {+ // If we have current chunk with remaining data, read from it+ if let Some(ref chunk) = self.current {+ let remaining = chunk.len() - self.pos;+ if remaining > 0 {+ let to_copy = std::cmp::min(remaining, buf.len());+ buf[..to_copy].copy_from_slice(&chunk[self.pos..self.pos + to_copy]);+ self.pos += to_copy;+ return Ok(to_copy);+ }+ }++ // Current chunk exhausted, get next from channel+ match self.rx.blocking_recv() {+ Some(bytes) => {+ self.current = Some(bytes);+ self.pos = 0;+ // Loop back to read from new chunk+ }+ None => return Ok(0), // Channel closed = EOF+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_json_array_to_ndjson_simple() {+ let input = r#"[{"a":1}, {"a":2}, {"a":3}]"#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ assert_eq!(output, "{\"a\":1}\n{\"a\":2}\n{\"a\":3}");+ }++ #[test]+ fn test_json_array_to_ndjson_nested() {+ let input = r#"[{"a":{"b":1}}, {"c":[1,2,3]}]"#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ assert_eq!(output, "{\"a\":{\"b\":1}}\n{\"c\":[1,2,3]}");+ }++ #[test]+ fn test_json_array_to_ndjson_strings_with_special_chars() {+ let input = r#"[{"a":"[1,2]"}, {"b":"x,y"}]"#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ assert_eq!(output, "{\"a\":\"[1,2]\"}\n{\"b\":\"x,y\"}");+ }++ #[test]+ fn test_json_array_to_ndjson_escaped_quotes() {+ let input = r#"[{"a":"say \"hello\""}, {"b":1}]"#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ assert_eq!(output, "{\"a\":\"say \\\"hello\\\"\"}\n{\"b\":1}");+ }++ #[test]+ fn test_json_array_to_ndjson_empty() {+ let input = r#"[]"#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ assert_eq!(output, "");+ }++ #[test]+ fn test_json_array_to_ndjson_single_element() {+ let input = r#"[{"a":1}]"#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ assert_eq!(output, "{\"a\":1}");+ }++ #[test]+ fn test_json_array_to_ndjson_bufread() {+ let input = r#"[{"a":1}, {"a":2}]"#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());++ let buf = reader.fill_buf().unwrap();+ assert!(!buf.is_empty());++ let first_len = buf.len();+ reader.consume(first_len);++ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ }++ #[test]+ fn test_json_array_to_ndjson_whitespace() {+ let input = r#" [ {"a":1} , {"a":2} ] "#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ // Top-level whitespace is skipped, internal whitespace preserved+ assert_eq!(output, "{\"a\":1}\n{\"a\":2}");+ }++ #[test]+ fn test_validate_complete_valid_json() {+ let valid_json = r#"[{"a":1},{"a":2}]"#;+ let mut reader = JsonArrayToNdjsonReader::new(valid_json.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ reader.validate_complete().unwrap();+ }++ #[test]+ fn test_json_array_with_trailing_junk() {+ let input = r#" [ {"a":1} , {"a":2} ] some { junk [ here ] "#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();++ // Should extract the valid array content+ assert_eq!(output, "{\"a\":1}\n{\"a\":2}");++ // But validation should catch the trailing junk+ let result = reader.validate_complete();+ assert!(result.is_err());+ let err_msg = result.unwrap_err().to_string();+ assert!(+ err_msg.contains("trailing content")+ || err_msg.contains("Unexpected trailing"),+ "Expected trailing content error, got: {err_msg}"+ );+ }++ #[test]+ fn test_validate_complete_incomplete_array() {+ let invalid_json = r#"[{"a":1},{"a":2}"#; // Missing closing ]+ let mut reader = JsonArrayToNdjsonReader::new(invalid_json.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();++ let result = reader.validate_complete();+ assert!(result.is_err());+ let err_msg = result.unwrap_err().to_string();+ assert!(+ err_msg.contains("expected closing bracket")+ || err_msg.contains("missing closing"),+ "Expected missing bracket error, got: {err_msg}"+ );+ }++ #[test]+ fn test_validate_complete_unbalanced_braces() {+ let invalid_json = r#"[{"a":1},{"a":2]"#; // Wrong closing bracket+ let mut reader = JsonArrayToNdjsonReader::new(invalid_json.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();++ let result = reader.validate_complete();+ assert!(result.is_err());+ let err_msg = result.unwrap_err().to_string();+ assert!(+ err_msg.contains("unbalanced")+ || err_msg.contains("expected closing bracket"),+ "Expected unbalanced or missing bracket error, got: {err_msg}"+ );+ }++ #[test]+ fn test_json_array_with_leading_junk() {+ let input = r#"junk[{"a":1}, {"a":2}]"#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();++ // Should still extract the valid array content+ assert_eq!(output, "{\"a\":1}\n{\"a\":2}");++ // But validation should catch the leading junk+ let result = reader.validate_complete();+ assert!(result.is_err());+ let err_msg = result.unwrap_err().to_string();+ assert!(+ err_msg.contains("leading content"),+ "Expected leading content error, got: {err_msg}"+ );+ }++ #[test]+ fn test_json_array_with_leading_whitespace_ok() {+ let input = r#"+ [{"a":1}, {"a":2}]"#;+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();+ assert_eq!(output, "{\"a\":1}\n{\"a\":2}");++ // Leading whitespace should be fine+ reader.validate_complete().unwrap();+ }++ #[test]+ fn test_validate_complete_valid_with_trailing_whitespace() {+ let input = r#"[{"a":1},{"a":2}]+ "#; // Trailing whitespace is OK+ let mut reader = JsonArrayToNdjsonReader::new(input.as_bytes());+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();++ // Whitespace after ] should be allowed+ reader.validate_complete().unwrap();+ }++ /// Test that data is not lost at buffer boundaries.+ ///+ /// This test creates input larger than the internal buffer to verify+ /// that newline characters are not dropped when they occur at buffer boundaries.+ #[test]+ fn test_buffer_boundary_no_data_loss() {+ // Create objects ~9KB each, so 10 objects = ~90KB+ let large_value = "x".repeat(9000);++ let mut objects = vec![];+ for i in 0..10 {+ objects.push(format!(r#"{{"id":{i},"data":"{large_value}"}}"#));+ }++ let input = format!("[{}]", objects.join(","));++ // Use small buffer to force multiple fill cycles+ let mut reader = JsonArrayToNdjsonReader::with_capacity(input.as_bytes(), 8192);+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();++ // Verify correct number of newlines (9 newlines separate 10 objects)+ let newline_count = output.matches('\n').count();+ assert_eq!(+ newline_count, 9,+ "Expected 9 newlines separating 10 objects, got {newline_count}"+ );++ // Verify each line is valid JSON+ for (i, line) in output.lines().enumerate() {+ let parsed: Result<serde_json::Value, _> = serde_json::from_str(line);+ assert!(+ parsed.is_ok(),+ "Line {} is not valid JSON: {}...",+ i,+ &line[..100.min(line.len())]+ );++ // Verify the id field matches expected value+ let value = parsed.unwrap();+ assert_eq!(+ value["id"].as_i64(),+ Some(i as i64),+ "Object {i} has wrong id"+ );+ }+ }++ /// Test with real-world-like data format (with leading whitespace and newlines)+ #[test]+ fn test_real_world_format_large() {+ let large_value = "x".repeat(8000);++ // Format similar to real files: opening bracket on its own line,+ // each object indented with 2 spaces+ let mut objects = vec![];+ for i in 0..10 {+ objects.push(format!(r#" {{"id":{i},"data":"{large_value}"}}"#));+ }++ let input = format!("[\n{}\n]", objects.join(",\n"));++ let mut reader = JsonArrayToNdjsonReader::with_capacity(input.as_bytes(), 8192);+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();++ let lines: Vec<&str> = output.lines().collect();+ assert_eq!(lines.len(), 10, "Expected 10 objects");++ for (i, line) in lines.iter().enumerate() {+ assert!(+ line.starts_with("{\"id\""),+ "Line {} should start with object, got: {}...",+ i,+ &line[..50.min(line.len())]+ );+ }+ }++ /// Test ChannelReader+ #[test]+ fn test_channel_reader() {+ let (tx, rx) = tokio::sync::mpsc::channel(4);++ // Send some chunks (try_send is non-async)+ tx.try_send(Bytes::from("Hello, ")).unwrap();+ tx.try_send(Bytes::from("World!")).unwrap();+ drop(tx); // Close channel++ let mut reader = ChannelReader::new(rx);+ let mut output = String::new();+ reader.read_to_string(&mut output).unwrap();++ assert_eq!(output, "Hello, World!");+ }++ /// Test ChannelReader with small reads+ #[test]+ fn test_channel_reader_small_reads() {+ let (tx, rx) = tokio::sync::mpsc::channel(4);++ tx.try_send(Bytes::from("ABCDEFGHIJ")).unwrap();+ drop(tx);++ let mut reader = ChannelReader::new(rx);+ let mut buf = [0u8; 3];++ // Read in small chunks+ assert_eq!(reader.read(&mut buf).unwrap(), 3);+ assert_eq!(&buf, b"ABC");++ assert_eq!(reader.read(&mut buf).unwrap(), 3);+ assert_eq!(&buf, b"DEF");++ assert_eq!(reader.read(&mut buf).unwrap(), 3);+ assert_eq!(&buf, b"GHI");++ assert_eq!(reader.read(&mut buf).unwrap(), 1);+ assert_eq!(&buf[..1], b"J");++ // EOF+ assert_eq!(reader.read(&mut buf).unwrap(), 0);+ }+}diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.protoindex 8a9185ca7f..62c6bbe856 100644--- a/datafusion/proto-common/proto/datafusion_common.proto+++ b/datafusion/proto-common/proto/datafusion_common.proto@@ -483,6 +483,7 @@ message JsonOptions {CompressionTypeVariant compression = 1; // Compression typeoptional uint64 schema_infer_max_rec = 2; // Optional max records for schema inferenceoptional uint32 compression_level = 3; // Optional compression level+ optional bool newline_delimited = 4; // Whether to read as newline-delimited JSON (default true). When false, expects JSON array format [{},...]}message TableParquetOptions {diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rsindex 967bda627f..ca8a269958 100644--- a/datafusion/proto-common/src/from_proto/mod.rs+++ b/datafusion/proto-common/src/from_proto/mod.rs@@ -1177,6 +1177,7 @@ impl TryFrom<&protobuf::JsonOptions> for JsonOptions {compression: compression.into(),compression_level: proto_opts.compression_level,schema_infer_max_rec: proto_opts.schema_infer_max_rec.map(|h| h as usize),+ newline_delimited: proto_opts.newline_delimited.unwrap_or(true),})}}diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rsindex 80dff4410b..b00e7546bb 100644--- a/datafusion/proto-common/src/generated/pbjson.rs+++ b/datafusion/proto-common/src/generated/pbjson.rs@@ -4603,6 +4603,9 @@ impl serde::Serialize for JsonOptions {if self.compression_level.is_some() {len += 1;}+ if self.newline_delimited.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion_common.JsonOptions", len)?;if self.compression != 0 {let v = CompressionTypeVariant::try_from(self.compression)@@ -4617,6 +4620,9 @@ impl serde::Serialize for JsonOptions {if let Some(v) = self.compression_level.as_ref() {struct_ser.serialize_field("compressionLevel", v)?;}+ if let Some(v) = self.newline_delimited.as_ref() {+ struct_ser.serialize_field("newlineDelimited", v)?;+ }struct_ser.end()}}@@ -4632,6 +4638,8 @@ impl<'de> serde::Deserialize<'de> for JsonOptions {"schemaInferMaxRec","compression_level","compressionLevel",+ "newline_delimited",+ "newlineDelimited",];#[allow(clippy::enum_variant_names)]@@ -4639,6 +4647,7 @@ impl<'de> serde::Deserialize<'de> for JsonOptions {Compression,SchemaInferMaxRec,CompressionLevel,+ NewlineDelimited,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -4663,6 +4672,7 @@ impl<'de> serde::Deserialize<'de> for JsonOptions {"compression" => Ok(GeneratedField::Compression),"schemaInferMaxRec" | "schema_infer_max_rec" => Ok(GeneratedField::SchemaInferMaxRec),"compressionLevel" | "compression_level" => Ok(GeneratedField::CompressionLevel),+ "newlineDelimited" | "newline_delimited" => Ok(GeneratedField::NewlineDelimited),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -4685,6 +4695,7 @@ impl<'de> serde::Deserialize<'de> for JsonOptions {let mut compression__ = None;let mut schema_infer_max_rec__ = None;let mut compression_level__ = None;+ let mut newline_delimited__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Compression => {@@ -4709,12 +4720,19 @@ impl<'de> serde::Deserialize<'de> for JsonOptions {map_.next_value::<::std::option::Option<::pbjson::private::NumberDeserialize<_>>>()?.map(|x| x.0);}+ GeneratedField::NewlineDelimited => {+ if newline_delimited__.is_some() {+ return Err(serde::de::Error::duplicate_field("newlineDelimited"));+ }+ newline_delimited__ = map_.next_value()?;+ }}}Ok(JsonOptions {compression: compression__.unwrap_or_default(),schema_infer_max_rec: schema_infer_max_rec__,compression_level: compression_level__,+ newline_delimited: newline_delimited__,})}}diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rsindex 30ce5a773d..a09826a29b 100644--- a/datafusion/proto-common/src/generated/prost.rs+++ b/datafusion/proto-common/src/generated/prost.rs@@ -685,6 +685,9 @@ pub struct JsonOptions {/// Optional compression level#[prost(uint32, optional, tag = "3")]pub compression_level: ::core::option::Option<u32>,+ /// Whether to read as newline-delimited JSON (default true). When false, expects JSON array format \[{},...\]+ #[prost(bool, optional, tag = "4")]+ pub newline_delimited: ::core::option::Option<bool>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct TableParquetOptions {diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rsindex 01b671e37d..79e3306a4d 100644--- a/datafusion/proto-common/src/to_proto/mod.rs+++ b/datafusion/proto-common/src/to_proto/mod.rs@@ -1011,6 +1011,7 @@ impl TryFrom<&JsonOptions> for protobuf::JsonOptions {compression: compression.into(),schema_infer_max_rec: opts.schema_infer_max_rec.map(|h| h as u64),compression_level: opts.compression_level,+ newline_delimited: Some(opts.newline_delimited),})}}diff --git a/datafusion/proto/src/generated/datafusion_proto_common.rs b/datafusion/proto/src/generated/datafusion_proto_common.rsindex 30ce5a773d..a09826a29b 100644--- a/datafusion/proto/src/generated/datafusion_proto_common.rs+++ b/datafusion/proto/src/generated/datafusion_proto_common.rs@@ -685,6 +685,9 @@ pub struct JsonOptions {/// Optional compression level#[prost(uint32, optional, tag = "3")]pub compression_level: ::core::option::Option<u32>,+ /// Whether to read as newline-delimited JSON (default true). When false, expects JSON array format \[{},...\]+ #[prost(bool, optional, tag = "4")]+ pub newline_delimited: ::core::option::Option<bool>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct TableParquetOptions {diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rsindex 436a064937..08f42b0af7 100644--- a/datafusion/proto/src/logical_plan/file_formats.rs+++ b/datafusion/proto/src/logical_plan/file_formats.rs@@ -241,6 +241,7 @@ impl JsonOptionsProto {compression: options.compression as i32,schema_infer_max_rec: options.schema_infer_max_rec.map(|v| v as u64),compression_level: options.compression_level,+ newline_delimited: Some(options.newline_delimited),}} else {JsonOptionsProto::default()@@ -260,6 +261,7 @@ impl From<&JsonOptionsProto> for JsonOptions {},schema_infer_max_rec: proto.schema_infer_max_rec.map(|v| v as usize),compression_level: proto.compression_level,+ newline_delimited: proto.newline_delimited.unwrap_or(true),}}}diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex f622cb52a5..9407cbf9a0 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -28,7 +28,7 @@ use datafusion::datasource::file_format::json::{JsonFormat, JsonFormatFactory};use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,};-use datafusion::execution::options::ArrowReadOptions;+use datafusion::execution::options::{ArrowReadOptions, JsonReadOptions};use datafusion::optimizer::Optimizer;use datafusion::optimizer::optimize_unions::OptimizeUnions;use datafusion_common::parquet_config::DFParquetWriterVersion;@@ -755,7 +755,7 @@ async fn create_json_scan(ctx: &SessionContext) -> Result<LogicalPlan, DataFusioctx.register_json("t1","../core/tests/data/1.json",- NdJsonReadOptions::default(),+ JsonReadOptions::default(),).await?;diff --git a/datafusion/sqllogictest/test_files/json.slt b/datafusion/sqllogictest/test_files/json.sltindex b46b8c49d6..60bec4213d 100644--- a/datafusion/sqllogictest/test_files/json.slt+++ b/datafusion/sqllogictest/test_files/json.slt@@ -146,3 +146,31 @@ EXPLAIN SELECT id FROM json_partitioned_test WHERE part = 2----logical_plan TableScan: json_partitioned_test projection=[id], full_filters=[json_partitioned_test.part = Int32(2)]physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_table_json/part=2/data.json]]}, projection=[id], file_type=json++##########+## JSON Array Format Tests+##########++# Test reading JSON array format file with newline_delimited=false+statement ok+CREATE EXTERNAL TABLE json_array_test+STORED AS JSON+OPTIONS ('format.newline_delimited' 'false')+LOCATION '../core/tests/data/json_array.json';++query IT rowsort+SELECT a, b FROM json_array_test+----+1 hello+2 world+3 test++statement ok+DROP TABLE json_array_test;++# Test that reading JSON array format WITHOUT newline_delimited option fails+# (default is newline_delimited=true which can't parse array format correctly)+statement error Not valid JSON+CREATE EXTERNAL TABLE json_array_as_ndjson+STORED AS JSON+LOCATION '../core/tests/data/json_array.json';