Apache DataFusion SQL Query Engine
feat: support Spark-compatible `json_tuple` function (#20412)
Yu-Chuan Hung0f7a405b8c99
diff --git a/Cargo.lock b/Cargo.lockindex 7fd3909957..d2ddda3013 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2595,6 +2595,7 @@ dependencies = ["log","percent-encoding","rand 0.9.2",+ "serde_json","sha1","sha2","url",diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.tomlindex 8a5c68a5d4..162b6d814e 100644--- a/datafusion/spark/Cargo.toml+++ b/datafusion/spark/Cargo.toml@@ -59,6 +59,7 @@ datafusion-functions-nested = { workspace = true }log = { workspace = true }percent-encoding = "2.3.2"rand = { workspace = true }+serde_json = { workspace = true }sha1 = "0.10"sha2 = { workspace = true }url = { workspace = true }diff --git a/datafusion/spark/src/function/json/json_tuple.rs b/datafusion/spark/src/function/json/json_tuple.rsnew file mode 100644index 0000000000..f3ba7e91ac--- /dev/null+++ b/datafusion/spark/src/function/json/json_tuple.rs@@ -0,0 +1,244 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use std::any::Any;+use std::sync::Arc;++use arrow::array::{Array, ArrayRef, NullBufferBuilder, StringBuilder, StructArray};+use arrow::datatypes::{DataType, Field, FieldRef, Fields};+use datafusion_common::cast::as_string_array;+use datafusion_common::{Result, exec_err, internal_err};+use datafusion_expr::{+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ Volatility,+};++/// Spark-compatible `json_tuple` expression+///+/// <https://spark.apache.org/docs/latest/api/sql/index.html#json_tuple>+///+/// Extracts top-level fields from a JSON string and returns them as a struct.+///+/// `json_tuple(json_string, field1, field2, ...) -> Struct<c0: Utf8, c1: Utf8, ...>`+///+/// Note: In Spark, `json_tuple` is a Generator that produces multiple columns directly.+/// In DataFusion, a ScalarUDF can only return one value per row, so the result is wrapped+/// in a Struct. The caller (e.g. Comet) is expected to destructure the struct fields.+///+/// - Returns NULL for each field that is missing from the JSON object+/// - Returns NULL for all fields if the input is NULL or not valid JSON+/// - Non-string JSON values are converted to their JSON string representation+/// - JSON `null` values are returned as NULL (not the string "null")+#[derive(Debug, PartialEq, Eq, Hash)]+pub struct JsonTuple {+ signature: Signature,+}++impl Default for JsonTuple {+ fn default() -> Self {+ Self::new()+ }+}++impl JsonTuple {+ pub fn new() -> Self {+ Self {+ signature: Signature::variadic(vec![DataType::Utf8], Volatility::Immutable),+ }+ }+}++impl ScalarUDFImpl for JsonTuple {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn name(&self) -> &str {+ "json_tuple"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ internal_err!("return_field_from_args should be used instead")+ }++ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ if args.arg_fields.len() < 2 {+ return exec_err!(+ "json_tuple requires at least 2 arguments (json_string, field1), got {}",+ args.arg_fields.len()+ );+ }++ let num_fields = args.arg_fields.len() - 1;+ let fields: Fields = (0..num_fields)+ .map(|i| Field::new(format!("c{i}"), DataType::Utf8, true))+ .collect::<Vec<_>>()+ .into();++ Ok(Arc::new(Field::new(+ self.name(),+ DataType::Struct(fields),+ true,+ )))+ }++ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ let ScalarFunctionArgs {+ args: arg_values,+ return_field,+ ..+ } = args;+ let arrays = ColumnarValue::values_to_arrays(&arg_values)?;+ let result = json_tuple_inner(&arrays, return_field.data_type())?;++ Ok(ColumnarValue::Array(result))+ }+}++fn json_tuple_inner(args: &[ArrayRef], return_type: &DataType) -> Result<ArrayRef> {+ let num_rows = args[0].len();+ let num_fields = args.len() - 1;++ let json_array = as_string_array(&args[0])?;++ let field_arrays = args[1..]+ .iter()+ .map(|arg| as_string_array(arg))+ .collect::<Result<Vec<_>>>()?;++ let mut builders: Vec<StringBuilder> =+ (0..num_fields).map(|_| StringBuilder::new()).collect();++ let mut null_buffer = NullBufferBuilder::new(num_rows);++ for row_idx in 0..num_rows {+ if json_array.is_null(row_idx) {+ for builder in &mut builders {+ builder.append_null();+ }+ null_buffer.append_null();+ continue;+ }++ let json_str = json_array.value(row_idx);+ match serde_json::from_str::<serde_json::Value>(json_str) {+ Ok(serde_json::Value::Object(map)) => {+ null_buffer.append_non_null();+ for (field_idx, builder) in builders.iter_mut().enumerate() {+ if field_arrays[field_idx].is_null(row_idx) {+ builder.append_null();+ continue;+ }+ let field_name = field_arrays[field_idx].value(row_idx);+ match map.get(field_name) {+ Some(serde_json::Value::Null) => {+ builder.append_null();+ }+ Some(serde_json::Value::String(s)) => {+ builder.append_value(s);+ }+ Some(other) => {+ builder.append_value(other.to_string());+ }+ None => {+ builder.append_null();+ }+ }+ }+ }+ _ => {+ for builder in &mut builders {+ builder.append_null();+ }+ null_buffer.append_null();+ }+ }+ }++ let struct_fields = match return_type {+ DataType::Struct(fields) => fields.clone(),+ _ => {+ return internal_err!(+ "json_tuple requires a Struct return type, got {:?}",+ return_type+ );+ }+ };++ let arrays: Vec<ArrayRef> = builders+ .into_iter()+ .map(|mut builder| Arc::new(builder.finish()) as ArrayRef)+ .collect();++ let struct_array = StructArray::try_new(struct_fields, arrays, null_buffer.finish())?;++ Ok(Arc::new(struct_array))+}++#[cfg(test)]+mod tests {+ use super::*;+ use datafusion_expr::ReturnFieldArgs;++ #[test]+ fn test_return_field_shape() {+ let func = JsonTuple::new();+ let fields = vec![+ Arc::new(Field::new("json", DataType::Utf8, false)),+ Arc::new(Field::new("f1", DataType::Utf8, false)),+ Arc::new(Field::new("f2", DataType::Utf8, false)),+ ];+ let result = func+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &fields,+ scalar_arguments: &[None, None, None],+ })+ .unwrap();++ match result.data_type() {+ DataType::Struct(inner) => {+ assert_eq!(inner.len(), 2);+ assert_eq!(inner[0].name(), "c0");+ assert_eq!(inner[1].name(), "c1");+ assert_eq!(inner[0].data_type(), &DataType::Utf8);+ assert!(inner[0].is_nullable());+ }+ other => panic!("Expected Struct, got {other:?}"),+ }+ }++ #[test]+ fn test_too_few_args() {+ let func = JsonTuple::new();+ let fields = vec![Arc::new(Field::new("json", DataType::Utf8, false))];+ let result = func.return_field_from_args(ReturnFieldArgs {+ arg_fields: &fields,+ scalar_arguments: &[None],+ });+ assert!(result.is_err());+ assert!(+ result+ .unwrap_err()+ .to_string()+ .contains("at least 2 arguments")+ );+ }+}diff --git a/datafusion/spark/src/function/json/mod.rs b/datafusion/spark/src/function/json/mod.rsindex a87df9a2c8..01378235d7 100644--- a/datafusion/spark/src/function/json/mod.rs+++ b/datafusion/spark/src/function/json/mod.rs@@ -15,11 +15,24 @@// specific language governing permissions and limitations// under the License.+pub mod json_tuple;+use datafusion_expr::ScalarUDF;+use datafusion_functions::make_udf_function;use std::sync::Arc;-pub mod expr_fn {}+make_udf_function!(json_tuple::JsonTuple, json_tuple);++pub mod expr_fn {+ use datafusion_functions::export_functions;++ export_functions!((+ json_tuple,+ "Extracts top-level fields from a JSON string and returns them as a struct.",+ args,+ ));+}pub fn functions() -> Vec<Arc<ScalarUDF>> {- vec![]+ vec![json_tuple()]}diff --git a/datafusion/sqllogictest/test_files/spark/json/json_tuple.slt b/datafusion/sqllogictest/test_files/spark/json/json_tuple.sltnew file mode 100644index 0000000000..c0c4249467--- /dev/null+++ b/datafusion/sqllogictest/test_files/spark/json/json_tuple.slt@@ -0,0 +1,154 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at++# http://www.apache.org/licenses/LICENSE-2.0++# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Tests for Spark-compatible json_tuple function+# https://spark.apache.org/docs/latest/api/sql/index.html#json_tuple+#+# Test cases derived from Spark JsonExpressionsSuite:+# https://github.com/apache/spark/blob/master/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala++# Scalar: hive key 1+query ?+SELECT json_tuple('{"f1":"value1","f2":"value2","f3":3,"f5":5.23}'::STRING, 'f1'::STRING, 'f2'::STRING, 'f3'::STRING, 'f4'::STRING, 'f5'::STRING);+----+{c0: value1, c1: value2, c2: 3, c3: NULL, c4: 5.23}++# Scalar: hive key 2+query ?+SELECT json_tuple('{"f1":"value12","f3":"value3","f2":2,"f4":4.01}'::STRING, 'f1'::STRING, 'f2'::STRING, 'f3'::STRING, 'f4'::STRING, 'f5'::STRING);+----+{c0: value12, c1: 2, c2: value3, c3: 4.01, c4: NULL}++# Scalar: hive key 3+query ?+SELECT json_tuple('{"f1":"value13","f4":"value44","f3":"value33","f2":2,"f5":5.01}'::STRING, 'f1'::STRING, 'f2'::STRING, 'f3'::STRING, 'f4'::STRING, 'f5'::STRING);+----+{c0: value13, c1: 2, c2: value33, c3: value44, c4: 5.01}++# Scalar: null JSON input+query ?+SELECT json_tuple(NULL::STRING, 'f1'::STRING, 'f2'::STRING, 'f3'::STRING, 'f4'::STRING, 'f5'::STRING);+----+NULL++# Scalar: null and empty values+query ?+SELECT json_tuple('{"f1":"","f5":null}'::STRING, 'f1'::STRING, 'f2'::STRING, 'f3'::STRING, 'f4'::STRING, 'f5'::STRING);+----+{c0: , c1: NULL, c2: NULL, c3: NULL, c4: NULL}++# Scalar: invalid JSON (array)+query ?+SELECT json_tuple('[invalid JSON string]'::STRING, 'f1'::STRING);+----+NULL++# Scalar: invalid JSON (start only)+query ?+SELECT json_tuple('{'::STRING, 'f1'::STRING);+----+NULL++# Scalar: invalid JSON (no closing brace)+query ?+SELECT json_tuple('{"foo":"bar"'::STRING, 'f1'::STRING);+----+NULL++# Scalar: invalid JSON (backslash)+query ?+SELECT json_tuple('\'::STRING, 'f1'::STRING);+----+NULL++# Scalar: invalid JSON (quoted string, not an object)+query ?+SELECT json_tuple('"quote'::STRING, '"quote'::STRING);+----+NULL++# Scalar: empty JSON object+query ?+SELECT json_tuple('{}'::STRING, 'a'::STRING);+----+{c0: NULL}++# Array: multi-row test+query ?+SELECT json_tuple(col, 'f1'::STRING, 'f2'::STRING) FROM (VALUES+ ('{"f1":"a","f2":"b"}'::STRING),+ (NULL::STRING),+ ('{"f1":"c"}'::STRING),+ ('invalid'::STRING)+) AS t(col);+----+{c0: a, c1: b}+NULL+{c0: c, c1: NULL}+NULL++# Array: SPARK-21677 null field key+query ?+SELECT json_tuple(col1, col2, col3, col4) FROM (VALUES+ ('{"f1":1,"f2":2}'::STRING, 'f1'::STRING, NULL::STRING, 'f2'::STRING)+) AS t(col1, col2, col3, col4);+----+{c0: 1, c1: NULL, c2: 2}++# Array: SPARK-21804 repeated field+query ?+SELECT json_tuple(col1, col2, col3, col4) FROM (VALUES+ ('{"f1":1,"f2":2}'::STRING, 'f1'::STRING, NULL::STRING, 'f1'::STRING)+) AS t(col1, col2, col3, col4);+----+{c0: 1, c1: NULL, c2: 1}++# Edge case: both json and field key are null+query ?+SELECT json_tuple(NULL::STRING, NULL::STRING);+----+NULL++# Edge case: empty string json and empty string key+query ?+SELECT json_tuple(''::STRING, ''::STRING);+----+NULL++# Edge case: mixed upper/lower case keys+query ?+SELECT json_tuple('{"Name":"Alice","name":"bob","NAME":"Charlie"}'::STRING, 'Name'::STRING, 'name'::STRING, 'NAME'::STRING);+----+{c0: Alice, c1: bob, c2: Charlie}++# Edge case: UTF-8 Chinese characters+query ?+SELECT json_tuple('{"姓名":"小明","城市":"台北"}'::STRING, '姓名'::STRING, '城市'::STRING);+----+{c0: 小明, c1: 台北}++# Edge case: UTF-8 Cyrillic characters+query ?+SELECT json_tuple('{"имя":"Иван","город":"Москва"}'::STRING, 'имя'::STRING, 'город'::STRING);+----+{c0: Иван, c1: Москва}++# Verify return type with arrow_typeof+query T+SELECT arrow_typeof(json_tuple('{"a":1}'::STRING, 'a'::STRING));+----+Struct("c0": Utf8)