Apache DataFusion SQL Query Engine
Support nested field access in `get_field` with multiple path arguments (#19389)
Adrian Garcia Badaraccoa405d3fe4a5c
diff --git a/datafusion/functions-nested/src/planner.rs b/datafusion/functions-nested/src/planner.rsindex 22b9074c8a..afb18a44f4 100644--- a/datafusion/functions-nested/src/planner.rs+++ b/datafusion/functions-nested/src/planner.rs@@ -148,6 +148,9 @@ impl ExprPlanner for FieldAccessPlanner {match field_access {// expr["field"] => get_field(expr, "field")+ // Nested accesses like expr["a"]["b"] create nested get_field calls,+ // which are then merged by the SimplifyExpressions optimizer pass via+ // the GetFieldFunc::simplify() method.GetFieldAccess::NamedStructField { name } => {Ok(PlannerResult::Planned(get_field(expr, name)))}diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rsindex a1b4167d69..3e961e4da4 100644--- a/datafusion/functions/src/core/getfield.rs+++ b/datafusion/functions/src/core/getfield.rs@@ -15,6 +15,9 @@// specific language governing permissions and limitations// under the License.+use std::any::Any;+use std::sync::Arc;+use arrow::array::{Array, BooleanArray, Capacities, MutableArrayData, Scalar, make_array,make_comparator,@@ -22,57 +25,66 @@ use arrow::array::{use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, FieldRef};use arrow_buffer::NullBuffer;+use datafusion_common::cast::{as_map_array, as_struct_array};use datafusion_common::{Result, ScalarValue, exec_err, internal_err, plan_datafusion_err,- utils::take_function_args,};+use datafusion_expr::expr::ScalarFunction;+use datafusion_expr::simplify::ExprSimplifyResult;use datafusion_expr::{- ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,+ ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,+ ScalarUDFImpl, Signature, Volatility,};-use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};use datafusion_macros::user_doc;-use std::any::Any;-use std::sync::Arc;#[user_doc(doc_section(label = "Other Functions"),description = r#"Returns a field within a map or a struct with the given key.+ Supports nested field access by providing multiple field names.Note: most users invoke `get_field` indirectly via field accesssyntax such as `my_struct_col['field_name']` which results in a call to- `get_field(my_struct_col, 'field_name')`."#,- syntax_example = "get_field(expression1, expression2)",+ `get_field(my_struct_col, 'field_name')`.+ Nested access like `my_struct['a']['b']` is optimized to a single call:+ `get_field(my_struct, 'a', 'b')`."#,+ syntax_example = "get_field(expression, field_name[, field_name2, ...])",sql_example = r#"```sql-> create table t (idx varchar, v varchar) as values ('data','fusion'), ('apache', 'arrow');-> select struct(idx, v) from t as c;-+-------------------------+-| struct(c.idx,c.v) |-+-------------------------+-| {c0: data, c1: fusion} |-| {c0: apache, c1: arrow} |-+-------------------------+-> select get_field((select struct(idx, v) from t), 'c0');-+-----------------------+-| struct(t.idx,t.v)[c0] |-+-----------------------+-| data |-| apache |-+-----------------------+-> select get_field((select struct(idx, v) from t), 'c1');-+-----------------------+-| struct(t.idx,t.v)[c1] |-+-----------------------+-| fusion |-| arrow |-+-----------------------++> -- Access a field from a struct column+> create table test( struct_col) as values+ ({name: 'Alice', age: 30}),+ ({name: 'Bob', age: 25});+> select struct_col from test;++-----------------------------++| struct_col |++-----------------------------++| {name: Alice, age: 30} |+| {name: Bob, age: 25} |++-----------------------------++> select struct_col['name'] as name from test;++-------++| name |++-------++| Alice |+| Bob |++-------+++> -- Nested field access with multiple arguments+> create table test(struct_col) as values+ ({outer: {inner_val: 42}});+> select struct_col['outer']['inner_val'] as result from test;++--------++| result |++--------++| 42 |++--------+```"#,argument(- name = "expression1",- description = "The map or struct to retrieve a field for."+ name = "expression",+ description = "The map or struct to retrieve a field from."),argument(- name = "expression2",- description = "The field name in the map or struct to retrieve data for. Must evaluate to a string."+ name = "field_name",+ description = "The field name(s) to access, in order for nested access. Must evaluate to strings."))]#[derive(Debug, PartialEq, Eq, Hash)]@@ -86,10 +98,144 @@ impl Default for GetFieldFunc {}}+/// Process a map array by finding matching keys and extracting corresponding values.+///+/// This function handles both simple (scalar) and nested key types by using+/// appropriate comparison strategies.+fn process_map_array(+ array: &dyn Array,+ key_array: Arc<dyn Array>,+) -> Result<ColumnarValue> {+ let map_array = as_map_array(array)?;+ let keys = if key_array.data_type().is_nested() {+ let comparator = make_comparator(+ map_array.keys().as_ref(),+ key_array.as_ref(),+ SortOptions::default(),+ )?;+ let len = map_array.keys().len().min(key_array.len());+ let values = (0..len).map(|i| comparator(i, i).is_eq()).collect();+ let nulls = NullBuffer::union(map_array.keys().nulls(), key_array.nulls());+ BooleanArray::new(values, nulls)+ } else {+ let be_compared = Scalar::new(key_array);+ arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())?+ };++ let original_data = map_array.entries().column(1).to_data();+ let capacity = Capacities::Array(original_data.len());+ let mut mutable =+ MutableArrayData::with_capacities(vec![&original_data], true, capacity);++ for entry in 0..map_array.len() {+ let start = map_array.value_offsets()[entry] as usize;+ let end = map_array.value_offsets()[entry + 1] as usize;++ let maybe_matched = keys+ .slice(start, end - start)+ .iter()+ .enumerate()+ .find(|(_, t)| t.unwrap());++ if maybe_matched.is_none() {+ mutable.extend_nulls(1);+ continue;+ }+ let (match_offset, _) = maybe_matched.unwrap();+ mutable.extend(0, start + match_offset, start + match_offset + 1);+ }++ let data = mutable.freeze();+ let data = make_array(data);+ Ok(ColumnarValue::Array(data))+}++/// Process a map array with a nested key type by iterating through entries+/// and using a comparator for key matching.+///+/// This specialized version is used when the key type is nested (e.g., struct, list).+fn process_map_with_nested_key(+ array: &dyn Array,+ key_array: &dyn Array,+) -> Result<ColumnarValue> {+ let map_array = as_map_array(array)?;++ let comparator =+ make_comparator(map_array.keys().as_ref(), key_array, SortOptions::default())?;++ let original_data = map_array.entries().column(1).to_data();+ let capacity = Capacities::Array(original_data.len());+ let mut mutable =+ MutableArrayData::with_capacities(vec![&original_data], true, capacity);++ for entry in 0..map_array.len() {+ let start = map_array.value_offsets()[entry] as usize;+ let end = map_array.value_offsets()[entry + 1] as usize;++ let mut found_match = false;+ for i in start..end {+ if comparator(i, 0).is_eq() {+ mutable.extend(0, i, i + 1);+ found_match = true;+ break;+ }+ }++ if !found_match {+ mutable.extend_nulls(1);+ }+ }++ let data = mutable.freeze();+ let data = make_array(data);+ Ok(ColumnarValue::Array(data))+}++/// Extract a single field from a struct or map array+fn extract_single_field(base: ColumnarValue, name: ScalarValue) -> Result<ColumnarValue> {+ let arrays = ColumnarValue::values_to_arrays(&[base])?;+ let array = Arc::clone(&arrays[0]);++ let string_value = name.try_as_str().flatten().map(|s| s.to_string());++ match (array.data_type(), name, string_value) {+ (DataType::Map(_, _), ScalarValue::List(arr), _) => {+ let key_array: Arc<dyn Array> = arr;+ process_map_array(&array, key_array)+ }+ (DataType::Map(_, _), ScalarValue::Struct(arr), _) => {+ process_map_array(&array, arr as Arc<dyn Array>)+ }+ (DataType::Map(_, _), other, _) => {+ let data_type = other.data_type();+ if data_type.is_nested() {+ process_map_with_nested_key(&array, &other.to_array()?)+ } else {+ process_map_array(&array, other.to_array()?)+ }+ }+ (DataType::Struct(_), _, Some(k)) => {+ let as_struct_array = as_struct_array(&array)?;+ match as_struct_array.column_by_name(&k) {+ None => exec_err!("Field {k} not found in struct"),+ Some(col) => Ok(ColumnarValue::Array(Arc::clone(col))),+ }+ }+ (DataType::Struct(_), name, _) => exec_err!(+ "get_field is only possible on struct with utf8 indexes. \+ Received with {name:?} index"+ ),+ (DataType::Null, _, _) => Ok(ColumnarValue::Scalar(ScalarValue::Null)),+ (dt, name, _) => exec_err!(+ "get_field is only possible on maps or structs. Received {dt} with {name:?} index"+ ),+ }+}+impl GetFieldFunc {pub fn new() -> Self {Self {- signature: Signature::any(2, Volatility::Immutable),+ signature: Signature::user_defined(Volatility::Immutable),}}}@@ -105,24 +251,47 @@ impl ScalarUDFImpl for GetFieldFunc {}fn display_name(&self, args: &[Expr]) -> Result<String> {- let [base, field_name] = take_function_args(self.name(), args)?;+ if args.len() < 2 {+ return exec_err!(+ "get_field requires at least 2 arguments, got {}",+ args.len()+ );+ }- let name = match field_name {- Expr::Literal(name, _) => name.to_string(),- other => other.schema_name().to_string(),- };+ let base = &args[0];+ let field_names: Vec<String> = args[1..]+ .iter()+ .map(|f| match f {+ Expr::Literal(name, _) => name.to_string(),+ other => other.schema_name().to_string(),+ })+ .collect();- Ok(format!("{base}[{name}]"))+ Ok(format!("{}[{}]", base, field_names.join("][")))}fn schema_name(&self, args: &[Expr]) -> Result<String> {- let [base, field_name] = take_function_args(self.name(), args)?;- let name = match field_name {- Expr::Literal(name, _) => name.to_string(),- other => other.schema_name().to_string(),- };+ if args.len() < 2 {+ return exec_err!(+ "get_field requires at least 2 arguments, got {}",+ args.len()+ );+ }- Ok(format!("{}[{}]", base.schema_name(), name))+ let base = &args[0];+ let field_names: Vec<String> = args[1..]+ .iter()+ .map(|f| match f {+ Expr::Literal(name, _) => name.to_string(),+ other => other.schema_name().to_string(),+ })+ .collect();++ Ok(format!(+ "{}[{}]",+ base.schema_name(),+ field_names.join("][")+ ))}fn signature(&self) -> &Signature {@@ -134,209 +303,243 @@ impl ScalarUDFImpl for GetFieldFunc {}fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {- // Length check handled in the signature- debug_assert_eq!(args.scalar_arguments.len(), 2);-- match (- &args.arg_fields[0].data_type(),- args.scalar_arguments[1].as_ref(),- ) {- (DataType::Map(fields, _), _) => {- match fields.data_type() {- DataType::Struct(fields) if fields.len() == 2 => {- // Arrow's MapArray is essentially a ListArray of structs with two columns. They are- // often named "key", and "value", but we don't require any specific naming here;- // instead, we assume that the second column is the "value" column both here and in- // execution.- let value_field = fields- .get(1)- .expect("fields should have exactly two members");-- Ok(value_field.as_ref().clone().with_nullable(true).into())+ // Validate minimum 2 arguments: base expression + at least one field name+ if args.scalar_arguments.len() < 2 {+ return exec_err!(+ "get_field requires at least 2 arguments, got {}",+ args.scalar_arguments.len()+ );+ }++ let mut current_field = Arc::clone(&args.arg_fields[0]);++ // Iterate through each field name (starting from index 1)+ for (i, sv) in args.scalar_arguments.iter().enumerate().skip(1) {+ match current_field.data_type() {+ DataType::Map(map_field, _) => {+ match map_field.data_type() {+ DataType::Struct(fields) if fields.len() == 2 => {+ // Arrow's MapArray is essentially a ListArray of structs with two columns. They are+ // often named "key", and "value", but we don't require any specific naming here;+ // instead, we assume that the second column is the "value" column both here and in+ // execution.+ let value_field = fields+ .get(1)+ .expect("fields should have exactly two members");++ current_field = Arc::new(+ value_field.as_ref().clone().with_nullable(true),+ );+ }+ _ => {+ return exec_err!(+ "Map fields must contain a Struct with exactly 2 fields"+ );+ }}- _ => exec_err!(- "Map fields must contain a Struct with exactly 2 fields"- ),+ }+ DataType::Struct(fields) => {+ let field_name = sv+ .as_ref()+ .and_then(|sv| {+ sv.try_as_str().flatten().filter(|s| !s.is_empty())+ })+ .ok_or_else(|| {+ datafusion_common::DataFusionError::Execution(+ "Field name must be a non-empty string".to_string(),+ )+ })?;++ let child_field = fields+ .iter()+ .find(|f| f.name() == field_name)+ .ok_or_else(|| {+ plan_datafusion_err!("Field {field_name} not found in struct")+ })?;++ let mut new_field = child_field.as_ref().clone();++ // If the parent is nullable, then getting the child must be nullable+ if current_field.is_nullable() {+ new_field = new_field.with_nullable(true);+ }+ current_field = Arc::new(new_field);+ }+ DataType::Null => {+ return Ok(Field::new(self.name(), DataType::Null, true).into());+ }+ other => {+ return exec_err!(+ "Cannot access field at argument {}: type {} is not Struct, Map, or Null",+ i,+ other+ );}}- (DataType::Struct(fields), sv) => {- sv.and_then(|sv| sv.try_as_str().flatten().filter(|s| !s.is_empty()))- .map_or_else(- || exec_err!("Field name must be a non-empty string"),- |field_name| {- fields- .iter()- .find(|f| f.name() == field_name)- .ok_or(plan_datafusion_err!(- "Field {field_name} not found in struct"- ))- .map(|f| {- let mut child_field = f.as_ref().clone();-- // If the parent is nullable, then getting the child must be nullable,- // so potentially override the return value-- if args.arg_fields[0].is_nullable() {- child_field = child_field.with_nullable(true);- }- Arc::new(child_field)- })- },- )- }- (DataType::Null, _) => {- Ok(Field::new(self.name(), DataType::Null, true).into())- }- (other, _) => exec_err!(- "The expression to get an indexed field is only valid for `Struct`, `Map` or `Null` types, got {other}"- ),}++ Ok(current_field)}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- let [base, field_name] = take_function_args(self.name(), args.args)?;+ if args.args.len() < 2 {+ return exec_err!(+ "get_field requires at least 2 arguments, got {}",+ args.args.len()+ );+ }- if base.data_type().is_null() {+ let mut current = args.args[0].clone();++ // Early exit for null base+ if current.data_type().is_null() {return Ok(ColumnarValue::Scalar(ScalarValue::Null));}- let arrays =- ColumnarValue::values_to_arrays(&[base.clone(), field_name.clone()])?;- let array = Arc::clone(&arrays[0]);- let name = match field_name {- ColumnarValue::Scalar(name) => name,- _ => {- return exec_err!(- "get_field function requires the argument field_name to be a string"- );- }- };-- fn process_map_array(- array: &dyn Array,- key_array: Arc<dyn Array>,- ) -> Result<ColumnarValue> {- let map_array = as_map_array(array)?;- let keys = if key_array.data_type().is_nested() {- let comparator = make_comparator(- map_array.keys().as_ref(),- key_array.as_ref(),- SortOptions::default(),- )?;- let len = map_array.keys().len().min(key_array.len());- let values = (0..len).map(|i| comparator(i, i).is_eq()).collect();- let nulls =- NullBuffer::union(map_array.keys().nulls(), key_array.nulls());- BooleanArray::new(values, nulls)- } else {- let be_compared = Scalar::new(key_array);- arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())?+ // Iterate through each field name+ for field_name in args.args.iter().skip(1) {+ let field_name_scalar = match field_name {+ ColumnarValue::Scalar(name) => name.clone(),+ _ => {+ return exec_err!(+ "get_field function requires all field_name arguments to be scalars"+ );+ }};- let original_data = map_array.entries().column(1).to_data();- let capacity = Capacities::Array(original_data.len());- let mut mutable =- MutableArrayData::with_capacities(vec![&original_data], true, capacity);+ current = extract_single_field(current, field_name_scalar)?;- for entry in 0..map_array.len() {- let start = map_array.value_offsets()[entry] as usize;- let end = map_array.value_offsets()[entry + 1] as usize;+ // Early exit if we hit null+ if current.data_type().is_null() {+ return Ok(ColumnarValue::Scalar(ScalarValue::Null));+ }+ }- let maybe_matched = keys- .slice(start, end - start)- .iter()- .enumerate()- .find(|(_, t)| t.unwrap());+ Ok(current)+ }- if maybe_matched.is_none() {- mutable.extend_nulls(1);- continue;- }- let (match_offset, _) = maybe_matched.unwrap();- mutable.extend(0, start + match_offset, start + match_offset + 1);+ fn simplify(+ &self,+ args: Vec<Expr>,+ _info: &dyn datafusion_expr::simplify::SimplifyInfo,+ ) -> Result<ExprSimplifyResult> {+ // Need at least 2 args (base + field)+ if args.len() < 2 {+ return Ok(ExprSimplifyResult::Original(args));+ }++ // Flatten all nested get_field calls in a single pass+ // Pattern: get_field(get_field(get_field(base, a), b), c) => get_field(base, a, b, c)++ // Collect path arguments from all nested levels+ let mut path_args_stack = Vec::new();+ let mut current_expr = &args[0];++ // Push the outermost path arguments first+ path_args_stack.push(&args[1..]);++ // Walk down the chain of nested get_field calls+ let base_expr = loop {+ if let Expr::ScalarFunction(ScalarFunction {+ func,+ args: inner_args,+ }) = current_expr+ && func+ .inner()+ .as_any()+ .downcast_ref::<GetFieldFunc>()+ .is_some()+ {+ // Store this level's path arguments (all except the first, which is base/nested call)+ path_args_stack.push(&inner_args[1..]);++ // Move to the next level down+ current_expr = &inner_args[0];+ continue;}+ // Not a get_field call, this is the base expression+ break current_expr;+ };- let data = mutable.freeze();- let data = make_array(data);- Ok(ColumnarValue::Array(data))+ // If no nested get_field calls were found, return original+ if path_args_stack.len() == args.len() - 1 {+ return Ok(ExprSimplifyResult::Original(args));}- fn process_map_with_nested_key(- array: &dyn Array,- key_array: &dyn Array,- ) -> Result<ColumnarValue> {- let map_array = as_map_array(array)?;-- let comparator = make_comparator(- map_array.keys().as_ref(),- key_array,- SortOptions::default(),- )?;-- let original_data = map_array.entries().column(1).to_data();- let capacity = Capacities::Array(original_data.len());- let mut mutable =- MutableArrayData::with_capacities(vec![&original_data], true, capacity);-- for entry in 0..map_array.len() {- let start = map_array.value_offsets()[entry] as usize;- let end = map_array.value_offsets()[entry + 1] as usize;-- let mut found_match = false;- for i in start..end {- if comparator(i, 0).is_eq() {- mutable.extend(0, i, i + 1);- found_match = true;- break;- }- }-- if !found_match {- mutable.extend_nulls(1);- }- }+ // If we found any nested get_field calls, flatten them+ // Build merged args: [base, ...all_path_args_in_correct_order]+ let mut merged_args = vec![base_expr.clone()];- let data = mutable.freeze();- let data = make_array(data);- Ok(ColumnarValue::Array(data))+ // Add path args in reverse order (innermost to outermost)+ // Stack is: [outermost_paths, ..., innermost_paths]+ // We want: [base, innermost_paths, ..., outermost_paths]+ for path_slice in path_args_stack.iter().rev() {+ merged_args.extend_from_slice(path_slice);}- match (array.data_type(), name) {- (DataType::Map(_, _), ScalarValue::List(arr)) => {- let key_array: Arc<dyn Array> = arr;- process_map_array(&array, key_array)- }- (DataType::Map(_, _), ScalarValue::Struct(arr)) => {- process_map_array(&array, arr as Arc<dyn Array>)- }- (DataType::Map(_, _), other) => {- let data_type = other.data_type();- if data_type.is_nested() {- process_map_with_nested_key(&array, &other.to_array()?)- } else {- process_map_array(&array, other.to_array()?)- }- }- (DataType::Struct(_), ScalarValue::Utf8(Some(k))) => {- let as_struct_array = as_struct_array(&array)?;- match as_struct_array.column_by_name(&k) {- None => exec_err!("get indexed field {k} not found in struct"),- Some(col) => Ok(ColumnarValue::Array(Arc::clone(col))),- }- }- (DataType::Struct(_), name) => exec_err!(- "get_field is only possible on struct with utf8 indexes. \- Received with {name:?} index"- ),- (DataType::Null, _) => Ok(ColumnarValue::Scalar(ScalarValue::Null)),- (dt, name) => exec_err!(- "get_field is only possible on maps with utf8 indexes or struct \- with utf8 indexes. Received {dt} with {name:?} index"+ Ok(ExprSimplifyResult::Simplified(Expr::ScalarFunction(+ ScalarFunction::new_udf(+ Arc::new(ScalarUDF::new_from_impl(GetFieldFunc::new())),+ merged_args,),+ )))+ }++ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {+ if arg_types.len() < 2 {+ return exec_err!(+ "get_field requires at least 2 arguments, got {}",+ arg_types.len()+ );}+ // Accept types as-is, validation happens in return_field_from_args+ Ok(arg_types.to_vec())}fn documentation(&self) -> Option<&Documentation> {self.doc()}}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::{ArrayRef, Int32Array, StructArray};+ use arrow::datatypes::Fields;++ #[test]+ fn test_get_field_utf8view_key() -> Result<()> {+ // Create a struct array with fields "a" and "b"+ let a_values = Int32Array::from(vec![Some(1), Some(2), Some(3)]);+ let b_values = Int32Array::from(vec![Some(10), Some(20), Some(30)]);++ let fields: Fields = vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ ]+ .into();++ let struct_array = StructArray::new(+ fields,+ vec![+ Arc::new(a_values) as ArrayRef,+ Arc::new(b_values) as ArrayRef,+ ],+ None,+ );++ let base = ColumnarValue::Array(Arc::new(struct_array));++ // Use Utf8View key to access field "a"+ let key = ScalarValue::Utf8View(Some("a".to_string()));++ let result = extract_single_field(base, key)?;++ let result_array = result.into_array(3)?;+ let expected = Int32Array::from(vec![Some(1), Some(2), Some(3)]);++ assert_eq!(result_array.as_ref(), &expected as &dyn Array);++ Ok(())+ }+}diff --git a/datafusion/functions/src/core/mod.rs b/datafusion/functions/src/core/mod.rsindex 82b0097aa7..a14d563737 100644--- a/datafusion/functions/src/core/mod.rs+++ b/datafusion/functions/src/core/mod.rs@@ -121,6 +121,13 @@ pub mod expr_fn {super::get_field().call(vec![arg1, arg2.lit()])}+ #[doc = "Returns the value of nested fields by traversing multiple field names"]+ pub fn get_field_path(base: Expr, field_names: Vec<Expr>) -> Expr {+ let mut args = vec![base];+ args.extend(field_names);+ super::get_field().call(args)+ }+#[doc = "Returns the value of the field with the given name from the union when it's selected, or NULL otherwise"]#[expect(clippy::needless_pass_by_value)]pub fn union_extract(arg1: Expr, arg2: impl Literal) -> Expr {diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex 3391408e04..5746a568e7 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -629,21 +629,27 @@ impl Unparser<'_> {}fn get_field_to_sql(&self, args: &[Expr]) -> Result<ast::Expr> {- assert_eq_or_internal_err!(- args.len(),- 2,- "get_field must have exactly 2 arguments"- );+ if args.len() < 2 {+ return internal_err!(+ "get_field must have at least 2 arguments, got {}",+ args.len()+ );+ }- let field = match &args[1] {- Expr::Literal(lit, _) => self.new_ident_quoted_if_needs(lit.to_string()),- _ => {- return internal_err!(- "get_field expects second argument to be a string, but received: {:?}",- &args[1]- );- }- };+ // Extract all field names (args[1..])+ let mut fields = Vec::with_capacity(args.len() - 1);+ for arg in &args[1..] {+ let field = match arg {+ Expr::Literal(lit, _) => self.new_ident_quoted_if_needs(lit.to_string()),+ _ => {+ return internal_err!(+ "get_field expects field arguments to be strings, but received: {:?}",+ arg+ );+ }+ };+ fields.push(field);+ }match &args[0] {Expr::Column(col) => {@@ -657,17 +663,19 @@ impl Unparser<'_> {);}};- id.push(field);+ id.extend(fields);Ok(ast::Expr::CompoundIdentifier(id))}Expr::ScalarFunction(struct_expr) => {let root = self.scalar_function_to_sql(struct_expr.func.name(), &struct_expr.args)?;+ let access_chain = fields+ .into_iter()+ .map(|field| ast::AccessExpr::Dot(ast::Expr::Identifier(field)))+ .collect();Ok(ast::Expr::CompoundFieldAccess {root: Box::new(root),- access_chain: vec![ast::AccessExpr::Dot(ast::Expr::Identifier(- field,- ))],+ access_chain,})}_ => {diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.sltindex 0989dd382d..d985af1104 100644--- a/datafusion/sqllogictest/test_files/struct.slt+++ b/datafusion/sqllogictest/test_files/struct.slt@@ -653,3 +653,174 @@ test1.struct_field['substruct']['subsubstruct'] = test2.struct_field['substruct'statement okdrop table test;++# Test nested get_field with multiple arguments+query I+select get_field({'a': {'b': 1}}, 'a', 'b');+----+1++# Test nested get_field with three levels+query I+select get_field({'a': {'b': {'c': 42}}}, 'a', 'b', 'c');+----+42++# Test type validation error - accessing field on non-struct+query error Cannot access field at argument 2: type Int64 is not Struct, Map, or Null+select get_field({'a': 1}, 'a', 'b');++# Test that bracket syntax produces a single get_field call (not nested)+# We use a table column to prevent constant folding+statement ok+create table explain_test (s struct(a struct(b int))) as values ({'a': {'b': 1}});++query TT+explain select s['a']['b'] from explain_test;+----+logical_plan+01)Projection: get_field(explain_test.s, Utf8("a"), Utf8("b"))+02)--TableScan: explain_test projection=[s]+physical_plan+01)ProjectionExec: expr=[get_field(s@0, a, b) as explain_test.s[a][b]]+02)--DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+drop table explain_test;++# Test with nested struct in table+statement ok+create table nested_test (s struct(inner struct(val int))) as values ({'inner': {'val': 100}});++query I+select s['inner']['val'] from nested_test;+----+100++query I+select get_field(s, 'inner', 'val') from nested_test;+----+100++statement ok+drop table nested_test;++# Test mixed struct and map access+query I+select get_field({'m': map(['k'], [42])}, 'm', 'k');+----+42++# Test nested map access+query I+select {'m': map(['outer'], [map(['inner'], [99])])}['m']['outer']['inner'];+----+99++###############################################+# Additional tests for nested get_field support+###############################################++# Backwards compatibility: original 2-argument form still works+query I+select get_field({'a': 1}, 'a');+----+1++# Error: get_field with no arguments+query error get_field requires at least 2 arguments, got 0+select get_field();++# Error: get_field with only 1 argument+query error get_field requires at least 2 arguments, got 1+select get_field({'a': 1});++# Null handling: null at base should return null+query I+select get_field(CAST(NULL AS STRUCT(a STRUCT(b INT))), 'a', 'b');+----+NULL++# Null handling: null in middle of chain+statement ok+create table null_mid_test (s STRUCT(a STRUCT(b INT)));++statement ok+insert into null_mid_test values ({'a': NULL});++query I+select s['a']['b'] from null_mid_test;+----+NULL++query I+select get_field(s, 'a', 'b') from null_mid_test;+----+NULL++statement ok+drop table null_mid_test;++# Type validation error at argument 3+query error Cannot access field at argument 3: type Int64 is not Struct, Map, or Null+select get_field({'a': {'b': 2}}, 'a', 'b', 'c');++# Type validation error at argument 4+query error Cannot access field at argument 4: type Int64 is not Struct, Map, or Null+select get_field({'a': {'b': {'c': 3}}}, 'a', 'b', 'c', 'd');++# Non-existent field at first level+query error Field x not found in struct+select get_field({'a': 1}, 'x');++# Non-existent field at second level+query error Field x not found in struct+select get_field({'a': {'b': 1}}, 'a', 'x');++# Deep nesting: 5-level access+query I+select get_field({'l1': {'l2': {'l3': {'l4': {'l5': 42}}}}}, 'l1', 'l2', 'l3', 'l4', 'l5');+----+42++# Deep nesting: 5-level access via bracket syntax+query I+select {'l1': {'l2': {'l3': {'l4': {'l5': 99}}}}}['l1']['l2']['l3']['l4']['l5'];+----+99++# Mixed array and struct access: array index should break the batching+statement ok+create table mixed_access_test (data STRUCT(items STRUCT(name VARCHAR)[]) );++statement ok+insert into mixed_access_test values ({'items': [{'name': 'first'}, {'name': 'second'}]});++query T+select data['items'][1]['name'] from mixed_access_test;+----+first++query T+select data['items'][2]['name'] from mixed_access_test;+----+second++statement ok+drop table mixed_access_test;++# Nullable parent propagation: null parent should propagate+statement ok+create table nullable_parent_test (s STRUCT(a STRUCT(b INT)));++statement ok+insert into nullable_parent_test values ({'a': {'b': 1}}), (NULL);++query I+select s['a']['b'] from nullable_parent_test;+----+1+NULL++statement ok+drop table nullable_parent_test;diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.mdindex 3f0939d00d..e454788003 100644--- a/docs/source/user-guide/sql/scalar_functions.md+++ b/docs/source/user-guide/sql/scalar_functions.md@@ -5098,44 +5098,53 @@ arrow_typeof(expression)### `get_field`Returns a field within a map or a struct with the given key.+Supports nested field access by providing multiple field names.Note: most users invoke `get_field` indirectly via field accesssyntax such as `my_struct_col['field_name']` which results in a call to`get_field(my_struct_col, 'field_name')`.+Nested access like `my_struct['a']['b']` is optimized to a single call:+`get_field(my_struct, 'a', 'b')`.```sql-get_field(expression1, expression2)+get_field(expression, field_name[, field_name2, ...])```#### Arguments-- **expression1**: The map or struct to retrieve a field for.-- **expression2**: The field name in the map or struct to retrieve data for. Must evaluate to a string.+- **expression**: The map or struct to retrieve a field from.+- **field_name**: The field name(s) to access, in order for nested access. Must evaluate to strings.#### Example```sql-> create table t (idx varchar, v varchar) as values ('data','fusion'), ('apache', 'arrow');-> select struct(idx, v) from t as c;-+-------------------------+-| struct(c.idx,c.v) |-+-------------------------+-| {c0: data, c1: fusion} |-| {c0: apache, c1: arrow} |-+-------------------------+-> select get_field((select struct(idx, v) from t), 'c0');-+-----------------------+-| struct(t.idx,t.v)[c0] |-+-----------------------+-| data |-| apache |-+-----------------------+-> select get_field((select struct(idx, v) from t), 'c1');-+-----------------------+-| struct(t.idx,t.v)[c1] |-+-----------------------+-| fusion |-| arrow |-+-----------------------++> -- Access a field from a struct column+> create table test( struct_col) as values+ ({name: 'Alice', age: 30}),+ ({name: 'Bob', age: 25});+> select struct_col from test;++-----------------------------++| struct_col |++-----------------------------++| {name: Alice, age: 30} |+| {name: Bob, age: 25} |++-----------------------------++> select struct_col['name'] as name from test;++-------++| name |++-------++| Alice |+| Bob |++-------+++> -- Nested field access with multiple arguments+> create table test(struct_col) as values+ ({outer: {inner_val: 42}});+> select struct_col['outer']['inner_val'] as result from test;++--------++| result |++--------++| 42 |++--------+```### `version`