Skip to content

Apache DataFusion SQL Query Engine

Support nested field access in `get_field` with multiple path arguments (#19389)

Adrian Garcia Badaraccoa405d3fe4a5c
diff --git a/datafusion/functions-nested/src/planner.rs b/datafusion/functions-nested/src/planner.rs
index 22b9074c8a..afb18a44f4 100644
--- a/datafusion/functions-nested/src/planner.rs
+++ b/datafusion/functions-nested/src/planner.rs
@@ -148,6 +148,9 @@ impl ExprPlanner for FieldAccessPlanner {
match field_access {
// expr["field"] => get_field(expr, "field")
+ // Nested accesses like expr["a"]["b"] create nested get_field calls,
+ // which are then merged by the SimplifyExpressions optimizer pass via
+ // the GetFieldFunc::simplify() method.
GetFieldAccess::NamedStructField { name } => {
Ok(PlannerResult::Planned(get_field(expr, name)))
}
diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rs
index a1b4167d69..3e961e4da4 100644
--- a/datafusion/functions/src/core/getfield.rs
+++ b/datafusion/functions/src/core/getfield.rs
@@ -15,6 +15,9 @@
// specific language governing permissions and limitations
// under the License.
+use std::any::Any;
+use std::sync::Arc;
+
use arrow::array::{
Array, BooleanArray, Capacities, MutableArrayData, Scalar, make_array,
make_comparator,
@@ -22,57 +25,66 @@ use arrow::array::{
use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, FieldRef};
use arrow_buffer::NullBuffer;
+
use datafusion_common::cast::{as_map_array, as_struct_array};
use datafusion_common::{
Result, ScalarValue, exec_err, internal_err, plan_datafusion_err,
- utils::take_function_args,
};
+use datafusion_expr::expr::ScalarFunction;
+use datafusion_expr::simplify::ExprSimplifyResult;
use datafusion_expr::{
- ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,
+ ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,
+ ScalarUDFImpl, Signature, Volatility,
};
-use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
use datafusion_macros::user_doc;
-use std::any::Any;
-use std::sync::Arc;
#[user_doc(
doc_section(label = "Other Functions"),
description = r#"Returns a field within a map or a struct with the given key.
+ Supports nested field access by providing multiple field names.
Note: most users invoke `get_field` indirectly via field access
syntax such as `my_struct_col['field_name']` which results in a call to
- `get_field(my_struct_col, 'field_name')`."#,
- syntax_example = "get_field(expression1, expression2)",
+ `get_field(my_struct_col, 'field_name')`.
+ Nested access like `my_struct['a']['b']` is optimized to a single call:
+ `get_field(my_struct, 'a', 'b')`."#,
+ syntax_example = "get_field(expression, field_name[, field_name2, ...])",
sql_example = r#"```sql
-> create table t (idx varchar, v varchar) as values ('data','fusion'), ('apache', 'arrow');
-> select struct(idx, v) from t as c;
-+-------------------------+
-| struct(c.idx,c.v) |
-+-------------------------+
-| {c0: data, c1: fusion} |
-| {c0: apache, c1: arrow} |
-+-------------------------+
-> select get_field((select struct(idx, v) from t), 'c0');
-+-----------------------+
-| struct(t.idx,t.v)[c0] |
-+-----------------------+
-| data |
-| apache |
-+-----------------------+
-> select get_field((select struct(idx, v) from t), 'c1');
-+-----------------------+
-| struct(t.idx,t.v)[c1] |
-+-----------------------+
-| fusion |
-| arrow |
-+-----------------------+
+> -- Access a field from a struct column
+> create table test( struct_col) as values
+ ({name: 'Alice', age: 30}),
+ ({name: 'Bob', age: 25});
+> select struct_col from test;
++-----------------------------+
+| struct_col |
++-----------------------------+
+| {name: Alice, age: 30} |
+| {name: Bob, age: 25} |
++-----------------------------+
+> select struct_col['name'] as name from test;
++-------+
+| name |
++-------+
+| Alice |
+| Bob |
++-------+
+
+> -- Nested field access with multiple arguments
+> create table test(struct_col) as values
+ ({outer: {inner_val: 42}});
+> select struct_col['outer']['inner_val'] as result from test;
++--------+
+| result |
++--------+
+| 42 |
++--------+
```"#,
argument(
- name = "expression1",
- description = "The map or struct to retrieve a field for."
+ name = "expression",
+ description = "The map or struct to retrieve a field from."
),
argument(
- name = "expression2",
- description = "The field name in the map or struct to retrieve data for. Must evaluate to a string."
+ name = "field_name",
+ description = "The field name(s) to access, in order for nested access. Must evaluate to strings."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
@@ -86,10 +98,144 @@ impl Default for GetFieldFunc {
}
}
+/// Process a map array by finding matching keys and extracting corresponding values.
+///
+/// This function handles both simple (scalar) and nested key types by using
+/// appropriate comparison strategies.
+fn process_map_array(
+ array: &dyn Array,
+ key_array: Arc<dyn Array>,
+) -> Result<ColumnarValue> {
+ let map_array = as_map_array(array)?;
+ let keys = if key_array.data_type().is_nested() {
+ let comparator = make_comparator(
+ map_array.keys().as_ref(),
+ key_array.as_ref(),
+ SortOptions::default(),
+ )?;
+ let len = map_array.keys().len().min(key_array.len());
+ let values = (0..len).map(|i| comparator(i, i).is_eq()).collect();
+ let nulls = NullBuffer::union(map_array.keys().nulls(), key_array.nulls());
+ BooleanArray::new(values, nulls)
+ } else {
+ let be_compared = Scalar::new(key_array);
+ arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())?
+ };
+
+ let original_data = map_array.entries().column(1).to_data();
+ let capacity = Capacities::Array(original_data.len());
+ let mut mutable =
+ MutableArrayData::with_capacities(vec![&original_data], true, capacity);
+
+ for entry in 0..map_array.len() {
+ let start = map_array.value_offsets()[entry] as usize;
+ let end = map_array.value_offsets()[entry + 1] as usize;
+
+ let maybe_matched = keys
+ .slice(start, end - start)
+ .iter()
+ .enumerate()
+ .find(|(_, t)| t.unwrap());
+
+ if maybe_matched.is_none() {
+ mutable.extend_nulls(1);
+ continue;
+ }
+ let (match_offset, _) = maybe_matched.unwrap();
+ mutable.extend(0, start + match_offset, start + match_offset + 1);
+ }
+
+ let data = mutable.freeze();
+ let data = make_array(data);
+ Ok(ColumnarValue::Array(data))
+}
+
+/// Process a map array with a nested key type by iterating through entries
+/// and using a comparator for key matching.
+///
+/// This specialized version is used when the key type is nested (e.g., struct, list).
+fn process_map_with_nested_key(
+ array: &dyn Array,
+ key_array: &dyn Array,
+) -> Result<ColumnarValue> {
+ let map_array = as_map_array(array)?;
+
+ let comparator =
+ make_comparator(map_array.keys().as_ref(), key_array, SortOptions::default())?;
+
+ let original_data = map_array.entries().column(1).to_data();
+ let capacity = Capacities::Array(original_data.len());
+ let mut mutable =
+ MutableArrayData::with_capacities(vec![&original_data], true, capacity);
+
+ for entry in 0..map_array.len() {
+ let start = map_array.value_offsets()[entry] as usize;
+ let end = map_array.value_offsets()[entry + 1] as usize;
+
+ let mut found_match = false;
+ for i in start..end {
+ if comparator(i, 0).is_eq() {
+ mutable.extend(0, i, i + 1);
+ found_match = true;
+ break;
+ }
+ }
+
+ if !found_match {
+ mutable.extend_nulls(1);
+ }
+ }
+
+ let data = mutable.freeze();
+ let data = make_array(data);
+ Ok(ColumnarValue::Array(data))
+}
+
+/// Extract a single field from a struct or map array
+fn extract_single_field(base: ColumnarValue, name: ScalarValue) -> Result<ColumnarValue> {
+ let arrays = ColumnarValue::values_to_arrays(&[base])?;
+ let array = Arc::clone(&arrays[0]);
+
+ let string_value = name.try_as_str().flatten().map(|s| s.to_string());
+
+ match (array.data_type(), name, string_value) {
+ (DataType::Map(_, _), ScalarValue::List(arr), _) => {
+ let key_array: Arc<dyn Array> = arr;
+ process_map_array(&array, key_array)
+ }
+ (DataType::Map(_, _), ScalarValue::Struct(arr), _) => {
+ process_map_array(&array, arr as Arc<dyn Array>)
+ }
+ (DataType::Map(_, _), other, _) => {
+ let data_type = other.data_type();
+ if data_type.is_nested() {
+ process_map_with_nested_key(&array, &other.to_array()?)
+ } else {
+ process_map_array(&array, other.to_array()?)
+ }
+ }
+ (DataType::Struct(_), _, Some(k)) => {
+ let as_struct_array = as_struct_array(&array)?;
+ match as_struct_array.column_by_name(&k) {
+ None => exec_err!("Field {k} not found in struct"),
+ Some(col) => Ok(ColumnarValue::Array(Arc::clone(col))),
+ }
+ }
+ (DataType::Struct(_), name, _) => exec_err!(
+ "get_field is only possible on struct with utf8 indexes. \
+ Received with {name:?} index"
+ ),
+ (DataType::Null, _, _) => Ok(ColumnarValue::Scalar(ScalarValue::Null)),
+ (dt, name, _) => exec_err!(
+ "get_field is only possible on maps or structs. Received {dt} with {name:?} index"
+ ),
+ }
+}
+
impl GetFieldFunc {
pub fn new() -> Self {
Self {
- signature: Signature::any(2, Volatility::Immutable),
+ signature: Signature::user_defined(Volatility::Immutable),
}
}
}
@@ -105,24 +251,47 @@ impl ScalarUDFImpl for GetFieldFunc {
}
fn display_name(&self, args: &[Expr]) -> Result<String> {
- let [base, field_name] = take_function_args(self.name(), args)?;
+ if args.len() < 2 {
+ return exec_err!(
+ "get_field requires at least 2 arguments, got {}",
+ args.len()
+ );
+ }
- let name = match field_name {
- Expr::Literal(name, _) => name.to_string(),
- other => other.schema_name().to_string(),
- };
+ let base = &args[0];
+ let field_names: Vec<String> = args[1..]
+ .iter()
+ .map(|f| match f {
+ Expr::Literal(name, _) => name.to_string(),
+ other => other.schema_name().to_string(),
+ })
+ .collect();
- Ok(format!("{base}[{name}]"))
+ Ok(format!("{}[{}]", base, field_names.join("][")))
}
fn schema_name(&self, args: &[Expr]) -> Result<String> {
- let [base, field_name] = take_function_args(self.name(), args)?;
- let name = match field_name {
- Expr::Literal(name, _) => name.to_string(),
- other => other.schema_name().to_string(),
- };
+ if args.len() < 2 {
+ return exec_err!(
+ "get_field requires at least 2 arguments, got {}",
+ args.len()
+ );
+ }
- Ok(format!("{}[{}]", base.schema_name(), name))
+ let base = &args[0];
+ let field_names: Vec<String> = args[1..]
+ .iter()
+ .map(|f| match f {
+ Expr::Literal(name, _) => name.to_string(),
+ other => other.schema_name().to_string(),
+ })
+ .collect();
+
+ Ok(format!(
+ "{}[{}]",
+ base.schema_name(),
+ field_names.join("][")
+ ))
}
fn signature(&self) -> &Signature {
@@ -134,209 +303,243 @@ impl ScalarUDFImpl for GetFieldFunc {
}
fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
- // Length check handled in the signature
- debug_assert_eq!(args.scalar_arguments.len(), 2);
-
- match (
- &args.arg_fields[0].data_type(),
- args.scalar_arguments[1].as_ref(),
- ) {
- (DataType::Map(fields, _), _) => {
- match fields.data_type() {
- DataType::Struct(fields) if fields.len() == 2 => {
- // Arrow's MapArray is essentially a ListArray of structs with two columns. They are
- // often named "key", and "value", but we don't require any specific naming here;
- // instead, we assume that the second column is the "value" column both here and in
- // execution.
- let value_field = fields
- .get(1)
- .expect("fields should have exactly two members");
-
- Ok(value_field.as_ref().clone().with_nullable(true).into())
+ // Validate minimum 2 arguments: base expression + at least one field name
+ if args.scalar_arguments.len() < 2 {
+ return exec_err!(
+ "get_field requires at least 2 arguments, got {}",
+ args.scalar_arguments.len()
+ );
+ }
+
+ let mut current_field = Arc::clone(&args.arg_fields[0]);
+
+ // Iterate through each field name (starting from index 1)
+ for (i, sv) in args.scalar_arguments.iter().enumerate().skip(1) {
+ match current_field.data_type() {
+ DataType::Map(map_field, _) => {
+ match map_field.data_type() {
+ DataType::Struct(fields) if fields.len() == 2 => {
+ // Arrow's MapArray is essentially a ListArray of structs with two columns. They are
+ // often named "key", and "value", but we don't require any specific naming here;
+ // instead, we assume that the second column is the "value" column both here and in
+ // execution.
+ let value_field = fields
+ .get(1)
+ .expect("fields should have exactly two members");
+
+ current_field = Arc::new(
+ value_field.as_ref().clone().with_nullable(true),
+ );
+ }
+ _ => {
+ return exec_err!(
+ "Map fields must contain a Struct with exactly 2 fields"
+ );
+ }
}
- _ => exec_err!(
- "Map fields must contain a Struct with exactly 2 fields"
- ),
+ }
+ DataType::Struct(fields) => {
+ let field_name = sv
+ .as_ref()
+ .and_then(|sv| {
+ sv.try_as_str().flatten().filter(|s| !s.is_empty())
+ })
+ .ok_or_else(|| {
+ datafusion_common::DataFusionError::Execution(
+ "Field name must be a non-empty string".to_string(),
+ )
+ })?;
+
+ let child_field = fields
+ .iter()
+ .find(|f| f.name() == field_name)
+ .ok_or_else(|| {
+ plan_datafusion_err!("Field {field_name} not found in struct")
+ })?;
+
+ let mut new_field = child_field.as_ref().clone();
+
+ // If the parent is nullable, then getting the child must be nullable
+ if current_field.is_nullable() {
+ new_field = new_field.with_nullable(true);
+ }
+ current_field = Arc::new(new_field);
+ }
+ DataType::Null => {
+ return Ok(Field::new(self.name(), DataType::Null, true).into());
+ }
+ other => {
+ return exec_err!(
+ "Cannot access field at argument {}: type {} is not Struct, Map, or Null",
+ i,
+ other
+ );
}
}
- (DataType::Struct(fields), sv) => {
- sv.and_then(|sv| sv.try_as_str().flatten().filter(|s| !s.is_empty()))
- .map_or_else(
- || exec_err!("Field name must be a non-empty string"),
- |field_name| {
- fields
- .iter()
- .find(|f| f.name() == field_name)
- .ok_or(plan_datafusion_err!(
- "Field {field_name} not found in struct"
- ))
- .map(|f| {
- let mut child_field = f.as_ref().clone();
-
- // If the parent is nullable, then getting the child must be nullable,
- // so potentially override the return value
-
- if args.arg_fields[0].is_nullable() {
- child_field = child_field.with_nullable(true);
- }
- Arc::new(child_field)
- })
- },
- )
- }
- (DataType::Null, _) => {
- Ok(Field::new(self.name(), DataType::Null, true).into())
- }
- (other, _) => exec_err!(
- "The expression to get an indexed field is only valid for `Struct`, `Map` or `Null` types, got {other}"
- ),
}
+
+ Ok(current_field)
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- let [base, field_name] = take_function_args(self.name(), args.args)?;
+ if args.args.len() < 2 {
+ return exec_err!(
+ "get_field requires at least 2 arguments, got {}",
+ args.args.len()
+ );
+ }
- if base.data_type().is_null() {
+ let mut current = args.args[0].clone();
+
+ // Early exit for null base
+ if current.data_type().is_null() {
return Ok(ColumnarValue::Scalar(ScalarValue::Null));
}
- let arrays =
- ColumnarValue::values_to_arrays(&[base.clone(), field_name.clone()])?;
- let array = Arc::clone(&arrays[0]);
- let name = match field_name {
- ColumnarValue::Scalar(name) => name,
- _ => {
- return exec_err!(
- "get_field function requires the argument field_name to be a string"
- );
- }
- };
-
- fn process_map_array(
- array: &dyn Array,
- key_array: Arc<dyn Array>,
- ) -> Result<ColumnarValue> {
- let map_array = as_map_array(array)?;
- let keys = if key_array.data_type().is_nested() {
- let comparator = make_comparator(
- map_array.keys().as_ref(),
- key_array.as_ref(),
- SortOptions::default(),
- )?;
- let len = map_array.keys().len().min(key_array.len());
- let values = (0..len).map(|i| comparator(i, i).is_eq()).collect();
- let nulls =
- NullBuffer::union(map_array.keys().nulls(), key_array.nulls());
- BooleanArray::new(values, nulls)
- } else {
- let be_compared = Scalar::new(key_array);
- arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())?
+ // Iterate through each field name
+ for field_name in args.args.iter().skip(1) {
+ let field_name_scalar = match field_name {
+ ColumnarValue::Scalar(name) => name.clone(),
+ _ => {
+ return exec_err!(
+ "get_field function requires all field_name arguments to be scalars"
+ );
+ }
};
- let original_data = map_array.entries().column(1).to_data();
- let capacity = Capacities::Array(original_data.len());
- let mut mutable =
- MutableArrayData::with_capacities(vec![&original_data], true, capacity);
+ current = extract_single_field(current, field_name_scalar)?;
- for entry in 0..map_array.len() {
- let start = map_array.value_offsets()[entry] as usize;
- let end = map_array.value_offsets()[entry + 1] as usize;
+ // Early exit if we hit null
+ if current.data_type().is_null() {
+ return Ok(ColumnarValue::Scalar(ScalarValue::Null));
+ }
+ }
- let maybe_matched = keys
- .slice(start, end - start)
- .iter()
- .enumerate()
- .find(|(_, t)| t.unwrap());
+ Ok(current)
+ }
- if maybe_matched.is_none() {
- mutable.extend_nulls(1);
- continue;
- }
- let (match_offset, _) = maybe_matched.unwrap();
- mutable.extend(0, start + match_offset, start + match_offset + 1);
+ fn simplify(
+ &self,
+ args: Vec<Expr>,
+ _info: &dyn datafusion_expr::simplify::SimplifyInfo,
+ ) -> Result<ExprSimplifyResult> {
+ // Need at least 2 args (base + field)
+ if args.len() < 2 {
+ return Ok(ExprSimplifyResult::Original(args));
+ }
+
+ // Flatten all nested get_field calls in a single pass
+ // Pattern: get_field(get_field(get_field(base, a), b), c) => get_field(base, a, b, c)
+
+ // Collect path arguments from all nested levels
+ let mut path_args_stack = Vec::new();
+ let mut current_expr = &args[0];
+
+ // Push the outermost path arguments first
+ path_args_stack.push(&args[1..]);
+
+ // Walk down the chain of nested get_field calls
+ let base_expr = loop {
+ if let Expr::ScalarFunction(ScalarFunction {
+ func,
+ args: inner_args,
+ }) = current_expr
+ && func
+ .inner()
+ .as_any()
+ .downcast_ref::<GetFieldFunc>()
+ .is_some()
+ {
+ // Store this level's path arguments (all except the first, which is base/nested call)
+ path_args_stack.push(&inner_args[1..]);
+
+ // Move to the next level down
+ current_expr = &inner_args[0];
+ continue;
}
+ // Not a get_field call, this is the base expression
+ break current_expr;
+ };
- let data = mutable.freeze();
- let data = make_array(data);
- Ok(ColumnarValue::Array(data))
+ // If no nested get_field calls were found, return original
+ if path_args_stack.len() == args.len() - 1 {
+ return Ok(ExprSimplifyResult::Original(args));
}
- fn process_map_with_nested_key(
- array: &dyn Array,
- key_array: &dyn Array,
- ) -> Result<ColumnarValue> {
- let map_array = as_map_array(array)?;
-
- let comparator = make_comparator(
- map_array.keys().as_ref(),
- key_array,
- SortOptions::default(),
- )?;
-
- let original_data = map_array.entries().column(1).to_data();
- let capacity = Capacities::Array(original_data.len());
- let mut mutable =
- MutableArrayData::with_capacities(vec![&original_data], true, capacity);
-
- for entry in 0..map_array.len() {
- let start = map_array.value_offsets()[entry] as usize;
- let end = map_array.value_offsets()[entry + 1] as usize;
-
- let mut found_match = false;
- for i in start..end {
- if comparator(i, 0).is_eq() {
- mutable.extend(0, i, i + 1);
- found_match = true;
- break;
- }
- }
-
- if !found_match {
- mutable.extend_nulls(1);
- }
- }
+ // If we found any nested get_field calls, flatten them
+ // Build merged args: [base, ...all_path_args_in_correct_order]
+ let mut merged_args = vec![base_expr.clone()];
- let data = mutable.freeze();
- let data = make_array(data);
- Ok(ColumnarValue::Array(data))
+ // Add path args in reverse order (innermost to outermost)
+ // Stack is: [outermost_paths, ..., innermost_paths]
+ // We want: [base, innermost_paths, ..., outermost_paths]
+ for path_slice in path_args_stack.iter().rev() {
+ merged_args.extend_from_slice(path_slice);
}
- match (array.data_type(), name) {
- (DataType::Map(_, _), ScalarValue::List(arr)) => {
- let key_array: Arc<dyn Array> = arr;
- process_map_array(&array, key_array)
- }
- (DataType::Map(_, _), ScalarValue::Struct(arr)) => {
- process_map_array(&array, arr as Arc<dyn Array>)
- }
- (DataType::Map(_, _), other) => {
- let data_type = other.data_type();
- if data_type.is_nested() {
- process_map_with_nested_key(&array, &other.to_array()?)
- } else {
- process_map_array(&array, other.to_array()?)
- }
- }
- (DataType::Struct(_), ScalarValue::Utf8(Some(k))) => {
- let as_struct_array = as_struct_array(&array)?;
- match as_struct_array.column_by_name(&k) {
- None => exec_err!("get indexed field {k} not found in struct"),
- Some(col) => Ok(ColumnarValue::Array(Arc::clone(col))),
- }
- }
- (DataType::Struct(_), name) => exec_err!(
- "get_field is only possible on struct with utf8 indexes. \
- Received with {name:?} index"
- ),
- (DataType::Null, _) => Ok(ColumnarValue::Scalar(ScalarValue::Null)),
- (dt, name) => exec_err!(
- "get_field is only possible on maps with utf8 indexes or struct \
- with utf8 indexes. Received {dt} with {name:?} index"
+ Ok(ExprSimplifyResult::Simplified(Expr::ScalarFunction(
+ ScalarFunction::new_udf(
+ Arc::new(ScalarUDF::new_from_impl(GetFieldFunc::new())),
+ merged_args,
),
+ )))
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ if arg_types.len() < 2 {
+ return exec_err!(
+ "get_field requires at least 2 arguments, got {}",
+ arg_types.len()
+ );
}
+ // Accept types as-is, validation happens in return_field_from_args
+ Ok(arg_types.to_vec())
}
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::{ArrayRef, Int32Array, StructArray};
+ use arrow::datatypes::Fields;
+
+ #[test]
+ fn test_get_field_utf8view_key() -> Result<()> {
+ // Create a struct array with fields "a" and "b"
+ let a_values = Int32Array::from(vec![Some(1), Some(2), Some(3)]);
+ let b_values = Int32Array::from(vec![Some(10), Some(20), Some(30)]);
+
+ let fields: Fields = vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ ]
+ .into();
+
+ let struct_array = StructArray::new(
+ fields,
+ vec![
+ Arc::new(a_values) as ArrayRef,
+ Arc::new(b_values) as ArrayRef,
+ ],
+ None,
+ );
+
+ let base = ColumnarValue::Array(Arc::new(struct_array));
+
+ // Use Utf8View key to access field "a"
+ let key = ScalarValue::Utf8View(Some("a".to_string()));
+
+ let result = extract_single_field(base, key)?;
+
+ let result_array = result.into_array(3)?;
+ let expected = Int32Array::from(vec![Some(1), Some(2), Some(3)]);
+
+ assert_eq!(result_array.as_ref(), &expected as &dyn Array);
+
+ Ok(())
+ }
+}
diff --git a/datafusion/functions/src/core/mod.rs b/datafusion/functions/src/core/mod.rs
index 82b0097aa7..a14d563737 100644
--- a/datafusion/functions/src/core/mod.rs
+++ b/datafusion/functions/src/core/mod.rs
@@ -121,6 +121,13 @@ pub mod expr_fn {
super::get_field().call(vec![arg1, arg2.lit()])
}
+ #[doc = "Returns the value of nested fields by traversing multiple field names"]
+ pub fn get_field_path(base: Expr, field_names: Vec<Expr>) -> Expr {
+ let mut args = vec![base];
+ args.extend(field_names);
+ super::get_field().call(args)
+ }
+
#[doc = "Returns the value of the field with the given name from the union when it's selected, or NULL otherwise"]
#[expect(clippy::needless_pass_by_value)]
pub fn union_extract(arg1: Expr, arg2: impl Literal) -> Expr {
diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs
index 3391408e04..5746a568e7 100644
--- a/datafusion/sql/src/unparser/expr.rs
+++ b/datafusion/sql/src/unparser/expr.rs
@@ -629,21 +629,27 @@ impl Unparser<'_> {
}
fn get_field_to_sql(&self, args: &[Expr]) -> Result<ast::Expr> {
- assert_eq_or_internal_err!(
- args.len(),
- 2,
- "get_field must have exactly 2 arguments"
- );
+ if args.len() < 2 {
+ return internal_err!(
+ "get_field must have at least 2 arguments, got {}",
+ args.len()
+ );
+ }
- let field = match &args[1] {
- Expr::Literal(lit, _) => self.new_ident_quoted_if_needs(lit.to_string()),
- _ => {
- return internal_err!(
- "get_field expects second argument to be a string, but received: {:?}",
- &args[1]
- );
- }
- };
+ // Extract all field names (args[1..])
+ let mut fields = Vec::with_capacity(args.len() - 1);
+ for arg in &args[1..] {
+ let field = match arg {
+ Expr::Literal(lit, _) => self.new_ident_quoted_if_needs(lit.to_string()),
+ _ => {
+ return internal_err!(
+ "get_field expects field arguments to be strings, but received: {:?}",
+ arg
+ );
+ }
+ };
+ fields.push(field);
+ }
match &args[0] {
Expr::Column(col) => {
@@ -657,17 +663,19 @@ impl Unparser<'_> {
);
}
};
- id.push(field);
+ id.extend(fields);
Ok(ast::Expr::CompoundIdentifier(id))
}
Expr::ScalarFunction(struct_expr) => {
let root = self
.scalar_function_to_sql(struct_expr.func.name(), &struct_expr.args)?;
+ let access_chain = fields
+ .into_iter()
+ .map(|field| ast::AccessExpr::Dot(ast::Expr::Identifier(field)))
+ .collect();
Ok(ast::Expr::CompoundFieldAccess {
root: Box::new(root),
- access_chain: vec![ast::AccessExpr::Dot(ast::Expr::Identifier(
- field,
- ))],
+ access_chain,
})
}
_ => {
diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt
index 0989dd382d..d985af1104 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -653,3 +653,174 @@ test1.struct_field['substruct']['subsubstruct'] = test2.struct_field['substruct'
statement ok
drop table test;
+
+# Test nested get_field with multiple arguments
+query I
+select get_field({'a': {'b': 1}}, 'a', 'b');
+----
+1
+
+# Test nested get_field with three levels
+query I
+select get_field({'a': {'b': {'c': 42}}}, 'a', 'b', 'c');
+----
+42
+
+# Test type validation error - accessing field on non-struct
+query error Cannot access field at argument 2: type Int64 is not Struct, Map, or Null
+select get_field({'a': 1}, 'a', 'b');
+
+# Test that bracket syntax produces a single get_field call (not nested)
+# We use a table column to prevent constant folding
+statement ok
+create table explain_test (s struct(a struct(b int))) as values ({'a': {'b': 1}});
+
+query TT
+explain select s['a']['b'] from explain_test;
+----
+logical_plan
+01)Projection: get_field(explain_test.s, Utf8("a"), Utf8("b"))
+02)--TableScan: explain_test projection=[s]
+physical_plan
+01)ProjectionExec: expr=[get_field(s@0, a, b) as explain_test.s[a][b]]
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+drop table explain_test;
+
+# Test with nested struct in table
+statement ok
+create table nested_test (s struct(inner struct(val int))) as values ({'inner': {'val': 100}});
+
+query I
+select s['inner']['val'] from nested_test;
+----
+100
+
+query I
+select get_field(s, 'inner', 'val') from nested_test;
+----
+100
+
+statement ok
+drop table nested_test;
+
+# Test mixed struct and map access
+query I
+select get_field({'m': map(['k'], [42])}, 'm', 'k');
+----
+42
+
+# Test nested map access
+query I
+select {'m': map(['outer'], [map(['inner'], [99])])}['m']['outer']['inner'];
+----
+99
+
+###############################################
+# Additional tests for nested get_field support
+###############################################
+
+# Backwards compatibility: original 2-argument form still works
+query I
+select get_field({'a': 1}, 'a');
+----
+1
+
+# Error: get_field with no arguments
+query error get_field requires at least 2 arguments, got 0
+select get_field();
+
+# Error: get_field with only 1 argument
+query error get_field requires at least 2 arguments, got 1
+select get_field({'a': 1});
+
+# Null handling: null at base should return null
+query I
+select get_field(CAST(NULL AS STRUCT(a STRUCT(b INT))), 'a', 'b');
+----
+NULL
+
+# Null handling: null in middle of chain
+statement ok
+create table null_mid_test (s STRUCT(a STRUCT(b INT)));
+
+statement ok
+insert into null_mid_test values ({'a': NULL});
+
+query I
+select s['a']['b'] from null_mid_test;
+----
+NULL
+
+query I
+select get_field(s, 'a', 'b') from null_mid_test;
+----
+NULL
+
+statement ok
+drop table null_mid_test;
+
+# Type validation error at argument 3
+query error Cannot access field at argument 3: type Int64 is not Struct, Map, or Null
+select get_field({'a': {'b': 2}}, 'a', 'b', 'c');
+
+# Type validation error at argument 4
+query error Cannot access field at argument 4: type Int64 is not Struct, Map, or Null
+select get_field({'a': {'b': {'c': 3}}}, 'a', 'b', 'c', 'd');
+
+# Non-existent field at first level
+query error Field x not found in struct
+select get_field({'a': 1}, 'x');
+
+# Non-existent field at second level
+query error Field x not found in struct
+select get_field({'a': {'b': 1}}, 'a', 'x');
+
+# Deep nesting: 5-level access
+query I
+select get_field({'l1': {'l2': {'l3': {'l4': {'l5': 42}}}}}, 'l1', 'l2', 'l3', 'l4', 'l5');
+----
+42
+
+# Deep nesting: 5-level access via bracket syntax
+query I
+select {'l1': {'l2': {'l3': {'l4': {'l5': 99}}}}}['l1']['l2']['l3']['l4']['l5'];
+----
+99
+
+# Mixed array and struct access: array index should break the batching
+statement ok
+create table mixed_access_test (data STRUCT(items STRUCT(name VARCHAR)[]) );
+
+statement ok
+insert into mixed_access_test values ({'items': [{'name': 'first'}, {'name': 'second'}]});
+
+query T
+select data['items'][1]['name'] from mixed_access_test;
+----
+first
+
+query T
+select data['items'][2]['name'] from mixed_access_test;
+----
+second
+
+statement ok
+drop table mixed_access_test;
+
+# Nullable parent propagation: null parent should propagate
+statement ok
+create table nullable_parent_test (s STRUCT(a STRUCT(b INT)));
+
+statement ok
+insert into nullable_parent_test values ({'a': {'b': 1}}), (NULL);
+
+query I
+select s['a']['b'] from nullable_parent_test;
+----
+1
+NULL
+
+statement ok
+drop table nullable_parent_test;
diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md
index 3f0939d00d..e454788003 100644
--- a/docs/source/user-guide/sql/scalar_functions.md
+++ b/docs/source/user-guide/sql/scalar_functions.md
@@ -5098,44 +5098,53 @@ arrow_typeof(expression)
### `get_field`
Returns a field within a map or a struct with the given key.
+Supports nested field access by providing multiple field names.
Note: most users invoke `get_field` indirectly via field access
syntax such as `my_struct_col['field_name']` which results in a call to
`get_field(my_struct_col, 'field_name')`.
+Nested access like `my_struct['a']['b']` is optimized to a single call:
+`get_field(my_struct, 'a', 'b')`.
```sql
-get_field(expression1, expression2)
+get_field(expression, field_name[, field_name2, ...])
```
#### Arguments
-- **expression1**: The map or struct to retrieve a field for.
-- **expression2**: The field name in the map or struct to retrieve data for. Must evaluate to a string.
+- **expression**: The map or struct to retrieve a field from.
+- **field_name**: The field name(s) to access, in order for nested access. Must evaluate to strings.
#### Example
```sql
-> create table t (idx varchar, v varchar) as values ('data','fusion'), ('apache', 'arrow');
-> select struct(idx, v) from t as c;
-+-------------------------+
-| struct(c.idx,c.v) |
-+-------------------------+
-| {c0: data, c1: fusion} |
-| {c0: apache, c1: arrow} |
-+-------------------------+
-> select get_field((select struct(idx, v) from t), 'c0');
-+-----------------------+
-| struct(t.idx,t.v)[c0] |
-+-----------------------+
-| data |
-| apache |
-+-----------------------+
-> select get_field((select struct(idx, v) from t), 'c1');
-+-----------------------+
-| struct(t.idx,t.v)[c1] |
-+-----------------------+
-| fusion |
-| arrow |
-+-----------------------+
+> -- Access a field from a struct column
+> create table test( struct_col) as values
+ ({name: 'Alice', age: 30}),
+ ({name: 'Bob', age: 25});
+> select struct_col from test;
++-----------------------------+
+| struct_col |
++-----------------------------+
+| {name: Alice, age: 30} |
+| {name: Bob, age: 25} |
++-----------------------------+
+> select struct_col['name'] as name from test;
++-------+
+| name |
++-------+
+| Alice |
+| Bob |
++-------+
+
+> -- Nested field access with multiple arguments
+> create table test(struct_col) as values
+ ({outer: {inner_val: 42}});
+> select struct_col['outer']['inner_val'] as result from test;
++--------+
+| result |
++--------+
+| 42 |
++--------+
```
### `version`