Skip to content

Apache DataFusion SQL Query Engine

substrait: add an extension point for per-field Arrow metadata on user-defined types (#25015)

Jorge Moraisd6825532c39d
diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs
index d7d7a69581..b0756ef060 100644
--- a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs
@@ -16,7 +16,9 @@
// under the License.
use crate::logical_plan::consumer::SubstraitConsumer;
-use crate::logical_plan::consumer::types::from_substrait_type;
+use crate::logical_plan::consumer::types::{
+ from_substrait_type, substrait_type_to_field,
+};
use crate::logical_plan::consumer::utils::{DEFAULT_TIMEZONE, next_struct_field_name};
use crate::variation_const::FLOAT_16_TYPE_NAME;
#[expect(deprecated)]
@@ -338,19 +340,17 @@ pub(crate) fn from_substrait_literal(
Some(v) => Ok(v),
_ => plan_err!("Missing value type for empty map"),
}?;
- let key_type = from_substrait_type(consumer, key, dfs_names, name_idx)?;
- let value_type = from_substrait_type(consumer, value, dfs_names, name_idx)?;
+ let key_field = substrait_type_to_field(consumer, key, dfs_names, name_idx)?
+ .with_name("key")
+ .with_nullable(false);
+ let value_field =
+ substrait_type_to_field(consumer, value, dfs_names, name_idx)?
+ .with_name("value");
// new_empty_array on a MapType creates a too empty array
// We want it to contain an empty struct array to align with an empty MapBuilder one
- let entries = Field::new_struct(
- "entries",
- vec![
- Field::new("key", key_type, false),
- Field::new("value", value_type, true),
- ],
- false,
- );
+ let entries =
+ Field::new_struct("entries", vec![key_field, value_field], false);
let struct_array =
new_empty_array(entries.data_type()).as_struct().to_owned();
ScalarValue::Map(Arc::new(MapArray::new(
diff --git a/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs b/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs
index bbd80b4cff..36ef39561e 100644
--- a/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/substrait_consumer.rs
@@ -36,7 +36,7 @@ use datafusion::common::{
use datafusion::execution::{FunctionRegistry, SessionState};
use datafusion::logical_expr::expr::LambdaVariable;
use datafusion::logical_expr::{Expr, Extension, LogicalPlan};
-use std::collections::VecDeque;
+use std::collections::{HashMap, VecDeque};
use std::sync::{Arc, RwLock};
use substrait::proto::expression as substrait_expression;
use substrait::proto::expression::{
@@ -500,6 +500,28 @@ pub trait SubstraitConsumer: Send + Sync + Sized {
)
}
+ /// Optional Arrow field metadata for a Substrait type.
+ ///
+ /// Arrow cannot represent some Substrait types natively. A user-defined
+ /// "json" type, for example, becomes a plain `DataType::Utf8`, which loses
+ /// the fact that the values are JSON. A consumer can return metadata here
+ /// to keep the original type recoverable from the Arrow schema.
+ ///
+ /// The consumer sees the whole [`Type`], so it can key the metadata off a
+ /// user-defined type, a type variation, or anything else it recognizes.
+ /// The metadata is attached to the Arrow [`Field`] built for this type,
+ /// including a type nested inside a list, map, or struct.
+ ///
+ /// The default attaches nothing.
+ ///
+ /// [`Field`]: datafusion::arrow::datatypes::Field
+ fn consume_type_metadata(
+ &self,
+ _typ: &Type,
+ ) -> datafusion::common::Result<Option<HashMap<String, String>>> {
+ Ok(None)
+ }
+
fn consume_user_defined_literal(
&self,
user_defined_literal: &proto::expression::literal::UserDefined,
diff --git a/datafusion/substrait/src/logical_plan/consumer/types.rs b/datafusion/substrait/src/logical_plan/consumer/types.rs
index 6ca9ced2fc..e588734782 100644
--- a/datafusion/substrait/src/logical_plan/consumer/types.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/types.rs
@@ -36,7 +36,6 @@ use crate::variation_const::{FLOAT_16_TYPE_NAME, NULL_TYPE_NAME};
use datafusion::arrow::datatypes::{
DataType, Field, FieldRef, Fields, IntervalUnit, Schema, TimeUnit,
};
-use datafusion::common::datatype::DataTypeExt;
use datafusion::common::{
DFSchema, not_impl_err, substrait_datafusion_err, substrait_err,
};
@@ -47,7 +46,8 @@ pub(crate) fn field_from_substrait_type_without_names(
consumer: &impl SubstraitConsumer,
dt: &Type,
) -> datafusion::common::Result<FieldRef> {
- Ok(from_substrait_type_without_names(consumer, dt)?.into_nullable_field_ref())
+ let field = substrait_type_to_field(consumer, dt, &[], &mut 0)?;
+ Ok(Arc::new(field))
}
pub(crate) fn from_substrait_type_without_names(
@@ -64,7 +64,32 @@ pub fn field_from_substrait_type(
name_idx: &mut usize,
) -> datafusion::common::Result<FieldRef> {
// We could add nullability here now that we are returning a Field
- Ok(from_substrait_type(consumer, dt, dfs_names, name_idx)?.into_nullable_field_ref())
+ let field = substrait_type_to_field(consumer, dt, dfs_names, name_idx)?;
+ Ok(Arc::new(field))
+}
+
+/// Converts a Substrait type to an Arrow [`Field`] and attaches the metadata
+/// that [`SubstraitConsumer::consume_type_metadata`] supplies for it.
+///
+/// This is the only place that attaches type metadata. Every Arrow field built
+/// from a Substrait type goes through here, including the fields of a nested
+/// list, map, or struct, so a nested type keeps its metadata without each call
+/// site having to ask for it.
+///
+/// The field is named `""` and is nullable. Callers set the name and the
+/// nullability that they need.
+pub(crate) fn substrait_type_to_field(
+ consumer: &impl SubstraitConsumer,
+ dt: &Type,
+ dfs_names: &[String],
+ name_idx: &mut usize,
+) -> datafusion::common::Result<Field> {
+ let data_type = from_substrait_type(consumer, dt, dfs_names, name_idx)?;
+ let field = Field::new("", data_type, true);
+ match consumer.consume_type_metadata(dt)? {
+ Some(metadata) => Ok(field.with_metadata(metadata)),
+ None => Ok(field),
+ }
}
pub fn from_substrait_type(
@@ -177,12 +202,13 @@ pub fn from_substrait_type(
let inner_type = list.r#type.as_ref().ok_or_else(|| {
substrait_datafusion_err!("List type must have inner type")
})?;
- let field = Arc::new(Field::new_list_field(
- from_substrait_type(consumer, inner_type, dfs_names, name_idx)?,
- // We ignore Substrait's nullability here to match to_substrait_literal
- // which always creates nullable lists
- true,
- ));
+ // `substrait_type_to_field` always returns a nullable field, so
+ // we ignore Substrait's nullability here to match
+ // to_substrait_literal, which always creates nullable lists
+ let field = Arc::new(
+ substrait_type_to_field(consumer, inner_type, dfs_names, name_idx)?
+ .with_name(Field::LIST_FIELD_DEFAULT_NAME),
+ );
match list.type_variation_reference {
DEFAULT_CONTAINER_TYPE_VARIATION_REF => Ok(DataType::List(field)),
LARGE_CONTAINER_TYPE_VARIATION_REF => Ok(DataType::LargeList(field)),
@@ -198,15 +224,16 @@ pub fn from_substrait_type(
let value_type = map.value.as_ref().ok_or_else(|| {
substrait_datafusion_err!("Map type must have value type")
})?;
- let key_type =
- from_substrait_type(consumer, key_type, dfs_names, name_idx)?;
- let value_type =
- from_substrait_type(consumer, value_type, dfs_names, name_idx)?;
+ let key_field =
+ substrait_type_to_field(consumer, key_type, dfs_names, name_idx)?;
+ let value_field =
+ substrait_type_to_field(consumer, value_type, dfs_names, name_idx)?;
match map.type_variation_reference {
DEFAULT_MAP_TYPE_VARIATION_REF => {
- let key_field = Arc::new(Field::new("key", key_type, false));
- let value_field = Arc::new(Field::new("value", value_type, true));
+ let key_field =
+ Arc::new(key_field.with_name("key").with_nullable(false));
+ let value_field = Arc::new(value_field.with_name("value"));
Ok(DataType::Map(
Arc::new(Field::new_struct(
"entries",
@@ -216,9 +243,11 @@ pub fn from_substrait_type(
false, // whether keys are sorted
))
}
+ // A Dictionary holds bare data types, so it cannot carry the
+ // metadata of its key and value types.
DICTIONARY_MAP_TYPE_VARIATION_REF => Ok(DataType::Dictionary(
- Box::new(key_type),
- Box::new(value_type),
+ Box::new(key_field.data_type().clone()),
+ Box::new(value_field.data_type().clone()),
)),
v => not_impl_err!(
"Unsupported Substrait type variation {v} of type {s_kind:?}"
@@ -348,8 +377,9 @@ fn from_substrait_struct_type(
let mut fields = vec![];
for (i, f) in s.types.iter().enumerate() {
let name = next_struct_field_name(i, dfs_names, name_idx)?;
- let data_type = from_substrait_type(consumer, f, dfs_names, name_idx)?;
- let field = Field::new(name, data_type, type_is_nullable(f)?);
+ let field = substrait_type_to_field(consumer, f, dfs_names, name_idx)?
+ .with_name(name)
+ .with_nullable(type_is_nullable(f)?);
fields.push(field);
}
Ok(fields.into())
@@ -412,8 +442,216 @@ fn is_nullable(nullability: i32) -> datafusion::common::Result<bool> {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::extensions::Extensions;
+ use crate::logical_plan::consumer::DefaultSubstraitConsumer;
+ use async_trait::async_trait;
+ use datafusion::catalog::TableProvider;
+ use datafusion::common::TableReference;
+ use datafusion::execution::{FunctionRegistry, SessionState, SessionStateBuilder};
+ use std::collections::HashMap;
+ use std::sync::Arc;
use substrait::proto::r#type::Kind;
+ /// Wraps [DefaultSubstraitConsumer], overriding `consume_user_defined_type`
+ /// (so a `UserDefined` type resolves instead of erroring) and
+ /// `consume_type_metadata`, which returns the fixed metadata below for a
+ /// user-defined type and nothing for every other type.
+ struct MetadataConsumer<'a> {
+ inner: DefaultSubstraitConsumer<'a>,
+ metadata: Option<HashMap<String, String>>,
+ }
+
+ #[async_trait]
+ impl SubstraitConsumer for MetadataConsumer<'_> {
+ async fn resolve_table_ref(
+ &self,
+ table_ref: &TableReference,
+ ) -> datafusion::common::Result<Option<Arc<dyn TableProvider>>> {
+ self.inner.resolve_table_ref(table_ref).await
+ }
+
+ fn get_extensions(&self) -> &Extensions {
+ self.inner.get_extensions()
+ }
+
+ fn get_function_registry(&self) -> &impl FunctionRegistry {
+ self.inner.get_function_registry()
+ }
+
+ fn consume_user_defined_type(
+ &self,
+ _user_defined_type: &r#type::UserDefined,
+ ) -> datafusion::common::Result<DataType> {
+ Ok(DataType::Utf8)
+ }
+
+ fn consume_type_metadata(
+ &self,
+ typ: &Type,
+ ) -> datafusion::common::Result<Option<HashMap<String, String>>> {
+ match typ.kind {
+ Some(Kind::UserDefined(_)) => Ok(self.metadata.clone()),
+ _ => Ok(None),
+ }
+ }
+ }
+
+ fn user_defined_type() -> Type {
+ Type {
+ kind: Some(Kind::UserDefined(r#type::UserDefined {
+ type_reference: 0,
+ type_variation_reference: 0,
+ nullability: r#type::Nullability::Nullable as i32,
+ type_parameters: vec![],
+ })),
+ }
+ }
+
+ fn user_defined_struct_type() -> r#type::Struct {
+ r#type::Struct {
+ types: vec![user_defined_type()],
+ type_variation_reference: 0,
+ nullability: r#type::Nullability::Nullable as i32,
+ }
+ }
+
+ fn test_metadata() -> HashMap<String, String> {
+ HashMap::from([("user-defined-type".to_string(), "json".to_string())])
+ }
+
+ fn consumer_with_metadata(
+ metadata: Option<HashMap<String, String>>,
+ ) -> MetadataConsumer<'static> {
+ let extensions: &'static Extensions = Box::leak(Box::new(Extensions::default()));
+ let session_state: &'static SessionState =
+ Box::leak(Box::new(SessionStateBuilder::new().build()));
+ MetadataConsumer {
+ inner: DefaultSubstraitConsumer::new(extensions, session_state),
+ metadata,
+ }
+ }
+
+ #[test]
+ fn struct_field_gets_consumer_supplied_metadata_for_user_defined_type() {
+ let metadata = test_metadata();
+ let consumer = consumer_with_metadata(Some(metadata.clone()));
+
+ let fields = from_substrait_struct_type(
+ &consumer,
+ &user_defined_struct_type(),
+ &["col".to_string()],
+ &mut 0,
+ )
+ .unwrap();
+
+ assert_eq!(fields.len(), 1);
+ assert_eq!(fields[0].name(), "col");
+ assert_eq!(fields[0].data_type(), &DataType::Utf8);
+ assert_eq!(fields[0].metadata(), &metadata);
+ }
+
+ #[test]
+ fn top_level_field_gets_consumer_supplied_metadata() {
+ let metadata = test_metadata();
+ let consumer = consumer_with_metadata(Some(metadata.clone()));
+
+ let field =
+ field_from_substrait_type_without_names(&consumer, &user_defined_type())
+ .unwrap();
+
+ assert_eq!(field.data_type(), &DataType::Utf8);
+ assert_eq!(field.metadata(), &metadata);
+ }
+
+ #[test]
+ fn named_top_level_field_gets_consumer_supplied_metadata() {
+ // Covers `field_from_substrait_type`, the public entry point that
+ // threads `dfs_names`/`name_idx` through, as opposed to the
+ // `_without_names` variant used by the tests above.
+ let metadata = test_metadata();
+ let consumer = consumer_with_metadata(Some(metadata.clone()));
+
+ let field = field_from_substrait_type(
+ &consumer,
+ &user_defined_type(),
+ &["col".to_string()],
+ &mut 0,
+ )
+ .unwrap();
+
+ assert_eq!(field.data_type(), &DataType::Utf8);
+ assert_eq!(field.metadata(), &metadata);
+ }
+
+ #[test]
+ fn list_element_field_gets_metadata_for_a_nested_user_defined_type() {
+ let metadata = test_metadata();
+ let consumer = consumer_with_metadata(Some(metadata.clone()));
+ let list = Type {
+ kind: Some(Kind::List(Box::new(r#type::List {
+ r#type: Some(Box::new(user_defined_type())),
+ type_variation_reference: DEFAULT_CONTAINER_TYPE_VARIATION_REF,
+ nullability: r#type::Nullability::Nullable as i32,
+ }))),
+ };
+
+ let field = field_from_substrait_type_without_names(&consumer, &list).unwrap();
+
+ let DataType::List(element) = field.data_type() else {
+ panic!("expected a List, got {:?}", field.data_type());
+ };
+ assert_eq!(element.name(), Field::LIST_FIELD_DEFAULT_NAME);
+ assert!(element.is_nullable());
+ assert_eq!(element.metadata(), &metadata);
+ // The list itself is not a user-defined type, so it carries no metadata.
+ assert!(field.metadata().is_empty());
+ }
+
+ #[test]
+ fn map_key_and_value_fields_get_metadata_for_a_nested_user_defined_type() {
+ let metadata = test_metadata();
+ let consumer = consumer_with_metadata(Some(metadata.clone()));
+ let map = Type {
+ kind: Some(Kind::Map(Box::new(r#type::Map {
+ key: Some(Box::new(user_defined_type())),
+ value: Some(Box::new(user_defined_type())),
+ type_variation_reference: DEFAULT_MAP_TYPE_VARIATION_REF,
+ nullability: r#type::Nullability::Nullable as i32,
+ }))),
+ };
+
+ let field = field_from_substrait_type_without_names(&consumer, &map).unwrap();
+
+ let DataType::Map(entries, _) = field.data_type() else {
+ panic!("expected a Map, got {:?}", field.data_type());
+ };
+ let DataType::Struct(entry_fields) = entries.data_type() else {
+ panic!("expected map entries to be a Struct");
+ };
+ assert_eq!(entry_fields[0].name(), "key");
+ assert!(!entry_fields[0].is_nullable());
+ assert_eq!(entry_fields[0].metadata(), &metadata);
+ assert_eq!(entry_fields[1].name(), "value");
+ assert!(entry_fields[1].is_nullable());
+ assert_eq!(entry_fields[1].metadata(), &metadata);
+ }
+
+ #[test]
+ fn struct_field_has_no_metadata_when_consumer_returns_none() {
+ let consumer = consumer_with_metadata(None);
+
+ let fields = from_substrait_struct_type(
+ &consumer,
+ &user_defined_struct_type(),
+ &["col".to_string()],
+ &mut 0,
+ )
+ .unwrap();
+
+ assert_eq!(fields.len(), 1);
+ assert!(fields[0].metadata().is_empty());
+ }
+
#[test]
fn type_is_nullable_user_defined_type_reference_is_required() {
// The deprecated `UserDefinedTypeReference` variant doesn't carry a