Skip to content

Apache DataFusion SQL Query Engine

feat: add fp16 support to Substrait (#18086)

Weston Pace522403bb4478
diff --git a/Cargo.lock b/Cargo.lock
index d69ece6d3f..7e1d2b332a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2680,6 +2680,7 @@ dependencies = [
"chrono",
"datafusion",
"datafusion-functions-aggregate",
+ "half",
"insta",
"itertools 0.14.0",
"object_store",
diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.toml
index 16bb5cff4a..605dfc15be 100644
--- a/datafusion/substrait/Cargo.toml
+++ b/datafusion/substrait/Cargo.toml
@@ -35,6 +35,7 @@ async-recursion = "1.0"
async-trait = { workspace = true }
chrono = { workspace = true }
datafusion = { workspace = true, features = ["sql"] }
+half = { workspace = true }
itertools = { workspace = true }
object_store = { workspace = true }
pbjson-types = { workspace = true }
diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs
index dc7a5935c0..eb3d345dc0 100644
--- a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs
@@ -18,6 +18,7 @@
use crate::logical_plan::consumer::types::from_substrait_type;
use crate::logical_plan::consumer::utils::{next_struct_field_name, DEFAULT_TIMEZONE};
use crate::logical_plan::consumer::SubstraitConsumer;
+use crate::variation_const::FLOAT_16_TYPE_NAME;
#[allow(deprecated)]
use crate::variation_const::{
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
@@ -38,6 +39,7 @@ use datafusion::common::{
not_impl_err, plan_err, substrait_datafusion_err, substrait_err, ScalarValue,
};
use datafusion::logical_expr::Expr;
+use prost::Message;
use std::sync::Arc;
use substrait::proto;
use substrait::proto::expression::literal::user_defined::Val;
@@ -440,8 +442,6 @@ pub(crate) fn from_substrait_literal(
return Ok(value);
}
- // TODO: remove the code below once the producer has been updated
-
// Helper function to prevent duplicating this code - can be inlined once the non-extension path is removed
let interval_month_day_nano =
|user_defined: &proto::expression::literal::UserDefined| -> datafusion::common::Result<ScalarValue> {
@@ -474,6 +474,36 @@ pub(crate) fn from_substrait_literal(
.get(&user_defined.type_reference)
{
match name.as_ref() {
+ FLOAT_16_TYPE_NAME => {
+ // Rules for encoding fp16 Substrait literals are defined as part of Arrow here:
+ //
+ // https://github.com/apache/arrow/blame/bab558061696ddc1841148d6210424b12923d48e/format/substrait/extension_types.yaml#L112
+
+ let Some(value) = user_defined.val.as_ref() else {
+ return substrait_err!("Float16 value is empty");
+ };
+ let Val::Value(value_any) = value else {
+ return substrait_err!(
+ "Float16 value is not a value type literal"
+ );
+ };
+ if value_any.type_url != "google.protobuf.UInt32Value" {
+ return substrait_err!(
+ "Float16 value is not a google.protobuf.UInt32Value"
+ );
+ }
+ let decoded_value =
+ pbjson_types::UInt32Value::decode(value_any.value.clone())
+ .map_err(|err| {
+ substrait_datafusion_err!(
+ "Failed to decode float16 value: {err}"
+ )
+ })?;
+ let u32_bytes = decoded_value.value.to_le_bytes();
+ let f16_val =
+ half::f16::from_le_bytes(u32_bytes[0..2].try_into().unwrap());
+ return Ok(ScalarValue::Float16(Some(f16_val)));
+ }
// Kept for backwards compatibility - producers should use IntervalCompound instead
#[allow(deprecated)]
INTERVAL_MONTH_DAY_NANO_TYPE_NAME => {
diff --git a/datafusion/substrait/src/logical_plan/consumer/types.rs b/datafusion/substrait/src/logical_plan/consumer/types.rs
index 80300af24a..772ea7177c 100644
--- a/datafusion/substrait/src/logical_plan/consumer/types.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/types.rs
@@ -17,6 +17,7 @@
use super::utils::{from_substrait_precision, next_struct_field_name, DEFAULT_TIMEZONE};
use super::SubstraitConsumer;
+use crate::variation_const::FLOAT_16_TYPE_NAME;
#[allow(deprecated)]
use crate::variation_const::{
DATE_32_TYPE_VARIATION_REF, DATE_64_TYPE_VARIATION_REF,
@@ -251,6 +252,7 @@ pub fn from_substrait_type(
match name.as_ref() {
// Kept for backwards compatibility, producers should use IntervalCompound instead
INTERVAL_MONTH_DAY_NANO_TYPE_NAME => Ok(DataType::Interval(IntervalUnit::MonthDayNano)),
+ FLOAT_16_TYPE_NAME => Ok(DataType::Float16),
_ => not_impl_err!(
"Unsupported Substrait user defined type with ref {} and variation {}",
u.type_reference,
@@ -304,7 +306,7 @@ pub fn from_substrait_named_struct(
})?,
&base_schema.names,
&mut name_idx,
- );
+ )?;
if name_idx != base_schema.names.len() {
return substrait_err!(
"Names list must match exactly to nested schema, but found {} uses for {} names",
@@ -312,7 +314,7 @@ pub fn from_substrait_named_struct(
base_schema.names.len()
);
}
- DFSchema::try_from(Schema::new(fields?))
+ DFSchema::try_from(Schema::new(fields))
}
fn from_substrait_struct_type(
diff --git a/datafusion/substrait/src/logical_plan/producer/expr/cast.rs b/datafusion/substrait/src/logical_plan/producer/expr/cast.rs
index 9741dcdd10..71c2140bac 100644
--- a/datafusion/substrait/src/logical_plan/producer/expr/cast.rs
+++ b/datafusion/substrait/src/logical_plan/producer/expr/cast.rs
@@ -40,7 +40,7 @@ pub fn from_cast(
nullable: true,
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
literal_type: Some(LiteralType::Null(to_substrait_type(
- data_type, true,
+ producer, data_type, true,
)?)),
};
return Ok(Expression {
@@ -51,7 +51,7 @@ pub fn from_cast(
Ok(Expression {
rex_type: Some(RexType::Cast(Box::new(
substrait::proto::expression::Cast {
- r#type: Some(to_substrait_type(data_type, true)?),
+ r#type: Some(to_substrait_type(producer, data_type, true)?),
input: Some(Box::new(producer.handle_expr(expr, schema)?)),
failure_behavior: FailureBehavior::ThrowException.into(),
},
@@ -68,7 +68,7 @@ pub fn from_try_cast(
Ok(Expression {
rex_type: Some(RexType::Cast(Box::new(
substrait::proto::expression::Cast {
- r#type: Some(to_substrait_type(data_type, true)?),
+ r#type: Some(to_substrait_type(producer, data_type, true)?),
input: Some(Box::new(producer.handle_expr(expr, schema)?)),
failure_behavior: FailureBehavior::ReturnNull.into(),
},
@@ -79,7 +79,9 @@ pub fn from_try_cast(
#[cfg(test)]
mod tests {
use super::*;
- use crate::logical_plan::producer::to_substrait_extended_expr;
+ use crate::logical_plan::producer::{
+ to_substrait_extended_expr, DefaultSubstraitProducer,
+ };
use datafusion::arrow::datatypes::{DataType, Field};
use datafusion::common::DFSchema;
use datafusion::execution::SessionStateBuilder;
@@ -92,6 +94,8 @@ mod tests {
let empty_schema = DFSchemaRef::new(DFSchema::empty());
let field = Field::new("out", DataType::Int32, false);
+ let mut producer = DefaultSubstraitProducer::new(&state);
+
let expr = Expr::Literal(ScalarValue::Null, None)
.cast_to(&DataType::Int32, &empty_schema)
.unwrap();
@@ -107,7 +111,7 @@ mod tests {
nullable: true,
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
literal_type: Some(LiteralType::Null(
- to_substrait_type(&DataType::Int32, true).unwrap(),
+ to_substrait_type(&mut producer, &DataType::Int32, true).unwrap(),
)),
};
let expected = Expression {
@@ -131,13 +135,16 @@ mod tests {
typed_null.referred_expr[0].expr_type.as_ref().unwrap()
{
let cast_expr = substrait::proto::expression::Cast {
- r#type: Some(to_substrait_type(&DataType::Int32, true).unwrap()),
+ r#type: Some(
+ to_substrait_type(&mut producer, &DataType::Int32, true).unwrap(),
+ ),
input: Some(Box::new(Expression {
rex_type: Some(RexType::Literal(Literal {
nullable: true,
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
literal_type: Some(LiteralType::Null(
- to_substrait_type(&DataType::Int64, true).unwrap(),
+ to_substrait_type(&mut producer, &DataType::Int64, true)
+ .unwrap(),
)),
})),
})),
diff --git a/datafusion/substrait/src/logical_plan/producer/expr/literal.rs b/datafusion/substrait/src/logical_plan/producer/expr/literal.rs
index 2c66e9f6b0..1bb24168e5 100644
--- a/datafusion/substrait/src/logical_plan/producer/expr/literal.rs
+++ b/datafusion/substrait/src/logical_plan/producer/expr/literal.rs
@@ -18,7 +18,7 @@
use crate::logical_plan::producer::{to_substrait_type, SubstraitProducer};
use crate::variation_const::{
DATE_32_TYPE_VARIATION_REF, DECIMAL_128_TYPE_VARIATION_REF,
- DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
+ DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF, FLOAT_16_TYPE_NAME,
LARGE_CONTAINER_TYPE_VARIATION_REF, TIME_32_TYPE_VARIATION_REF,
TIME_64_TYPE_VARIATION_REF, UNSIGNED_INTEGER_TYPE_VARIATION_REF,
VIEW_CONTAINER_TYPE_VARIATION_REF,
@@ -61,6 +61,7 @@ pub(crate) fn to_substrait_literal(
nullable: true,
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
literal_type: Some(LiteralType::Null(to_substrait_type(
+ producer,
&value.data_type(),
true,
)?)),
@@ -94,6 +95,41 @@ pub(crate) fn to_substrait_literal(
LiteralType::I64(*n as i64),
UNSIGNED_INTEGER_TYPE_VARIATION_REF,
),
+ ScalarValue::Float16(Some(f)) => {
+ // Rules for encoding fp16 Substrait literals are defined as part of Arrow here:
+ //
+ // https://github.com/apache/arrow/blame/bab558061696ddc1841148d6210424b12923d48e/format/substrait/extension_types.yaml#L112
+ //
+ // fp16 literals are encoded as user defined literals with
+ // a google.protobuf.UInt32Value message where the lower 16 bits are
+ // the fp16 value.
+ let type_anchor = producer.register_type(FLOAT_16_TYPE_NAME.to_string());
+
+ // The spec says "lower 16 bits" but neglects to mention the endianness.
+ // Let's just use little-endian for now.
+ //
+ // See https://github.com/apache/arrow/issues/47846
+ let f_bytes = f.to_le_bytes();
+ let value = u32::from_le_bytes([f_bytes[0], f_bytes[1], 0, 0]);
+
+ let value = pbjson_types::UInt32Value { value };
+ let encoded_value = prost::Message::encode_to_vec(&value);
+ (
+ LiteralType::UserDefined(
+ substrait::proto::expression::literal::UserDefined {
+ type_reference: type_anchor,
+ type_parameters: vec![],
+ val: Some(substrait::proto::expression::literal::user_defined::Val::Value(
+ pbjson_types::Any {
+ type_url: "google.protobuf.UInt32Value".to_string(),
+ value: encoded_value.into(),
+ },
+ )),
+ },
+ ),
+ DEFAULT_TYPE_VARIATION_REF,
+ )
+ }
ScalarValue::Float32(Some(f)) => {
(LiteralType::Fp32(*f), DEFAULT_TYPE_VARIATION_REF)
}
@@ -241,7 +277,7 @@ pub(crate) fn to_substrait_literal(
),
ScalarValue::Map(m) => {
let map = if m.is_empty() || m.value(0).is_empty() {
- let mt = to_substrait_type(m.data_type(), m.is_nullable())?;
+ let mt = to_substrait_type(producer, m.data_type(), m.is_nullable())?;
let mt = match mt {
substrait::proto::Type {
kind: Some(r#type::Kind::Map(mt)),
@@ -354,12 +390,13 @@ fn convert_array_to_literal_list<T: OffsetSizeTrait>(
.collect::<datafusion::common::Result<Vec<_>>>()?;
if values.is_empty() {
- let lt = match to_substrait_type(array.data_type(), array.is_nullable())? {
- substrait::proto::Type {
- kind: Some(r#type::Kind::List(lt)),
- } => lt.as_ref().to_owned(),
- _ => unreachable!(),
- };
+ let lt =
+ match to_substrait_type(producer, array.data_type(), array.is_nullable())? {
+ substrait::proto::Type {
+ kind: Some(r#type::Kind::List(lt)),
+ } => lt.as_ref().to_owned(),
+ _ => unreachable!(),
+ };
Ok(LiteralType::EmptyList(lt))
} else {
Ok(LiteralType::List(List { values }))
diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs
index 42e1f962f1..d37694ccea 100644
--- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs
+++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs
@@ -78,7 +78,7 @@ pub fn to_substrait_extended_expr(
})
})
.collect::<datafusion::common::Result<Vec<_>>>()?;
- let substrait_schema = to_substrait_named_struct(schema)?;
+ let substrait_schema = to_substrait_named_struct(&mut producer, schema)?;
let extensions = producer.get_extensions();
Ok(Box::new(ExtendedExpression {
diff --git a/datafusion/substrait/src/logical_plan/producer/plan.rs b/datafusion/substrait/src/logical_plan/producer/plan.rs
index 7d5b775412..28f6acd089 100644
--- a/datafusion/substrait/src/logical_plan/producer/plan.rs
+++ b/datafusion/substrait/src/logical_plan/producer/plan.rs
@@ -36,7 +36,7 @@ pub fn to_substrait_plan(
let plan_rels = vec![PlanRel {
rel_type: Some(plan_rel::RelType::Root(RelRoot {
input: Some(*producer.handle_plan(plan)?),
- names: to_substrait_named_struct(plan.schema())?.names,
+ names: to_substrait_named_struct(&mut producer, plan.schema())?.names,
})),
}];
diff --git a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
index 212874e791..4b2e378210 100644
--- a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
+++ b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs
@@ -48,7 +48,7 @@ pub fn from_table_scan(
});
let table_schema = scan.source.schema().to_dfschema_ref()?;
- let base_schema = to_substrait_named_struct(&table_schema)?;
+ let base_schema = to_substrait_named_struct(producer, &table_schema)?;
let filter_option = if scan.filters.is_empty() {
None
@@ -83,7 +83,10 @@ pub fn from_table_scan(
}))
}
-pub fn from_empty_relation(e: &EmptyRelation) -> datafusion::common::Result<Box<Rel>> {
+pub fn from_empty_relation(
+ producer: &mut impl SubstraitProducer,
+ e: &EmptyRelation,
+) -> datafusion::common::Result<Box<Rel>> {
if e.produce_one_row {
return not_impl_err!("Producing a row from empty relation is unsupported");
}
@@ -91,7 +94,7 @@ pub fn from_empty_relation(e: &EmptyRelation) -> datafusion::common::Result<Box<
Ok(Box::new(Rel {
rel_type: Some(RelType::Read(Box::new(ReadRel {
common: None,
- base_schema: Some(to_substrait_named_struct(&e.schema)?),
+ base_schema: Some(to_substrait_named_struct(producer, &e.schema)?),
filter: None,
best_effort_filter: None,
projection: None,
@@ -135,7 +138,7 @@ pub fn from_values(
Ok(Box::new(Rel {
rel_type: Some(RelType::Read(Box::new(ReadRel {
common: None,
- base_schema: Some(to_substrait_named_struct(&v.schema)?),
+ base_schema: Some(to_substrait_named_struct(producer, &v.schema)?),
filter: None,
best_effort_filter: None,
projection: None,
diff --git a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs
index 56edfac576..db08e0f7bf 100644
--- a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs
+++ b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs
@@ -70,6 +70,10 @@ use substrait::proto::{
/// self.extensions.register_function(signature)
/// }
///
+/// fn register_type(&mut self, type_name: String) -> u32 {
+/// self.extensions.register_type(type_name)
+/// }
+///
/// fn get_extensions(self) -> Extensions {
/// self.extensions
/// }
@@ -114,6 +118,15 @@ pub trait SubstraitProducer: Send + Sync + Sized {
/// there is one. Otherwise, it should generate a new anchor.
fn register_function(&mut self, signature: String) -> u32;
+ /// Within a Substrait plan, user defined types are referenced using type anchors that are stored at
+ /// the top level of the [Plan](substrait::proto::Plan) within
+ /// [ExtensionType](substrait::proto::extensions::simple_extension_declaration::ExtensionType)
+ /// messages.
+ ///
+ /// When given a type name, this method should return the existing anchor for it if
+ /// there is one. Otherwise, it should generate a new anchor.
+ fn register_type(&mut self, name: String) -> u32;
+
/// Consume the producer to generate the [Extensions] for the Substrait plan based on the
/// functions that have been registered
fn get_extensions(self) -> Extensions;
@@ -182,7 +195,7 @@ pub trait SubstraitProducer: Send + Sync + Sized {
&mut self,
plan: &EmptyRelation,
) -> datafusion::common::Result<Box<Rel>> {
- from_empty_relation(plan)
+ from_empty_relation(self, plan)
}
fn handle_subquery_alias(
@@ -367,6 +380,10 @@ impl SubstraitProducer for DefaultSubstraitProducer<'_> {
self.extensions.register_function(fn_name)
}
+ fn register_type(&mut self, type_name: String) -> u32 {
+ self.extensions.register_type(type_name)
+ }
+
fn get_extensions(self) -> Extensions {
self.extensions
}
diff --git a/datafusion/substrait/src/logical_plan/producer/types.rs b/datafusion/substrait/src/logical_plan/producer/types.rs
index 3da9269c5b..2079d7fd34 100644
--- a/datafusion/substrait/src/logical_plan/producer/types.rs
+++ b/datafusion/substrait/src/logical_plan/producer/types.rs
@@ -15,15 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-use crate::logical_plan::producer::to_substrait_precision;
use crate::logical_plan::producer::utils::flatten_names;
+use crate::logical_plan::producer::{to_substrait_precision, SubstraitProducer};
use crate::variation_const::{
DATE_32_TYPE_VARIATION_REF, DATE_64_TYPE_VARIATION_REF,
DECIMAL_128_TYPE_VARIATION_REF, DECIMAL_256_TYPE_VARIATION_REF,
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_INTERVAL_DAY_TYPE_VARIATION_REF,
DEFAULT_MAP_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
DICTIONARY_MAP_TYPE_VARIATION_REF, DURATION_INTERVAL_DAY_TYPE_VARIATION_REF,
- LARGE_CONTAINER_TYPE_VARIATION_REF, TIME_32_TYPE_VARIATION_REF,
+ FLOAT_16_TYPE_NAME, LARGE_CONTAINER_TYPE_VARIATION_REF, TIME_32_TYPE_VARIATION_REF,
TIME_64_TYPE_VARIATION_REF, UNSIGNED_INTEGER_TYPE_VARIATION_REF,
VIEW_CONTAINER_TYPE_VARIATION_REF,
};
@@ -32,6 +32,7 @@ use datafusion::common::{internal_err, not_impl_err, plan_err, DFSchemaRef};
use substrait::proto::{r#type, NamedStruct};
pub(crate) fn to_substrait_type(
+ producer: &mut impl SubstraitProducer,
dt: &DataType,
nullable: bool,
) -> datafusion::common::Result<substrait::proto::Type> {
@@ -96,7 +97,17 @@ pub(crate) fn to_substrait_type(
nullability,
})),
}),
- // Float16 is not supported in Substrait
+ DataType::Float16 => {
+ let type_anchor = producer.register_type(FLOAT_16_TYPE_NAME.to_string());
+ Ok(substrait::proto::Type {
+ kind: Some(r#type::Kind::UserDefined(r#type::UserDefined {
+ type_reference: type_anchor,
+ type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
+ nullability,
+ type_parameters: vec![],
+ })),
+ })
+ }
DataType::Float32 => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Fp32(r#type::Fp32 {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
@@ -244,7 +255,8 @@ pub(crate) fn to_substrait_type(
})),
}),
DataType::List(inner) => {
- let inner_type = to_substrait_type(inner.data_type(), inner.is_nullable())?;
+ let inner_type =
+ to_substrait_type(producer, inner.data_type(), inner.is_nullable())?;
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::List(Box::new(r#type::List {
r#type: Some(Box::new(inner_type)),
@@ -254,7 +266,8 @@ pub(crate) fn to_substrait_type(
})
}
DataType::LargeList(inner) => {
- let inner_type = to_substrait_type(inner.data_type(), inner.is_nullable())?;
+ let inner_type =
+ to_substrait_type(producer, inner.data_type(), inner.is_nullable())?;
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::List(Box::new(r#type::List {
r#type: Some(Box::new(inner_type)),
@@ -266,10 +279,12 @@ pub(crate) fn to_substrait_type(
DataType::Map(inner, _) => match inner.data_type() {
DataType::Struct(key_and_value) if key_and_value.len() == 2 => {
let key_type = to_substrait_type(
+ producer,
key_and_value[0].data_type(),
key_and_value[0].is_nullable(),
)?;
let value_type = to_substrait_type(
+ producer,
key_and_value[1].data_type(),
key_and_value[1].is_nullable(),
)?;
@@ -285,8 +300,8 @@ pub(crate) fn to_substrait_type(
_ => plan_err!("Map fields must contain a Struct with exactly 2 fields"),
},
DataType::Dictionary(key_type, value_type) => {
- let key_type = to_substrait_type(key_type, nullable)?;
- let value_type = to_substrait_type(value_type, nullable)?;
+ let key_type = to_substrait_type(producer, key_type, nullable)?;
+ let value_type = to_substrait_type(producer, value_type, nullable)?;
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Map(Box::new(r#type::Map {
key: Some(Box::new(key_type)),
@@ -299,7 +314,9 @@ pub(crate) fn to_substrait_type(
DataType::Struct(fields) => {
let field_types = fields
.iter()
- .map(|field| to_substrait_type(field.data_type(), field.is_nullable()))
+ .map(|field| {
+ to_substrait_type(producer, field.data_type(), field.is_nullable())
+ })
.collect::<datafusion::common::Result<Vec<_>>>()?;
Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Struct(r#type::Struct {
@@ -330,6 +347,7 @@ pub(crate) fn to_substrait_type(
}
pub(crate) fn to_substrait_named_struct(
+ producer: &mut impl SubstraitProducer,
schema: &DFSchemaRef,
) -> datafusion::common::Result<NamedStruct> {
let mut names = Vec::with_capacity(schema.fields().len());
@@ -341,7 +359,7 @@ pub(crate) fn to_substrait_named_struct(
types: schema
.fields()
.iter()
- .map(|f| to_substrait_type(f.data_type(), f.is_nullable()))
+ .map(|f| to_substrait_type(producer, f.data_type(), f.is_nullable()))
.collect::<datafusion::common::Result<_>>()?,
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
nullability: r#type::Nullability::Required as i32,
@@ -360,8 +378,10 @@ mod tests {
use crate::logical_plan::consumer::{
from_substrait_named_struct, from_substrait_type_without_names,
};
+ use crate::logical_plan::producer::DefaultSubstraitProducer;
use datafusion::arrow::datatypes::{Field, Fields, Schema, TimeUnit};
use datafusion::common::{DFSchema, Result};
+ use datafusion::prelude::SessionContext;
use std::sync::Arc;
#[test]
@@ -448,9 +468,12 @@ mod tests {
fn round_trip_type(dt: DataType) -> Result<()> {
println!("Checking round trip of {dt}");
+ let state = SessionContext::default().state();
+ let mut producer = DefaultSubstraitProducer::new(&state);
+
// As DataFusion doesn't consider nullability as a property of the type, but field,
// it doesn't matter if we set nullability to true or false here.
- let substrait = to_substrait_type(&dt, true)?;
+ let substrait = to_substrait_type(&mut producer, &dt, true)?;
let consumer = test_consumer();
let roundtrip_dt = from_substrait_type_without_names(&consumer, &substrait)?;
assert_eq!(dt, roundtrip_dt);
@@ -473,7 +496,10 @@ mod tests {
Field::new("trailer", DataType::Float64, true),
]))?);
- let named_struct = to_substrait_named_struct(&schema)?;
+ let state = SessionContext::default().state();
+ let mut producer = DefaultSubstraitProducer::new(&state);
+
+ let named_struct = to_substrait_named_struct(&mut producer, &schema)?;
// Struct field names should be flattened DFS style
// List field names should be omitted
diff --git a/datafusion/substrait/src/variation_const.rs b/datafusion/substrait/src/variation_const.rs
index f78b3d7853..591f33aeb4 100644
--- a/datafusion/substrait/src/variation_const.rs
+++ b/datafusion/substrait/src/variation_const.rs
@@ -122,3 +122,6 @@ pub const INTERVAL_MONTH_DAY_NANO_TYPE_REF: u32 = 3;
note = "Use Substrait `IntervalCompound` type instead"
)]
pub const INTERVAL_MONTH_DAY_NANO_TYPE_NAME: &str = "interval-month-day-nano";
+
+/// Defined in <https://github.com/apache/arrow/blame/main/format/substrait/extension_types.yaml>
+pub const FLOAT_16_TYPE_NAME: &str = "fp16";
diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
index 39e4984ab9..f14d4cbf1f 100644
--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs
@@ -1144,6 +1144,7 @@ async fn all_type_literal() -> Result<()> {
uint32_col = arrow_cast('0', 'UInt32') AND
int64_col = arrow_cast('0', 'Int64') AND
uint64_col = arrow_cast('0', 'UInt64') AND
+ float16_col = arrow_cast(0.0, 'Float16') AND
float32_col = arrow_cast('0', 'Float32') AND
float64_col = arrow_cast('0', 'Float64') AND
sec_timestamp_col = arrow_cast('2020-01-01 00:00:00', 'Timestamp (Second, None)') AND
@@ -1856,6 +1857,7 @@ async fn create_all_type_context() -> Result<SessionContext> {
Field::new("uint32_col", DataType::UInt32, true),
Field::new("int64_col", DataType::Int64, true),
Field::new("uint64_col", DataType::UInt64, true),
+ Field::new("float16_col", DataType::Float16, true),
Field::new("float32_col", DataType::Float32, true),
Field::new("float64_col", DataType::Float64, true),
Field::new(