Apache DataFusion SQL Query Engine
feat: add fp16 support to Substrait (#18086)
Weston Pace522403bb4478
diff --git a/Cargo.lock b/Cargo.lockindex d69ece6d3f..7e1d2b332a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2680,6 +2680,7 @@ dependencies = ["chrono","datafusion","datafusion-functions-aggregate",+ "half","insta","itertools 0.14.0","object_store",diff --git a/datafusion/substrait/Cargo.toml b/datafusion/substrait/Cargo.tomlindex 16bb5cff4a..605dfc15be 100644--- a/datafusion/substrait/Cargo.toml+++ b/datafusion/substrait/Cargo.toml@@ -35,6 +35,7 @@ async-recursion = "1.0"async-trait = { workspace = true }chrono = { workspace = true }datafusion = { workspace = true, features = ["sql"] }+half = { workspace = true }itertools = { workspace = true }object_store = { workspace = true }pbjson-types = { workspace = true }diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rsindex dc7a5935c0..eb3d345dc0 100644--- a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs+++ b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs@@ -18,6 +18,7 @@use crate::logical_plan::consumer::types::from_substrait_type;use crate::logical_plan::consumer::utils::{next_struct_field_name, DEFAULT_TIMEZONE};use crate::logical_plan::consumer::SubstraitConsumer;+use crate::variation_const::FLOAT_16_TYPE_NAME;#[allow(deprecated)]use crate::variation_const::{DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,@@ -38,6 +39,7 @@ use datafusion::common::{not_impl_err, plan_err, substrait_datafusion_err, substrait_err, ScalarValue,};use datafusion::logical_expr::Expr;+use prost::Message;use std::sync::Arc;use substrait::proto;use substrait::proto::expression::literal::user_defined::Val;@@ -440,8 +442,6 @@ pub(crate) fn from_substrait_literal(return Ok(value);}- // TODO: remove the code below once the producer has been updated-// Helper function to prevent duplicating this code - can be inlined once the non-extension path is removedlet interval_month_day_nano =|user_defined: &proto::expression::literal::UserDefined| -> datafusion::common::Result<ScalarValue> {@@ -474,6 +474,36 @@ pub(crate) fn from_substrait_literal(.get(&user_defined.type_reference){match name.as_ref() {+ FLOAT_16_TYPE_NAME => {+ // Rules for encoding fp16 Substrait literals are defined as part of Arrow here:+ //+ // https://github.com/apache/arrow/blame/bab558061696ddc1841148d6210424b12923d48e/format/substrait/extension_types.yaml#L112++ let Some(value) = user_defined.val.as_ref() else {+ return substrait_err!("Float16 value is empty");+ };+ let Val::Value(value_any) = value else {+ return substrait_err!(+ "Float16 value is not a value type literal"+ );+ };+ if value_any.type_url != "google.protobuf.UInt32Value" {+ return substrait_err!(+ "Float16 value is not a google.protobuf.UInt32Value"+ );+ }+ let decoded_value =+ pbjson_types::UInt32Value::decode(value_any.value.clone())+ .map_err(|err| {+ substrait_datafusion_err!(+ "Failed to decode float16 value: {err}"+ )+ })?;+ let u32_bytes = decoded_value.value.to_le_bytes();+ let f16_val =+ half::f16::from_le_bytes(u32_bytes[0..2].try_into().unwrap());+ return Ok(ScalarValue::Float16(Some(f16_val)));+ }// Kept for backwards compatibility - producers should use IntervalCompound instead#[allow(deprecated)]INTERVAL_MONTH_DAY_NANO_TYPE_NAME => {diff --git a/datafusion/substrait/src/logical_plan/consumer/types.rs b/datafusion/substrait/src/logical_plan/consumer/types.rsindex 80300af24a..772ea7177c 100644--- a/datafusion/substrait/src/logical_plan/consumer/types.rs+++ b/datafusion/substrait/src/logical_plan/consumer/types.rs@@ -17,6 +17,7 @@use super::utils::{from_substrait_precision, next_struct_field_name, DEFAULT_TIMEZONE};use super::SubstraitConsumer;+use crate::variation_const::FLOAT_16_TYPE_NAME;#[allow(deprecated)]use crate::variation_const::{DATE_32_TYPE_VARIATION_REF, DATE_64_TYPE_VARIATION_REF,@@ -251,6 +252,7 @@ pub fn from_substrait_type(match name.as_ref() {// Kept for backwards compatibility, producers should use IntervalCompound insteadINTERVAL_MONTH_DAY_NANO_TYPE_NAME => Ok(DataType::Interval(IntervalUnit::MonthDayNano)),+ FLOAT_16_TYPE_NAME => Ok(DataType::Float16),_ => not_impl_err!("Unsupported Substrait user defined type with ref {} and variation {}",u.type_reference,@@ -304,7 +306,7 @@ pub fn from_substrait_named_struct(})?,&base_schema.names,&mut name_idx,- );+ )?;if name_idx != base_schema.names.len() {return substrait_err!("Names list must match exactly to nested schema, but found {} uses for {} names",@@ -312,7 +314,7 @@ pub fn from_substrait_named_struct(base_schema.names.len());}- DFSchema::try_from(Schema::new(fields?))+ DFSchema::try_from(Schema::new(fields))}fn from_substrait_struct_type(diff --git a/datafusion/substrait/src/logical_plan/producer/expr/cast.rs b/datafusion/substrait/src/logical_plan/producer/expr/cast.rsindex 9741dcdd10..71c2140bac 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/cast.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/cast.rs@@ -40,7 +40,7 @@ pub fn from_cast(nullable: true,type_variation_reference: DEFAULT_TYPE_VARIATION_REF,literal_type: Some(LiteralType::Null(to_substrait_type(- data_type, true,+ producer, data_type, true,)?)),};return Ok(Expression {@@ -51,7 +51,7 @@ pub fn from_cast(Ok(Expression {rex_type: Some(RexType::Cast(Box::new(substrait::proto::expression::Cast {- r#type: Some(to_substrait_type(data_type, true)?),+ r#type: Some(to_substrait_type(producer, data_type, true)?),input: Some(Box::new(producer.handle_expr(expr, schema)?)),failure_behavior: FailureBehavior::ThrowException.into(),},@@ -68,7 +68,7 @@ pub fn from_try_cast(Ok(Expression {rex_type: Some(RexType::Cast(Box::new(substrait::proto::expression::Cast {- r#type: Some(to_substrait_type(data_type, true)?),+ r#type: Some(to_substrait_type(producer, data_type, true)?),input: Some(Box::new(producer.handle_expr(expr, schema)?)),failure_behavior: FailureBehavior::ReturnNull.into(),},@@ -79,7 +79,9 @@ pub fn from_try_cast(#[cfg(test)]mod tests {use super::*;- use crate::logical_plan::producer::to_substrait_extended_expr;+ use crate::logical_plan::producer::{+ to_substrait_extended_expr, DefaultSubstraitProducer,+ };use datafusion::arrow::datatypes::{DataType, Field};use datafusion::common::DFSchema;use datafusion::execution::SessionStateBuilder;@@ -92,6 +94,8 @@ mod tests {let empty_schema = DFSchemaRef::new(DFSchema::empty());let field = Field::new("out", DataType::Int32, false);+ let mut producer = DefaultSubstraitProducer::new(&state);+let expr = Expr::Literal(ScalarValue::Null, None).cast_to(&DataType::Int32, &empty_schema).unwrap();@@ -107,7 +111,7 @@ mod tests {nullable: true,type_variation_reference: DEFAULT_TYPE_VARIATION_REF,literal_type: Some(LiteralType::Null(- to_substrait_type(&DataType::Int32, true).unwrap(),+ to_substrait_type(&mut producer, &DataType::Int32, true).unwrap(),)),};let expected = Expression {@@ -131,13 +135,16 @@ mod tests {typed_null.referred_expr[0].expr_type.as_ref().unwrap(){let cast_expr = substrait::proto::expression::Cast {- r#type: Some(to_substrait_type(&DataType::Int32, true).unwrap()),+ r#type: Some(+ to_substrait_type(&mut producer, &DataType::Int32, true).unwrap(),+ ),input: Some(Box::new(Expression {rex_type: Some(RexType::Literal(Literal {nullable: true,type_variation_reference: DEFAULT_TYPE_VARIATION_REF,literal_type: Some(LiteralType::Null(- to_substrait_type(&DataType::Int64, true).unwrap(),+ to_substrait_type(&mut producer, &DataType::Int64, true)+ .unwrap(),)),})),})),diff --git a/datafusion/substrait/src/logical_plan/producer/expr/literal.rs b/datafusion/substrait/src/logical_plan/producer/expr/literal.rsindex 2c66e9f6b0..1bb24168e5 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/literal.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/literal.rs@@ -18,7 +18,7 @@use crate::logical_plan::producer::{to_substrait_type, SubstraitProducer};use crate::variation_const::{DATE_32_TYPE_VARIATION_REF, DECIMAL_128_TYPE_VARIATION_REF,- DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,+ DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF, FLOAT_16_TYPE_NAME,LARGE_CONTAINER_TYPE_VARIATION_REF, TIME_32_TYPE_VARIATION_REF,TIME_64_TYPE_VARIATION_REF, UNSIGNED_INTEGER_TYPE_VARIATION_REF,VIEW_CONTAINER_TYPE_VARIATION_REF,@@ -61,6 +61,7 @@ pub(crate) fn to_substrait_literal(nullable: true,type_variation_reference: DEFAULT_TYPE_VARIATION_REF,literal_type: Some(LiteralType::Null(to_substrait_type(+ producer,&value.data_type(),true,)?)),@@ -94,6 +95,41 @@ pub(crate) fn to_substrait_literal(LiteralType::I64(*n as i64),UNSIGNED_INTEGER_TYPE_VARIATION_REF,),+ ScalarValue::Float16(Some(f)) => {+ // Rules for encoding fp16 Substrait literals are defined as part of Arrow here:+ //+ // https://github.com/apache/arrow/blame/bab558061696ddc1841148d6210424b12923d48e/format/substrait/extension_types.yaml#L112+ //+ // fp16 literals are encoded as user defined literals with+ // a google.protobuf.UInt32Value message where the lower 16 bits are+ // the fp16 value.+ let type_anchor = producer.register_type(FLOAT_16_TYPE_NAME.to_string());++ // The spec says "lower 16 bits" but neglects to mention the endianness.+ // Let's just use little-endian for now.+ //+ // See https://github.com/apache/arrow/issues/47846+ let f_bytes = f.to_le_bytes();+ let value = u32::from_le_bytes([f_bytes[0], f_bytes[1], 0, 0]);++ let value = pbjson_types::UInt32Value { value };+ let encoded_value = prost::Message::encode_to_vec(&value);+ (+ LiteralType::UserDefined(+ substrait::proto::expression::literal::UserDefined {+ type_reference: type_anchor,+ type_parameters: vec![],+ val: Some(substrait::proto::expression::literal::user_defined::Val::Value(+ pbjson_types::Any {+ type_url: "google.protobuf.UInt32Value".to_string(),+ value: encoded_value.into(),+ },+ )),+ },+ ),+ DEFAULT_TYPE_VARIATION_REF,+ )+ }ScalarValue::Float32(Some(f)) => {(LiteralType::Fp32(*f), DEFAULT_TYPE_VARIATION_REF)}@@ -241,7 +277,7 @@ pub(crate) fn to_substrait_literal(),ScalarValue::Map(m) => {let map = if m.is_empty() || m.value(0).is_empty() {- let mt = to_substrait_type(m.data_type(), m.is_nullable())?;+ let mt = to_substrait_type(producer, m.data_type(), m.is_nullable())?;let mt = match mt {substrait::proto::Type {kind: Some(r#type::Kind::Map(mt)),@@ -354,12 +390,13 @@ fn convert_array_to_literal_list<T: OffsetSizeTrait>(.collect::<datafusion::common::Result<Vec<_>>>()?;if values.is_empty() {- let lt = match to_substrait_type(array.data_type(), array.is_nullable())? {- substrait::proto::Type {- kind: Some(r#type::Kind::List(lt)),- } => lt.as_ref().to_owned(),- _ => unreachable!(),- };+ let lt =+ match to_substrait_type(producer, array.data_type(), array.is_nullable())? {+ substrait::proto::Type {+ kind: Some(r#type::Kind::List(lt)),+ } => lt.as_ref().to_owned(),+ _ => unreachable!(),+ };Ok(LiteralType::EmptyList(lt))} else {Ok(LiteralType::List(List { values }))diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rsindex 42e1f962f1..d37694ccea 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs@@ -78,7 +78,7 @@ pub fn to_substrait_extended_expr(})}).collect::<datafusion::common::Result<Vec<_>>>()?;- let substrait_schema = to_substrait_named_struct(schema)?;+ let substrait_schema = to_substrait_named_struct(&mut producer, schema)?;let extensions = producer.get_extensions();Ok(Box::new(ExtendedExpression {diff --git a/datafusion/substrait/src/logical_plan/producer/plan.rs b/datafusion/substrait/src/logical_plan/producer/plan.rsindex 7d5b775412..28f6acd089 100644--- a/datafusion/substrait/src/logical_plan/producer/plan.rs+++ b/datafusion/substrait/src/logical_plan/producer/plan.rs@@ -36,7 +36,7 @@ pub fn to_substrait_plan(let plan_rels = vec![PlanRel {rel_type: Some(plan_rel::RelType::Root(RelRoot {input: Some(*producer.handle_plan(plan)?),- names: to_substrait_named_struct(plan.schema())?.names,+ names: to_substrait_named_struct(&mut producer, plan.schema())?.names,})),}];diff --git a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rsindex 212874e791..4b2e378210 100644--- a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs+++ b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs@@ -48,7 +48,7 @@ pub fn from_table_scan(});let table_schema = scan.source.schema().to_dfschema_ref()?;- let base_schema = to_substrait_named_struct(&table_schema)?;+ let base_schema = to_substrait_named_struct(producer, &table_schema)?;let filter_option = if scan.filters.is_empty() {None@@ -83,7 +83,10 @@ pub fn from_table_scan(}))}-pub fn from_empty_relation(e: &EmptyRelation) -> datafusion::common::Result<Box<Rel>> {+pub fn from_empty_relation(+ producer: &mut impl SubstraitProducer,+ e: &EmptyRelation,+) -> datafusion::common::Result<Box<Rel>> {if e.produce_one_row {return not_impl_err!("Producing a row from empty relation is unsupported");}@@ -91,7 +94,7 @@ pub fn from_empty_relation(e: &EmptyRelation) -> datafusion::common::Result<Box<Ok(Box::new(Rel {rel_type: Some(RelType::Read(Box::new(ReadRel {common: None,- base_schema: Some(to_substrait_named_struct(&e.schema)?),+ base_schema: Some(to_substrait_named_struct(producer, &e.schema)?),filter: None,best_effort_filter: None,projection: None,@@ -135,7 +138,7 @@ pub fn from_values(Ok(Box::new(Rel {rel_type: Some(RelType::Read(Box::new(ReadRel {common: None,- base_schema: Some(to_substrait_named_struct(&v.schema)?),+ base_schema: Some(to_substrait_named_struct(producer, &v.schema)?),filter: None,best_effort_filter: None,projection: None,diff --git a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rsindex 56edfac576..db08e0f7bf 100644--- a/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs+++ b/datafusion/substrait/src/logical_plan/producer/substrait_producer.rs@@ -70,6 +70,10 @@ use substrait::proto::{/// self.extensions.register_function(signature)/// }///+/// fn register_type(&mut self, type_name: String) -> u32 {+/// self.extensions.register_type(type_name)+/// }+////// fn get_extensions(self) -> Extensions {/// self.extensions/// }@@ -114,6 +118,15 @@ pub trait SubstraitProducer: Send + Sync + Sized {/// there is one. Otherwise, it should generate a new anchor.fn register_function(&mut self, signature: String) -> u32;+ /// Within a Substrait plan, user defined types are referenced using type anchors that are stored at+ /// the top level of the [Plan](substrait::proto::Plan) within+ /// [ExtensionType](substrait::proto::extensions::simple_extension_declaration::ExtensionType)+ /// messages.+ ///+ /// When given a type name, this method should return the existing anchor for it if+ /// there is one. Otherwise, it should generate a new anchor.+ fn register_type(&mut self, name: String) -> u32;+/// Consume the producer to generate the [Extensions] for the Substrait plan based on the/// functions that have been registeredfn get_extensions(self) -> Extensions;@@ -182,7 +195,7 @@ pub trait SubstraitProducer: Send + Sync + Sized {&mut self,plan: &EmptyRelation,) -> datafusion::common::Result<Box<Rel>> {- from_empty_relation(plan)+ from_empty_relation(self, plan)}fn handle_subquery_alias(@@ -367,6 +380,10 @@ impl SubstraitProducer for DefaultSubstraitProducer<'_> {self.extensions.register_function(fn_name)}+ fn register_type(&mut self, type_name: String) -> u32 {+ self.extensions.register_type(type_name)+ }+fn get_extensions(self) -> Extensions {self.extensions}diff --git a/datafusion/substrait/src/logical_plan/producer/types.rs b/datafusion/substrait/src/logical_plan/producer/types.rsindex 3da9269c5b..2079d7fd34 100644--- a/datafusion/substrait/src/logical_plan/producer/types.rs+++ b/datafusion/substrait/src/logical_plan/producer/types.rs@@ -15,15 +15,15 @@// specific language governing permissions and limitations// under the License.-use crate::logical_plan::producer::to_substrait_precision;use crate::logical_plan::producer::utils::flatten_names;+use crate::logical_plan::producer::{to_substrait_precision, SubstraitProducer};use crate::variation_const::{DATE_32_TYPE_VARIATION_REF, DATE_64_TYPE_VARIATION_REF,DECIMAL_128_TYPE_VARIATION_REF, DECIMAL_256_TYPE_VARIATION_REF,DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_INTERVAL_DAY_TYPE_VARIATION_REF,DEFAULT_MAP_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,DICTIONARY_MAP_TYPE_VARIATION_REF, DURATION_INTERVAL_DAY_TYPE_VARIATION_REF,- LARGE_CONTAINER_TYPE_VARIATION_REF, TIME_32_TYPE_VARIATION_REF,+ FLOAT_16_TYPE_NAME, LARGE_CONTAINER_TYPE_VARIATION_REF, TIME_32_TYPE_VARIATION_REF,TIME_64_TYPE_VARIATION_REF, UNSIGNED_INTEGER_TYPE_VARIATION_REF,VIEW_CONTAINER_TYPE_VARIATION_REF,};@@ -32,6 +32,7 @@ use datafusion::common::{internal_err, not_impl_err, plan_err, DFSchemaRef};use substrait::proto::{r#type, NamedStruct};pub(crate) fn to_substrait_type(+ producer: &mut impl SubstraitProducer,dt: &DataType,nullable: bool,) -> datafusion::common::Result<substrait::proto::Type> {@@ -96,7 +97,17 @@ pub(crate) fn to_substrait_type(nullability,})),}),- // Float16 is not supported in Substrait+ DataType::Float16 => {+ let type_anchor = producer.register_type(FLOAT_16_TYPE_NAME.to_string());+ Ok(substrait::proto::Type {+ kind: Some(r#type::Kind::UserDefined(r#type::UserDefined {+ type_reference: type_anchor,+ type_variation_reference: DEFAULT_TYPE_VARIATION_REF,+ nullability,+ type_parameters: vec![],+ })),+ })+ }DataType::Float32 => Ok(substrait::proto::Type {kind: Some(r#type::Kind::Fp32(r#type::Fp32 {type_variation_reference: DEFAULT_TYPE_VARIATION_REF,@@ -244,7 +255,8 @@ pub(crate) fn to_substrait_type(})),}),DataType::List(inner) => {- let inner_type = to_substrait_type(inner.data_type(), inner.is_nullable())?;+ let inner_type =+ to_substrait_type(producer, inner.data_type(), inner.is_nullable())?;Ok(substrait::proto::Type {kind: Some(r#type::Kind::List(Box::new(r#type::List {r#type: Some(Box::new(inner_type)),@@ -254,7 +266,8 @@ pub(crate) fn to_substrait_type(})}DataType::LargeList(inner) => {- let inner_type = to_substrait_type(inner.data_type(), inner.is_nullable())?;+ let inner_type =+ to_substrait_type(producer, inner.data_type(), inner.is_nullable())?;Ok(substrait::proto::Type {kind: Some(r#type::Kind::List(Box::new(r#type::List {r#type: Some(Box::new(inner_type)),@@ -266,10 +279,12 @@ pub(crate) fn to_substrait_type(DataType::Map(inner, _) => match inner.data_type() {DataType::Struct(key_and_value) if key_and_value.len() == 2 => {let key_type = to_substrait_type(+ producer,key_and_value[0].data_type(),key_and_value[0].is_nullable(),)?;let value_type = to_substrait_type(+ producer,key_and_value[1].data_type(),key_and_value[1].is_nullable(),)?;@@ -285,8 +300,8 @@ pub(crate) fn to_substrait_type(_ => plan_err!("Map fields must contain a Struct with exactly 2 fields"),},DataType::Dictionary(key_type, value_type) => {- let key_type = to_substrait_type(key_type, nullable)?;- let value_type = to_substrait_type(value_type, nullable)?;+ let key_type = to_substrait_type(producer, key_type, nullable)?;+ let value_type = to_substrait_type(producer, value_type, nullable)?;Ok(substrait::proto::Type {kind: Some(r#type::Kind::Map(Box::new(r#type::Map {key: Some(Box::new(key_type)),@@ -299,7 +314,9 @@ pub(crate) fn to_substrait_type(DataType::Struct(fields) => {let field_types = fields.iter()- .map(|field| to_substrait_type(field.data_type(), field.is_nullable()))+ .map(|field| {+ to_substrait_type(producer, field.data_type(), field.is_nullable())+ }).collect::<datafusion::common::Result<Vec<_>>>()?;Ok(substrait::proto::Type {kind: Some(r#type::Kind::Struct(r#type::Struct {@@ -330,6 +347,7 @@ pub(crate) fn to_substrait_type(}pub(crate) fn to_substrait_named_struct(+ producer: &mut impl SubstraitProducer,schema: &DFSchemaRef,) -> datafusion::common::Result<NamedStruct> {let mut names = Vec::with_capacity(schema.fields().len());@@ -341,7 +359,7 @@ pub(crate) fn to_substrait_named_struct(types: schema.fields().iter()- .map(|f| to_substrait_type(f.data_type(), f.is_nullable()))+ .map(|f| to_substrait_type(producer, f.data_type(), f.is_nullable())).collect::<datafusion::common::Result<_>>()?,type_variation_reference: DEFAULT_TYPE_VARIATION_REF,nullability: r#type::Nullability::Required as i32,@@ -360,8 +378,10 @@ mod tests {use crate::logical_plan::consumer::{from_substrait_named_struct, from_substrait_type_without_names,};+ use crate::logical_plan::producer::DefaultSubstraitProducer;use datafusion::arrow::datatypes::{Field, Fields, Schema, TimeUnit};use datafusion::common::{DFSchema, Result};+ use datafusion::prelude::SessionContext;use std::sync::Arc;#[test]@@ -448,9 +468,12 @@ mod tests {fn round_trip_type(dt: DataType) -> Result<()> {println!("Checking round trip of {dt}");+ let state = SessionContext::default().state();+ let mut producer = DefaultSubstraitProducer::new(&state);+// As DataFusion doesn't consider nullability as a property of the type, but field,// it doesn't matter if we set nullability to true or false here.- let substrait = to_substrait_type(&dt, true)?;+ let substrait = to_substrait_type(&mut producer, &dt, true)?;let consumer = test_consumer();let roundtrip_dt = from_substrait_type_without_names(&consumer, &substrait)?;assert_eq!(dt, roundtrip_dt);@@ -473,7 +496,10 @@ mod tests {Field::new("trailer", DataType::Float64, true),]))?);- let named_struct = to_substrait_named_struct(&schema)?;+ let state = SessionContext::default().state();+ let mut producer = DefaultSubstraitProducer::new(&state);++ let named_struct = to_substrait_named_struct(&mut producer, &schema)?;// Struct field names should be flattened DFS style// List field names should be omitteddiff --git a/datafusion/substrait/src/variation_const.rs b/datafusion/substrait/src/variation_const.rsindex f78b3d7853..591f33aeb4 100644--- a/datafusion/substrait/src/variation_const.rs+++ b/datafusion/substrait/src/variation_const.rs@@ -122,3 +122,6 @@ pub const INTERVAL_MONTH_DAY_NANO_TYPE_REF: u32 = 3;note = "Use Substrait `IntervalCompound` type instead")]pub const INTERVAL_MONTH_DAY_NANO_TYPE_NAME: &str = "interval-month-day-nano";++/// Defined in <https://github.com/apache/arrow/blame/main/format/substrait/extension_types.yaml>+pub const FLOAT_16_TYPE_NAME: &str = "fp16";diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rsindex 39e4984ab9..f14d4cbf1f 100644--- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs@@ -1144,6 +1144,7 @@ async fn all_type_literal() -> Result<()> {uint32_col = arrow_cast('0', 'UInt32') ANDint64_col = arrow_cast('0', 'Int64') ANDuint64_col = arrow_cast('0', 'UInt64') AND+ float16_col = arrow_cast(0.0, 'Float16') ANDfloat32_col = arrow_cast('0', 'Float32') ANDfloat64_col = arrow_cast('0', 'Float64') ANDsec_timestamp_col = arrow_cast('2020-01-01 00:00:00', 'Timestamp (Second, None)') AND@@ -1856,6 +1857,7 @@ async fn create_all_type_context() -> Result<SessionContext> {Field::new("uint32_col", DataType::UInt32, true),Field::new("int64_col", DataType::Int64, true),Field::new("uint64_col", DataType::UInt64, true),+ Field::new("float16_col", DataType::Float16, true),Field::new("float32_col", DataType::Float32, true),Field::new("float64_col", DataType::Float64, true),Field::new(