Skip to content

Apache DataFusion SQL Query Engine

fix: null cast not valid in substrait round trip (#18414)

Gene Bordegaray720d4f15e31c
diff --git a/datafusion/substrait/src/logical_plan/consumer/types.rs b/datafusion/substrait/src/logical_plan/consumer/types.rs
index 772ea7177c..ef1000a1cc 100644
--- a/datafusion/substrait/src/logical_plan/consumer/types.rs
+++ b/datafusion/substrait/src/logical_plan/consumer/types.rs
@@ -17,7 +17,6 @@
use super::utils::{from_substrait_precision, next_struct_field_name, DEFAULT_TIMEZONE};
use super::SubstraitConsumer;
-use crate::variation_const::FLOAT_16_TYPE_NAME;
#[allow(deprecated)]
use crate::variation_const::{
DATE_32_TYPE_VARIATION_REF, DATE_64_TYPE_VARIATION_REF,
@@ -33,6 +32,7 @@ use crate::variation_const::{
TIME_64_TYPE_VARIATION_REF, UNSIGNED_INTEGER_TYPE_VARIATION_REF,
VIEW_CONTAINER_TYPE_VARIATION_REF,
};
+use crate::variation_const::{FLOAT_16_TYPE_NAME, NULL_TYPE_NAME};
use datafusion::arrow::datatypes::{
DataType, Field, Fields, IntervalUnit, Schema, TimeUnit,
};
@@ -253,6 +253,7 @@ pub fn from_substrait_type(
// Kept for backwards compatibility, producers should use IntervalCompound instead
INTERVAL_MONTH_DAY_NANO_TYPE_NAME => Ok(DataType::Interval(IntervalUnit::MonthDayNano)),
FLOAT_16_TYPE_NAME => Ok(DataType::Float16),
+ NULL_TYPE_NAME => Ok(DataType::Null),
_ => not_impl_err!(
"Unsupported Substrait user defined type with ref {} and variation {}",
u.type_reference,
diff --git a/datafusion/substrait/src/logical_plan/producer/types.rs b/datafusion/substrait/src/logical_plan/producer/types.rs
index 2079d7fd34..0613ed07be 100644
--- a/datafusion/substrait/src/logical_plan/producer/types.rs
+++ b/datafusion/substrait/src/logical_plan/producer/types.rs
@@ -23,12 +23,12 @@ use crate::variation_const::{
DEFAULT_CONTAINER_TYPE_VARIATION_REF, DEFAULT_INTERVAL_DAY_TYPE_VARIATION_REF,
DEFAULT_MAP_TYPE_VARIATION_REF, DEFAULT_TYPE_VARIATION_REF,
DICTIONARY_MAP_TYPE_VARIATION_REF, DURATION_INTERVAL_DAY_TYPE_VARIATION_REF,
- FLOAT_16_TYPE_NAME, LARGE_CONTAINER_TYPE_VARIATION_REF, TIME_32_TYPE_VARIATION_REF,
- TIME_64_TYPE_VARIATION_REF, UNSIGNED_INTEGER_TYPE_VARIATION_REF,
- VIEW_CONTAINER_TYPE_VARIATION_REF,
+ FLOAT_16_TYPE_NAME, LARGE_CONTAINER_TYPE_VARIATION_REF, NULL_TYPE_NAME,
+ TIME_32_TYPE_VARIATION_REF, TIME_64_TYPE_VARIATION_REF,
+ UNSIGNED_INTEGER_TYPE_VARIATION_REF, VIEW_CONTAINER_TYPE_VARIATION_REF,
};
use datafusion::arrow::datatypes::{DataType, IntervalUnit};
-use datafusion::common::{internal_err, not_impl_err, plan_err, DFSchemaRef};
+use datafusion::common::{not_impl_err, plan_err, DFSchemaRef};
use substrait::proto::{r#type, NamedStruct};
pub(crate) fn to_substrait_type(
@@ -42,7 +42,17 @@ pub(crate) fn to_substrait_type(
r#type::Nullability::Required as i32
};
match dt {
- DataType::Null => internal_err!("Null cast is not valid"),
+ DataType::Null => {
+ let type_anchor = producer.register_type(NULL_TYPE_NAME.to_string());
+ Ok(substrait::proto::Type {
+ kind: Some(r#type::Kind::UserDefined(r#type::UserDefined {
+ type_reference: type_anchor,
+ type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
+ nullability,
+ type_parameters: vec![],
+ })),
+ })
+ }
DataType::Boolean => Ok(substrait::proto::Type {
kind: Some(r#type::Kind::Bool(r#type::Boolean {
type_variation_reference: DEFAULT_TYPE_VARIATION_REF,
@@ -377,6 +387,7 @@ mod tests {
use crate::logical_plan::consumer::tests::test_consumer;
use crate::logical_plan::consumer::{
from_substrait_named_struct, from_substrait_type_without_names,
+ DefaultSubstraitConsumer,
};
use crate::logical_plan::producer::DefaultSubstraitProducer;
use datafusion::arrow::datatypes::{Field, Fields, Schema, TimeUnit};
@@ -386,6 +397,7 @@ mod tests {
#[test]
fn round_trip_types() -> Result<()> {
+ round_trip_type(DataType::Null)?;
round_trip_type(DataType::Boolean)?;
round_trip_type(DataType::Int8)?;
round_trip_type(DataType::UInt8)?;
@@ -474,7 +486,12 @@ mod tests {
// As DataFusion doesn't consider nullability as a property of the type, but field,
// it doesn't matter if we set nullability to true or false here.
let substrait = to_substrait_type(&mut producer, &dt, true)?;
- let consumer = test_consumer();
+
+ // Get the extensions from the producer so the consumer can look up
+ // any registered user-defined types (like "null" or "f16")
+ let extensions = producer.get_extensions();
+ let consumer = DefaultSubstraitConsumer::new(&extensions, &state);
+
let roundtrip_dt = from_substrait_type_without_names(&consumer, &substrait)?;
assert_eq!(dt, roundtrip_dt);
Ok(())
diff --git a/datafusion/substrait/src/variation_const.rs b/datafusion/substrait/src/variation_const.rs
index 591f33aeb4..b1a922899e 100644
--- a/datafusion/substrait/src/variation_const.rs
+++ b/datafusion/substrait/src/variation_const.rs
@@ -125,3 +125,8 @@ pub const INTERVAL_MONTH_DAY_NANO_TYPE_NAME: &str = "interval-month-day-nano";
/// Defined in <https://github.com/apache/arrow/blame/main/format/substrait/extension_types.yaml>
pub const FLOAT_16_TYPE_NAME: &str = "fp16";
+
+/// For [`DataType::Null`]
+///
+/// [`DataType::Null`]: datafusion::arrow::datatypes::DataType::Null
+pub const NULL_TYPE_NAME: &str = "null";