Skip to content

Apache DataFusion SQL Query Engine

Feat/map sql extension types (#21714)

cyberbeam52449c67c296b9b
diff --git a/Cargo.lock b/Cargo.lock
index cbf3c06465..b581b3c8d4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2709,6 +2709,7 @@ name = "datafusion-sql"
version = "55.1.0"
dependencies = [
"arrow",
+ "arrow-schema",
"bigdecimal",
"chrono",
"ctor",
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
index 3c523f6014..c9575e7a55 100644
--- a/datafusion/expr/src/logical_plan/builder.rs
+++ b/datafusion/expr/src/logical_plan/builder.rs
@@ -429,7 +429,6 @@ impl LogicalPlanBuilder {
let mut fields = ValuesFields::new();
for j in 0..n_cols {
let field_type = schema.field(j).data_type();
- let field_nullable = schema.field(j).is_nullable();
for row in values.iter() {
let value = &row[j];
let data_type = value.get_type(schema)?;
@@ -444,7 +443,13 @@ impl LogicalPlanBuilder {
);
}
}
- fields.push(field_type.to_owned(), field_nullable);
+ let field = schema.field(j);
+
+ fields.push_with_metadata(
+ field_type.clone(),
+ field.is_nullable(),
+ Some(FieldMetadata::new_from_field(field)),
+ );
}
Self::infer_inner(values, fields, schema)
@@ -1835,10 +1840,6 @@ impl ValuesFields {
Self::default()
}
- pub fn push(&mut self, data_type: DataType, nullable: bool) {
- self.push_with_metadata(data_type, nullable, None);
- }
-
pub fn push_with_metadata(
&mut self,
data_type: DataType,
diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml
index 305a6d03d5..4b4e74b22e 100644
--- a/datafusion/sql/Cargo.toml
+++ b/datafusion/sql/Cargo.toml
@@ -52,6 +52,7 @@ recursive_protection = ["dep:recursive", "dep:stacker"]
# They are used for testing purposes only, so they are in the dev-dependencies section.
[dependencies]
arrow = { workspace = true }
+arrow-schema = { workspace = true }
bigdecimal = { workspace = true }
chrono = { workspace = true }
datafusion-common = { workspace = true, features = ["sql"] }
diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs
index b02d312e62..b51d1ab777 100644
--- a/datafusion/sql/src/planner.rs
+++ b/datafusion/sql/src/planner.rs
@@ -10,6 +10,7 @@ use std::vec;
use crate::utils::make_decimal_type;
use arrow::datatypes::*;
+use arrow_schema::extension::{Json, Uuid};
use datafusion_common::TableReference;
use datafusion_common::config::SqlParserOptions;
use datafusion_common::datatype::{DataTypeExt, FieldExt};
@@ -664,6 +665,14 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
// If no type_planner can handle this type, use the default conversion
match sql_type {
+ // Canonical Arrow extension types
+ SQLDataType::Uuid => Ok(Arc::new(
+ Field::new("", DataType::FixedSizeBinary(16), true)
+ .with_extension_type(Uuid),
+ )),
+ SQLDataType::JSON => Ok(Arc::new(
+ Field::new("", DataType::Utf8, true).with_extension_type(Json::default()),
+ )),
SQLDataType::Array(ArrayElemTypeDef::AngleBracket(inner_sql_type)) => {
// Arrays may be multi-dimensional.
Ok(self.convert_data_type_to_field(inner_sql_type)?.into_list())
diff --git a/datafusion/sql/tests/cases/params.rs b/datafusion/sql/tests/cases/params.rs
index 43710fc7d3..75a19d0de8 100644
--- a/datafusion/sql/tests/cases/params.rs
+++ b/datafusion/sql/tests/cases/params.rs
@@ -822,11 +822,11 @@ fn test_insert_infer_with_metadata() {
@r#"
** Initial Plan:
Dml: op=[Insert Into] table=[person_with_uuid_extension]
- Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
+ Projection: column1 AS id, column2 AS first_name, column3 AS last_name
Values: ($1, $2, $3)
** Final Plan:
Dml: op=[Insert Into] table=[person_with_uuid_extension]
- Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
+ Projection: column1 AS id, column2 AS first_name, column3 AS last_name
Values: (FixedSizeBinary(16, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16") FieldMetadata { inner: {"ARROW:extension:name": "arrow.uuid"} } AS $1, Utf8("Alan") AS $2, Utf8("Turing") AS $3)
"#
);
@@ -844,11 +844,11 @@ fn test_insert_infer_with_metadata() {
** Initial Plan:
Prepare: "my_plan" [FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>, Utf8, Utf8]
Dml: op=[Insert Into] table=[person_with_uuid_extension]
- Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
+ Projection: column1 AS id, column2 AS first_name, column3 AS last_name
Values: ($1, $2, $3)
** Final Plan:
Dml: op=[Insert Into] table=[person_with_uuid_extension]
- Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
+ Projection: column1 AS id, column2 AS first_name, column3 AS last_name
Values: (FixedSizeBinary(16, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16") FieldMetadata { inner: {"ARROW:extension:name": "arrow.uuid"} } AS $1, Utf8("Alan") AS $2, Utf8("Turing") AS $3)
"#
);
diff --git a/datafusion/sql/tests/common/mod.rs b/datafusion/sql/tests/common/mod.rs
index 5dc4c55fe3..c1707fdcd6 100644
--- a/datafusion/sql/tests/common/mod.rs
+++ b/datafusion/sql/tests/common/mod.rs
@@ -387,11 +387,6 @@ impl TypePlanner for CustomTypePlanner {
sql_type: &sqlparser::ast::DataType,
) -> Result<Option<FieldRef>> {
match sql_type {
- sqlparser::ast::DataType::Uuid => Ok(Some(Arc::new(
- Field::new("", DataType::FixedSizeBinary(16), true).with_metadata(
- Metadata::new().with("ARROW:extension:name", "arrow.uuid"),
- ),
- ))),
sqlparser::ast::DataType::Datetime(precision) => {
let precision = match precision {
Some(0) => TimeUnit::Second,
diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs
index f6e2c25623..0b5ada362e 100644
--- a/datafusion/sqllogictest/src/test_context.rs
+++ b/datafusion/sqllogictest/src/test_context.rs
@@ -16,8 +16,7 @@ use arrow::array::{
};
use arrow::buffer::ScalarBuffer;
use arrow::datatypes::{
- DataType, Field, FieldRef, Fields, Metadata, Schema, SchemaRef, TimeUnit, UInt32Type,
- UnionFields,
+ DataType, Field, Fields, Schema, SchemaRef, TimeUnit, UInt32Type, UnionFields,
};
use arrow::record_batch::RecordBatch;
use datafusion::catalog::{
@@ -28,7 +27,6 @@ use datafusion::common::stats::Precision;
use datafusion::common::{DataFusionError, Result, not_impl_err};
use datafusion::functions::math::abs;
use datafusion::logical_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
-use datafusion::logical_expr::planner::TypePlanner;
use datafusion::logical_expr::{
ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
Volatility, create_udf,
@@ -58,7 +56,6 @@ use datafusion::physical_plan::operator_statistics::{
};
use datafusion::physical_plan::statistics::StatisticsArgs;
use log::info;
-use sqlparser::ast;
use tempfile::TempDir;
mod custom_sort_pushdown;
@@ -72,22 +69,6 @@ pub struct TestContext {
test_dir: Option<TempDir>,
}
-#[derive(Debug)]
-struct SqlLogicTestTypePlanner;
-
-impl TypePlanner for SqlLogicTestTypePlanner {
- fn plan_type_field(&self, sql_type: &ast::DataType) -> Result<Option<FieldRef>> {
- match sql_type {
- ast::DataType::Uuid => Ok(Some(Arc::new(
- Field::new("", DataType::FixedSizeBinary(16), true).with_metadata(
- Metadata::new().with("ARROW:extension:name", "arrow.uuid"),
- ),
- ))),
- _ => Ok(None),
- }
- }
-}
-
impl TestContext {
pub fn new(ctx: SessionContext) -> Self {
Self {
@@ -133,14 +114,6 @@ impl TestContext {
}
}
- if matches!(
- relative_path.file_name().and_then(|name| name.to_str()),
- Some("cast_extension_type_metadata.slt")
- ) {
- state_builder =
- state_builder.with_type_planner(Arc::new(SqlLogicTestTypePlanner));
- }
-
if matches!(
relative_path.file_name().and_then(|name| name.to_str()),
Some("statistics_registry.slt")
diff --git a/datafusion/sqllogictest/test_files/sql_extension_types.slt b/datafusion/sqllogictest/test_files/sql_extension_types.slt
new file mode 100644
index 0000000000..1d81fee43e
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/sql_extension_types.slt
@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+
+# Tests for built-in mapping of SQL types to Arrow canonical extension types.
+# See: https://arrow.apache.org/docs/format/CanonicalExtensions.html
+
+#######
+# UUID Extension Type
+#######
+
+# CAST to UUID preserves extension metadata
+query ?T
+SELECT CAST(X'00010203040506070809000102030506' AS UUID),
+ arrow_metadata(CAST(X'00010203040506070809000102030506' AS UUID), 'ARROW:extension:name');
+----
+00010203040506070809000102030506 arrow.uuid
+
+# Test table creation and various insertion methods (Null, Naked Binary, and Cast)
+statement ok
+CREATE TABLE test_uuid (value UUID);
+
+query TTT
+DESCRIBE test_uuid;
+----
+value FixedSizeBinary(16) YES
+
+# 1. NULL insertion
+statement ok
+INSERT INTO test_uuid VALUES (NULL);
+
+# 2. Exercises Implicit Promotion to UUID from binary
+statement ok
+INSERT INTO test_uuid VALUES (X'00010203040506070809000102030506');
+
+# 3. Explicit CAST
+statement ok
+INSERT INTO test_uuid VALUES (CAST(X'00010203040506070809000102030507' AS UUID));
+
+query T
+SELECT arrow_metadata(value, 'ARROW:extension:name') FROM test_uuid;
+----
+arrow.uuid
+arrow.uuid
+arrow.uuid
+
+# Batch of multiple values (some NULL)
+statement ok
+INSERT INTO test_uuid VALUES
+ (CAST(X'00010203040506070809000102030506' AS UUID)),
+ (NULL),
+ (CAST(X'00010203040506070809000102030507' AS UUID));
+
+query I
+SELECT COUNT(*) FROM test_uuid;
+----
+6
+
+statement ok
+DROP TABLE test_uuid;
+
+#######
+# JSON Extension Type
+#######
+
+# CAST to JSON preserves extension metadata
+query TT
+SELECT CAST('{"a": 1}' AS JSON),
+ arrow_metadata(CAST('{"a": 1}' AS JSON), 'ARROW:extension:name');
+----
+{"a": 1} arrow.json
+
+statement ok
+CREATE TABLE test_json (data JSON);
+
+query TTT
+DESCRIBE test_json;
+----
+data Utf8 YES
+
+# 1. NULL insertion
+statement ok
+INSERT INTO test_json VALUES (NULL);
+
+# 2. Bare string literal (Implicit Promotion)
+statement ok
+INSERT INTO test_json VALUES ('{"bare": "literal"}');
+
+# 3. Explicit CAST
+statement ok
+INSERT INTO test_json VALUES (CAST('{"hello": "world"}' AS JSON));
+
+query T
+SELECT arrow_metadata(data, 'ARROW:extension:name') FROM test_json;
+----
+arrow.json
+arrow.json
+arrow.json
+
+# Verify Type Decay: Concatenation results in a bare Utf8 string
+query T
+SELECT arrow_metadata(data || '{"extra": 1}', 'ARROW:extension:name') FROM test_json;
+----
+NULL
+NULL
+NULL
+
+statement ok
+DROP TABLE test_json;
+
+#######
+# Error Handling
+#######
+
+# Error: Cannot cast an integer to UUID
+statement error Unsupported CAST from Int64 to FixedSizeBinary\(16\)
+SELECT CAST(12345 AS UUID);
+
+# Error: Cannot cast a short string to UUID
+statement error Unsupported CAST from Utf8 to FixedSizeBinary\(16\)
+SELECT CAST('too-short' AS UUID);
+
+# This should trigger cast plan_err: "Types don't match and no valid cast exists"
+# Because 'not-a-uuid' is Utf8, and it's being compared to a UUID target in a VALUES list.
+statement ok
+CREATE TABLE t1 (id UUID);
+
+# Then, try to insert the invalid type
+statement error Types don't match and no valid cast exists, received data of type Utf8 for field of type FixedSizeBinary\(16\)
+INSERT INTO t1 VALUES ('not-a-uuid');
+
+statement ok
+INSERT INTO t1 VALUES (NULL);
+
+statement ok
+DROP TABLE t1;