Apache DataFusion SQL Query Engine
Feat/map sql extension types (#21714)
cyberbeam52449c67c296b9b
diff --git a/Cargo.lock b/Cargo.lockindex cbf3c06465..b581b3c8d4 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2709,6 +2709,7 @@ name = "datafusion-sql"version = "55.1.0"dependencies = ["arrow",+ "arrow-schema","bigdecimal","chrono","ctor",diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex 3c523f6014..c9575e7a55 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -429,7 +429,6 @@ impl LogicalPlanBuilder {let mut fields = ValuesFields::new();for j in 0..n_cols {let field_type = schema.field(j).data_type();- let field_nullable = schema.field(j).is_nullable();for row in values.iter() {let value = &row[j];let data_type = value.get_type(schema)?;@@ -444,7 +443,13 @@ impl LogicalPlanBuilder {);}}- fields.push(field_type.to_owned(), field_nullable);+ let field = schema.field(j);++ fields.push_with_metadata(+ field_type.clone(),+ field.is_nullable(),+ Some(FieldMetadata::new_from_field(field)),+ );}Self::infer_inner(values, fields, schema)@@ -1835,10 +1840,6 @@ impl ValuesFields {Self::default()}- pub fn push(&mut self, data_type: DataType, nullable: bool) {- self.push_with_metadata(data_type, nullable, None);- }-pub fn push_with_metadata(&mut self,data_type: DataType,diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.tomlindex 305a6d03d5..4b4e74b22e 100644--- a/datafusion/sql/Cargo.toml+++ b/datafusion/sql/Cargo.toml@@ -52,6 +52,7 @@ recursive_protection = ["dep:recursive", "dep:stacker"]# They are used for testing purposes only, so they are in the dev-dependencies section.[dependencies]arrow = { workspace = true }+arrow-schema = { workspace = true }bigdecimal = { workspace = true }chrono = { workspace = true }datafusion-common = { workspace = true, features = ["sql"] }diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rsindex b02d312e62..b51d1ab777 100644--- a/datafusion/sql/src/planner.rs+++ b/datafusion/sql/src/planner.rs@@ -10,6 +10,7 @@ use std::vec;use crate::utils::make_decimal_type;use arrow::datatypes::*;+use arrow_schema::extension::{Json, Uuid};use datafusion_common::TableReference;use datafusion_common::config::SqlParserOptions;use datafusion_common::datatype::{DataTypeExt, FieldExt};@@ -664,6 +665,14 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {// If no type_planner can handle this type, use the default conversionmatch sql_type {+ // Canonical Arrow extension types+ SQLDataType::Uuid => Ok(Arc::new(+ Field::new("", DataType::FixedSizeBinary(16), true)+ .with_extension_type(Uuid),+ )),+ SQLDataType::JSON => Ok(Arc::new(+ Field::new("", DataType::Utf8, true).with_extension_type(Json::default()),+ )),SQLDataType::Array(ArrayElemTypeDef::AngleBracket(inner_sql_type)) => {// Arrays may be multi-dimensional.Ok(self.convert_data_type_to_field(inner_sql_type)?.into_list())diff --git a/datafusion/sql/tests/cases/params.rs b/datafusion/sql/tests/cases/params.rsindex 43710fc7d3..75a19d0de8 100644--- a/datafusion/sql/tests/cases/params.rs+++ b/datafusion/sql/tests/cases/params.rs@@ -822,11 +822,11 @@ fn test_insert_infer_with_metadata() {@r#"** Initial Plan:Dml: op=[Insert Into] table=[person_with_uuid_extension]- Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name+ Projection: column1 AS id, column2 AS first_name, column3 AS last_nameValues: ($1, $2, $3)** Final Plan:Dml: op=[Insert Into] table=[person_with_uuid_extension]- Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name+ Projection: column1 AS id, column2 AS first_name, column3 AS last_nameValues: (FixedSizeBinary(16, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16") FieldMetadata { inner: {"ARROW:extension:name": "arrow.uuid"} } AS $1, Utf8("Alan") AS $2, Utf8("Turing") AS $3)"#);@@ -844,11 +844,11 @@ fn test_insert_infer_with_metadata() {** Initial Plan:Prepare: "my_plan" [FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>, Utf8, Utf8]Dml: op=[Insert Into] table=[person_with_uuid_extension]- Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name+ Projection: column1 AS id, column2 AS first_name, column3 AS last_nameValues: ($1, $2, $3)** Final Plan:Dml: op=[Insert Into] table=[person_with_uuid_extension]- Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name+ Projection: column1 AS id, column2 AS first_name, column3 AS last_nameValues: (FixedSizeBinary(16, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16") FieldMetadata { inner: {"ARROW:extension:name": "arrow.uuid"} } AS $1, Utf8("Alan") AS $2, Utf8("Turing") AS $3)"#);diff --git a/datafusion/sql/tests/common/mod.rs b/datafusion/sql/tests/common/mod.rsindex 5dc4c55fe3..c1707fdcd6 100644--- a/datafusion/sql/tests/common/mod.rs+++ b/datafusion/sql/tests/common/mod.rs@@ -387,11 +387,6 @@ impl TypePlanner for CustomTypePlanner {sql_type: &sqlparser::ast::DataType,) -> Result<Option<FieldRef>> {match sql_type {- sqlparser::ast::DataType::Uuid => Ok(Some(Arc::new(- Field::new("", DataType::FixedSizeBinary(16), true).with_metadata(- Metadata::new().with("ARROW:extension:name", "arrow.uuid"),- ),- ))),sqlparser::ast::DataType::Datetime(precision) => {let precision = match precision {Some(0) => TimeUnit::Second,diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex f6e2c25623..0b5ada362e 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -16,8 +16,7 @@ use arrow::array::{};use arrow::buffer::ScalarBuffer;use arrow::datatypes::{- DataType, Field, FieldRef, Fields, Metadata, Schema, SchemaRef, TimeUnit, UInt32Type,- UnionFields,+ DataType, Field, Fields, Schema, SchemaRef, TimeUnit, UInt32Type, UnionFields,};use arrow::record_batch::RecordBatch;use datafusion::catalog::{@@ -28,7 +27,6 @@ use datafusion::common::stats::Precision;use datafusion::common::{DataFusionError, Result, not_impl_err};use datafusion::functions::math::abs;use datafusion::logical_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};-use datafusion::logical_expr::planner::TypePlanner;use datafusion::logical_expr::{ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,Volatility, create_udf,@@ -58,7 +56,6 @@ use datafusion::physical_plan::operator_statistics::{};use datafusion::physical_plan::statistics::StatisticsArgs;use log::info;-use sqlparser::ast;use tempfile::TempDir;mod custom_sort_pushdown;@@ -72,22 +69,6 @@ pub struct TestContext {test_dir: Option<TempDir>,}-#[derive(Debug)]-struct SqlLogicTestTypePlanner;--impl TypePlanner for SqlLogicTestTypePlanner {- fn plan_type_field(&self, sql_type: &ast::DataType) -> Result<Option<FieldRef>> {- match sql_type {- ast::DataType::Uuid => Ok(Some(Arc::new(- Field::new("", DataType::FixedSizeBinary(16), true).with_metadata(- Metadata::new().with("ARROW:extension:name", "arrow.uuid"),- ),- ))),- _ => Ok(None),- }- }-}-impl TestContext {pub fn new(ctx: SessionContext) -> Self {Self {@@ -133,14 +114,6 @@ impl TestContext {}}- if matches!(- relative_path.file_name().and_then(|name| name.to_str()),- Some("cast_extension_type_metadata.slt")- ) {- state_builder =- state_builder.with_type_planner(Arc::new(SqlLogicTestTypePlanner));- }-if matches!(relative_path.file_name().and_then(|name| name.to_str()),Some("statistics_registry.slt")diff --git a/datafusion/sqllogictest/test_files/sql_extension_types.slt b/datafusion/sqllogictest/test_files/sql_extension_types.sltnew file mode 100644index 0000000000..1d81fee43e--- /dev/null+++ b/datafusion/sqllogictest/test_files/sql_extension_types.slt@@ -0,0 +1,136 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++# Tests for built-in mapping of SQL types to Arrow canonical extension types.+# See: https://arrow.apache.org/docs/format/CanonicalExtensions.html++#######+# UUID Extension Type+#######++# CAST to UUID preserves extension metadata+query ?T+SELECT CAST(X'00010203040506070809000102030506' AS UUID),+ arrow_metadata(CAST(X'00010203040506070809000102030506' AS UUID), 'ARROW:extension:name');+----+00010203040506070809000102030506 arrow.uuid++# Test table creation and various insertion methods (Null, Naked Binary, and Cast)+statement ok+CREATE TABLE test_uuid (value UUID);++query TTT+DESCRIBE test_uuid;+----+value FixedSizeBinary(16) YES++# 1. NULL insertion+statement ok+INSERT INTO test_uuid VALUES (NULL);++# 2. Exercises Implicit Promotion to UUID from binary+statement ok+INSERT INTO test_uuid VALUES (X'00010203040506070809000102030506');++# 3. Explicit CAST+statement ok+INSERT INTO test_uuid VALUES (CAST(X'00010203040506070809000102030507' AS UUID));++query T+SELECT arrow_metadata(value, 'ARROW:extension:name') FROM test_uuid;+----+arrow.uuid+arrow.uuid+arrow.uuid++# Batch of multiple values (some NULL)+statement ok+INSERT INTO test_uuid VALUES+ (CAST(X'00010203040506070809000102030506' AS UUID)),+ (NULL),+ (CAST(X'00010203040506070809000102030507' AS UUID));++query I+SELECT COUNT(*) FROM test_uuid;+----+6++statement ok+DROP TABLE test_uuid;++#######+# JSON Extension Type+#######++# CAST to JSON preserves extension metadata+query TT+SELECT CAST('{"a": 1}' AS JSON),+ arrow_metadata(CAST('{"a": 1}' AS JSON), 'ARROW:extension:name');+----+{"a": 1} arrow.json++statement ok+CREATE TABLE test_json (data JSON);++query TTT+DESCRIBE test_json;+----+data Utf8 YES++# 1. NULL insertion+statement ok+INSERT INTO test_json VALUES (NULL);++# 2. Bare string literal (Implicit Promotion)+statement ok+INSERT INTO test_json VALUES ('{"bare": "literal"}');++# 3. Explicit CAST+statement ok+INSERT INTO test_json VALUES (CAST('{"hello": "world"}' AS JSON));++query T+SELECT arrow_metadata(data, 'ARROW:extension:name') FROM test_json;+----+arrow.json+arrow.json+arrow.json++# Verify Type Decay: Concatenation results in a bare Utf8 string+query T+SELECT arrow_metadata(data || '{"extra": 1}', 'ARROW:extension:name') FROM test_json;+----+NULL+NULL+NULL++statement ok+DROP TABLE test_json;++#######+# Error Handling+#######++# Error: Cannot cast an integer to UUID+statement error Unsupported CAST from Int64 to FixedSizeBinary\(16\)+SELECT CAST(12345 AS UUID);++# Error: Cannot cast a short string to UUID+statement error Unsupported CAST from Utf8 to FixedSizeBinary\(16\)+SELECT CAST('too-short' AS UUID);++# This should trigger cast plan_err: "Types don't match and no valid cast exists"+# Because 'not-a-uuid' is Utf8, and it's being compared to a UUID target in a VALUES list.+statement ok+CREATE TABLE t1 (id UUID);++# Then, try to insert the invalid type+statement error Types don't match and no valid cast exists, received data of type Utf8 for field of type FixedSizeBinary\(16\)+INSERT INTO t1 VALUES ('not-a-uuid');++statement ok+INSERT INTO t1 VALUES (NULL);++statement ok+DROP TABLE t1;