Apache DataFusion SQL Query Engine
Support UDTFs in information_schema.routines / SHOW FUNCTIONS (#23438)
Qi Zhu7ca6e543ec30
diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rsindex 5f65823b9c..ca5060896f 100644--- a/datafusion/catalog/src/information_schema.rs+++ b/datafusion/catalog/src/information_schema.rs@@ -20,6 +20,7 @@//! [Information Schema]: https://en.wikipedia.org/wiki/Information_schemause crate::streaming::StreamingTable;+use crate::table::TableFunction;use crate::{CatalogProviderList, SchemaProvider, TableProvider};use arrow::array::builder::{BooleanBuilder, UInt8Builder};use arrow::{@@ -81,14 +82,28 @@ impl InformationSchemaProvider {/// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list`pub fn new(catalog_list: Arc<dyn CatalogProviderList>) -> Self {Self {- config: InformationSchemaConfig { catalog_list },+ config: InformationSchemaConfig {+ catalog_list,+ table_functions: HashMap::new(),+ },}}++ /// Attach the session's table (UDTF) functions so that they appear in+ /// `information_schema.routines` / `SHOW FUNCTIONS`.+ pub fn with_table_functions(+ mut self,+ table_functions: HashMap<String, Arc<TableFunction>>,+ ) -> Self {+ self.config.table_functions = table_functions;+ self+ }}#[derive(Clone, Debug)]struct InformationSchemaConfig {catalog_list: Arc<dyn CatalogProviderList>,+ table_functions: HashMap<String, Arc<TableFunction>>,}impl InformationSchemaConfig {@@ -301,6 +316,26 @@ impl InformationSchemaConfig {)}}++ // Table functions (UDTFs) don't have scalar signatures; their return+ // type is always a table, so emit a single row per UDTF with+ // routine_type = "FUNCTION", function_type = "TABLE" and+ // data_type = "TABLE".+ for name in self.table_functions.keys() {+ builder.add_routine(+ catalog_name,+ schema_name,+ name,+ "FUNCTION",+ // No signature is available for UDTFs; report deterministic+ // = false to stay conservative.+ false,+ Some(&"TABLE"),+ "TABLE",+ None::<String>,+ None::<String>,+ )+ }Ok(())}@@ -400,6 +435,14 @@ impl InformationSchemaConfig {}}+ // UDTFs deliberately do NOT appear in `information_schema.parameters`.+ // A same-named scalar UDF (e.g. `generate_series` exists as both a+ // scalar UDF in functions-nested and a UDTF in functions-table) would+ // cross-join with a UDTF row keyed only by (name, rid) and produce+ // spurious `TABLE`-typed variants of every scalar signature in+ // SHOW FUNCTIONS. `show_functions_to_plan` sources UDTFs directly+ // from `information_schema.routines` via a UNION branch instead.+Ok(())}@@ -1522,6 +1565,7 @@ mod tests {async fn make_tables_uses_table_type() {let config = InformationSchemaConfig {catalog_list: Arc::new(Fixture),+ table_functions: HashMap::new(),};let mut builder = InformationSchemaTablesBuilder {catalog_names: StringBuilder::new(),diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex 9ca16d5ba2..f7117c89ef 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -349,9 +349,10 @@ impl SessionState {let resolved_ref = self.resolve_table_ref(table_ref);if self.config.information_schema() && *resolved_ref.schema == *INFORMATION_SCHEMA{- return Ok(Arc::new(InformationSchemaProvider::new(Arc::clone(- &self.catalog_list,- ))));+ return Ok(Arc::new(+ InformationSchemaProvider::new(Arc::clone(&self.catalog_list))+ .with_table_functions(self.table_functions.clone()),+ ));}self.catalog_listdiff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rsindex 838228a3e0..2b60d79b34 100644--- a/datafusion/sql/src/statement.rs+++ b/datafusion/sql/src/statement.rs@@ -2631,17 +2631,35 @@ impl<S: ContextProvider> SqlToRel<'_, S> {"".to_string()};+ // Scalar / aggregate / window functions are resolved by joining+ // parameters (IN rows aggregated per OUT row) with routines.+ // Table functions (UDTFs) don't have parameter rows, so they are+ // sourced directly from routines via a UNION branch. Restricting+ // the JOIN to non-TABLE routines prevents same-named scalar+UDTF+ // pairs (e.g. `generate_series`) from cross-joining.+ let where_clause = where_clause.replace("p.function_name", "sc.function_name");let query = format!(r#"SELECT DISTINCT- p.*,- r.function_type function_type,- r.description description,- r.syntax_example syntax_example-FROM- (+ sc.function_name,+ sc.return_type,+ sc.parameters,+ sc.parameter_types,+ sc.function_type,+ sc.description,+ sc.syntax_example+FROM (+ SELECT+ p.function_name,+ p.return_type,+ p.parameters,+ p.parameter_types,+ r.function_type function_type,+ r.description description,+ r.syntax_example syntax_example+ FROM (SELECT- i.specific_name function_name,+ o.specific_name function_name,o.data_type return_type,array_agg(i.parameter_name ORDER BY i.ordinal_position ASC) parameters,array_agg(i.data_type ORDER BY i.ordinal_position ASC) parameter_types@@ -2657,9 +2675,9 @@ FROMFROMinformation_schema.parametersWHERE- parameter_mode = 'IN'- ) i- JOIN+ parameter_mode = 'OUT'+ ) o+ LEFT JOIN(SELECTspecific_catalog,@@ -2672,16 +2690,32 @@ FROMFROMinformation_schema.parametersWHERE- parameter_mode = 'OUT'- ) o+ parameter_mode = 'IN'+ ) iON i.specific_catalog = o.specific_catalogAND i.specific_schema = o.specific_schemaAND i.specific_name = o.specific_nameAND i.rid = o.rid- GROUP BY 1, 2, i.rid+ GROUP BY 1, 2, o.rid) as p-JOIN information_schema.routines r-ON p.function_name = r.routine_name+ JOIN information_schema.routines r+ ON p.function_name = r.routine_name+ AND r.function_type <> 'TABLE'++ UNION ALL++ SELECT+ routine_name function_name,+ data_type return_type,+ array_agg(NULL) FILTER (WHERE FALSE) parameters,+ array_agg(NULL) FILTER (WHERE FALSE) parameter_types,+ function_type,+ description,+ syntax_example+ FROM information_schema.routines+ WHERE function_type = 'TABLE'+ GROUP BY routine_name, data_type, function_type, description, syntax_example+) sc{where_clause}"#);diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex 1adf98f67f..12306b4529 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -897,6 +897,17 @@ date_trunc Time(ns) [precision, expression] [String, Time(ns)] SCALAR Truncatesdate_trunc Timestamp(ns) [precision, expression] [String, Timestamp(ns)] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression)date_trunc Timestamp(ns, "+TZ") [precision, expression] [String, Timestamp(ns, "+TZ")] SCALAR Truncates a timestamp or time value to a specified precision. date_trunc(precision, expression)+# Table functions (UDTFs) appear in information_schema.routines with+# function_type = TABLE and data_type = TABLE.+# Note: built-in `generate_series` and `range` are registered as BOTH a+# scalar UDF and a UDTF, so this test filters to the TABLE rows to make+# a stable assertion.+query TTT rowsort+select routine_name, data_type, function_type from information_schema.routines where function_type = 'TABLE' order by routine_name;+----+generate_series TABLE TABLE+range TABLE TABLE+statement okshow functions