Apache DataFusion SQL Query Engine
Rename `LogicalPlan::DescribeTable` to `LogicalPlan::Show` move statement implementations out of `sql` module (#26057)
Pepijn Van Eeckhoudte81ab3614c2d
diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rsindex e1c12a315b..cb09bb64a0 100644--- a/datafusion-cli/src/exec.rs+++ b/datafusion-cli/src/exec.rs@@ -365,9 +365,7 @@ impl AdjustedPrintOptions {// all rowsif matches!(plan,- LogicalPlan::Explain(_)- | LogicalPlan::DescribeTable(_)- | LogicalPlan::Analyze(_)+ LogicalPlan::Explain(_) | LogicalPlan::Show(_) | LogicalPlan::Analyze(_)) {self.inner.maxrows = MaxRows::Unlimited;}diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rsindex 751bf8fbdd..e9e6f2d9ae 100644--- a/datafusion/catalog/src/information_schema.rs+++ b/datafusion/catalog/src/information_schema.rs@@ -9,34 +9,49 @@use crate::streaming::StreamingTable;use crate::table::TableFunction;use crate::{CatalogProviderList, SchemaProvider, TableProvider};-use arrow::array::builder::{BooleanBuilder, UInt8Builder};-use arrow::{- array::{StringBuilder, UInt64Builder},- datatypes::{DataType, Field, FieldRef, Schema, SchemaRef},- record_batch::RecordBatch,-};+use arrow::array::StringArray;+use arrow::array::builder::{ListBuilder, StringBuilder};+use arrow::compute::kernels::filter::filter_record_batch;+use arrow::compute::like;+use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};+use arrow::record_batch::RecordBatch;use async_trait::async_trait;-use datafusion_common::DataFusionError;-use datafusion_common::config::{ConfigEntry, ConfigOptions};+use datafusion_common::config::ConfigOptions;use datafusion_common::error::Result;use datafusion_common::types::{LogicalType, NativeType};+use datafusion_common::{+ DataFusionError, ResolvedTableReference, plan_datafusion_err, plan_err,+};+use datafusion_datasource::memory::MemorySourceConfig;use datafusion_execution::TaskContext;use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_expr::function::WindowUDFFieldArgs;+use datafusion_expr::logical_plan::information_schema::{+ ColumnsDetail, InformationSchemaColumnsBuilder, InformationSchemaDfSettingsBuilder,+ InformationSchemaParametersBuilder, InformationSchemaRoutinesBuilder,+ InformationSchemaTablesBuilder, InformationSchemaViewBuilder, columns_schema,+ df_settings_schema, parameters_schema, routines_schema, show_functions_schema,+ tables_schema, views_schema,+};+pub use datafusion_expr::logical_plan::information_schema::{+ InformationSchemataBuilder, schemata_schema,+};use datafusion_expr::type_coercion::functions::fields_with_udf;use datafusion_expr::{- AggregateUDF, ReturnFieldArgs, ScalarUDF, Signature, TypeSignature, WindowUDF,+ AggregateUDF, Documentation, ReturnFieldArgs, ScalarUDF, Signature, TypeSignature,+ WindowUDF,};use datafusion_expr::{TableType, Volatility};-use datafusion_physical_plan::SendableRecordBatchStream;use datafusion_physical_plan::stream::RecordBatchStreamAdapter;-use datafusion_physical_plan::streaming::PartitionStream;+use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec};+use datafusion_physical_plan::{ExecutionPlan, SendableRecordBatchStream};use datafusion_session::CatalogProvider;use itertools::Itertools;use std::collections::{BTreeSet, HashMap, HashSet};use std::fmt::Debug;use std::sync::Arc;+/// Name of the `information_schema` schemapub const INFORMATION_SCHEMA: &str = "information_schema";pub(crate) const TABLES: &str = "tables";pub(crate) const VIEWS: &str = "views";@@ -57,7 +72,8 @@ pub const INFORMATION_SCHEMA_TABLES: &[&str] = &[PARAMETERS,];-/// Implements the `information_schema` virtual schema and tables+/// Implements the `information_schema` virtual schema and tables, and the+/// `SHOW` statements which are based on the same data.////// The underlying tables in the `information_schema` are created on/// demand. This means that if more tables are added to the underlying@@ -69,59 +85,13 @@ pub struct InformationSchemaProvider {}impl InformationSchemaProvider {- /// Creates a new [`InformationSchemaProvider`] for the provided `catalog`- pub fn new_from_catalog(- catalog_name: String,- catalog: Arc<dyn CatalogProvider>,- ) -> Self {- #[derive(Debug)]- struct SingleCatalogList {- catalog_name: String,- catalog: Arc<dyn CatalogProvider>,- }-- impl CatalogProviderList for SingleCatalogList {- fn register_catalog(- &self,- _: String,- _: Arc<dyn CatalogProvider>,- ) -> Option<Arc<dyn CatalogProvider>> {- None- }-- fn catalog_names(&self) -> Vec<String> {- vec![self.catalog_name.clone()]- }-- fn catalog(&self, name: &str) -> Option<Arc<dyn CatalogProvider>> {- if name == self.catalog_name {- Some(Arc::clone(&self.catalog))- } else {- None- }- }- }-- let catalog_list = Arc::new(SingleCatalogList {- catalog_name,- catalog,- });-- Self {- config: InformationSchemaConfig {- system_catalog_name: None,- catalog_list,- table_functions: HashMap::new(),- },- }- }-/// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list`pub fn new(catalog_list: Arc<dyn CatalogProviderList>) -> Self {Self {config: InformationSchemaConfig {- system_catalog_name: None,+ information_schema_enabled: true,catalog_list,+ catalog_filter: None,table_functions: HashMap::new(),},}@@ -137,51 +107,368 @@ impl InformationSchemaProvider {self}- pub fn with_system_catalog(mut self, catalog_name: String) -> Self {- self.config.system_catalog_name = Some(catalog_name);+ /// Sets whether the `information_schema` is enabled. When disabled, its own+ /// tables are not reported and cannot be described.+ pub fn with_information_schema_enabled(mut self, enabled: bool) -> Self {+ self.config.information_schema_enabled = enabled;+ self+ }++ /// Restricts the reported objects to the given catalog. When `None`, all+ /// catalogs are reported.+ pub fn with_catalog_filter(mut self, catalog_name: Option<String>) -> Self {+ self.config.catalog_filter = catalog_name;self}++ /// Creates an execution plan for the `SHOW TABLES` statement.+ pub fn show_tables(&self) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(Arc::new(StreamingTableExec::try_new(+ SchemaRef::from(tables_schema()),+ vec![Arc::new(InformationSchemaTables::new(self.config.clone()))],+ None,+ vec![],+ false,+ None,+ )?))+ }++ /// Creates an execution plan for a `DESCRIBE <query>` statement, where+ /// `schema` is the output schema of the query.+ pub fn describe_schema(&self, schema: &Schema) -> Result<Arc<dyn ExecutionPlan>> {+ let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Describe);+ for (position, field) in schema.fields().iter().enumerate() {+ builder.add_column("", "", "", position, field);+ }+ record_batch_to_exec(builder.finish())+ }++ /// Creates an execution plan for a `DESCRIBE <table>` or+ /// `SHOW [FULL|EXTENDED] COLUMNS FROM <table>` statement.+ pub async fn show_columns(+ &self,+ table_ref: &ResolvedTableReference,+ detail: ColumnsDetail,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let table = self.resolve_table(table_ref).await?;++ let mut builder = InformationSchemaColumnsBuilder::new(detail);+ for (position, field) in table.schema().fields().iter().enumerate() {+ builder.add_column(+ &table_ref.catalog,+ &table_ref.schema,+ &table_ref.table,+ position,+ field,+ );+ }+ record_batch_to_exec(builder.finish())+ }++ /// Creates an execution plan for a `SHOW CREATE TABLE <table>` statement.+ pub async fn show_create_table(+ &self,+ table_ref: &ResolvedTableReference,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let table = self.resolve_table(table_ref).await?;++ let mut builder = InformationSchemaViewBuilder::new();+ builder.add_view(+ table_ref.catalog.as_ref(),+ table_ref.schema.as_ref(),+ table_ref.table.as_ref(),+ table.get_table_definition(),+ );+ let batch = builder.finish();+ record_batch_to_exec(batch)+ }++ async fn resolve_table(+ &self,+ table_ref: &ResolvedTableReference,+ ) -> Result<Arc<dyn TableProvider>, DataFusionError> {+ let not_found = || plan_datafusion_err!("table '{table_ref}' not found");++ // Look up the catalog before the `information_schema` special case so that+ // `information_schema` tables in a non-existent catalog are reported as not found.+ let catalog = self+ .config+ .catalog_list+ .catalog(&table_ref.catalog)+ .ok_or_else(not_found)?;++ let table = if table_ref.schema.as_ref() == INFORMATION_SCHEMA+ && self.config.information_schema_enabled+ {+ self.table(&table_ref.table).await?+ } else {+ catalog+ .schema(&table_ref.schema)+ .ok_or_else(not_found)?+ .table(&table_ref.table)+ .await?+ }+ .ok_or_else(not_found)?;+ Ok(table)+ }++ /// Creates an execution plan for a `SHOW <variable>` statement, or for+ /// `SHOW ALL` when `name` is `None`.+ pub fn show_variables(+ &self,+ config_options: &ConfigOptions,+ runtime_env: &Arc<RuntimeEnv>,+ name: Option<&str>,+ verbose: bool,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let name = name.map(|n| {+ if n == "timezone" || n == "time.zone" {+ // we could introduce alias in OptionDefinition if this string matching thing grows+ "datafusion.execution.time_zone"+ } else {+ n+ }+ });++ if let Some(name) = name {+ // Report unknown variables as an error rather than an empty result+ let is_valid_variable =+ config_options.entries().iter().any(|opt| opt.key == name);+ let is_runtime_variable = name.starts_with("datafusion.runtime.");+ if !is_valid_variable && !is_runtime_variable {+ return plan_err!(+ "'{name}' is not a variable which can be viewed with 'SHOW'"+ );+ }+ }+ let mut builder = InformationSchemaDfSettingsBuilder::new(verbose);+ self.config+ .make_df_settings(config_options, runtime_env, name, &mut builder);+ let batch = builder.finish();+ record_batch_to_exec(batch)+ }++ /// Creates an execution plan for a `SHOW FUNCTIONS [LIKE <pattern>]`+ /// statement. `filter` is the `LIKE` pattern.+ pub fn show_functions(+ &self,+ udfs: &HashMap<String, Arc<ScalarUDF>>,+ udafs: &HashMap<String, Arc<AggregateUDF>>,+ udwfs: &HashMap<String, Arc<WindowUDF>>,+ filter: Option<&str>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let mut builder = ShowFunctionsBuilder::new();++ for (name, udf) in udfs {+ for (arg_types, return_type) in get_udf_args_and_return_types(udf)? {+ builder.add_overload(+ name,+ &arg_types,+ return_type.as_deref(),+ udf.documentation(),+ "SCALAR",+ );+ }+ }+ for (name, udaf) in udafs {+ for (arg_types, return_type) in get_udaf_args_and_return_types(udaf)? {+ builder.add_overload(+ name,+ &arg_types,+ return_type.as_deref(),+ udaf.documentation(),+ "AGGREGATE",+ );+ }+ }+ for (name, udwf) in udwfs {+ for (arg_types, return_type) in get_udwf_args_and_return_types(udwf)? {+ builder.add_overload(+ name,+ &arg_types,+ return_type.as_deref(),+ udwf.documentation(),+ "WINDOW",+ );+ }+ }+ // UDTFs have no signature or documentation+ for name in self.config.table_functions.keys() {+ builder.add_table_function(name);+ }++ record_batch_to_exec(builder.finish(filter)?)+ }+}++/// Wraps a [`RecordBatch`] in a single partition in-memory [`ExecutionPlan`].+fn record_batch_to_exec(record_batch: RecordBatch) -> Result<Arc<dyn ExecutionPlan>> {+ let schema = record_batch.schema();+ let partitions = vec![vec![record_batch]];+ let mem_exec = MemorySourceConfig::try_new_exec(&partitions, schema, None)?;+ Ok(mem_exec)+}++/// Builds `SHOW FUNCTIONS` rows: one per signature of each scalar, aggregate+/// and window function, plus one per table function.+struct ShowFunctionsBuilder {+ function_names: StringBuilder,+ return_types: StringBuilder,+ parameters: ListBuilder<StringBuilder>,+ parameter_types: ListBuilder<StringBuilder>,+ function_types: StringBuilder,+ descriptions: StringBuilder,+ syntax_examples: StringBuilder,+}++impl ShowFunctionsBuilder {+ fn new() -> Self {+ Self {+ function_names: StringBuilder::new(),+ return_types: StringBuilder::new(),+ parameters: ListBuilder::new(StringBuilder::new()),+ parameter_types: ListBuilder::new(StringBuilder::new()),+ function_types: StringBuilder::new(),+ descriptions: StringBuilder::new(),+ syntax_examples: StringBuilder::new(),+ }+ }++ /// Append one row for a single function signature.+ fn add_overload(+ &mut self,+ name: &str,+ arg_types: &[String],+ return_type: Option<&str>,+ documentation: Option<&Documentation>,+ function_type: &str,+ ) {+ self.function_names.append_value(name);+ self.return_types.append_option(return_type);++ // Documented argument names are matched to the signature by position+ let arg_names = documentation.and_then(|d| d.arguments.as_ref());+ for (position, type_name) in arg_types.iter().enumerate() {+ let param_name = arg_names+ .and_then(|args| args.get(position))+ .map(|(arg_name, _)| arg_name.as_str());+ self.parameters.values().append_option(param_name);+ self.parameter_types.values().append_value(type_name);+ }+ self.parameters.append(true);+ self.parameter_types.append(true);++ self.function_types.append_value(function_type);+ self.descriptions+ .append_option(documentation.map(|d| d.description.as_str()));+ self.syntax_examples+ .append_option(documentation.map(|d| d.syntax_example.as_str()));+ }++ /// Append one row for a table function.+ fn add_table_function(&mut self, name: &str) {+ self.function_names.append_value(name);+ self.return_types.append_value("TABLE");+ self.parameters.append(false);+ self.parameter_types.append(false);+ self.function_types.append_value("TABLE");+ self.descriptions.append_null();+ self.syntax_examples.append_null();+ }++ /// Finalize into a [`RecordBatch`], keeping only the functions whose name+ /// matches the `LIKE` pattern `filter`.+ fn finish(mut self, filter: Option<&str>) -> Result<RecordBatch> {+ let batch = RecordBatch::try_new(+ SchemaRef::from(show_functions_schema()),+ vec![+ Arc::new(self.function_names.finish()),+ Arc::new(self.return_types.finish()),+ Arc::new(self.parameters.finish()),+ Arc::new(self.parameter_types.finish()),+ Arc::new(self.function_types.finish()),+ Arc::new(self.descriptions.finish()),+ Arc::new(self.syntax_examples.finish()),+ ],+ )?;+ match filter {+ Some(pattern) => {+ let mask = like(batch.column(0), &StringArray::new_scalar(pattern))?;+ Ok(filter_record_batch(&batch, &mask)?)+ }+ None => Ok(batch),+ }+ }}#[derive(Clone, Debug)]struct InformationSchemaConfig {- system_catalog_name: Option<String>,+ information_schema_enabled: bool,catalog_list: Arc<dyn CatalogProviderList>,+ catalog_filter: Option<String>,table_functions: HashMap<String, Arc<TableFunction>>,}impl InformationSchemaConfig {+ /// The catalogs to report: only `catalog_filter` if set, otherwise all.+ fn catalogs(&self) -> Result<Vec<(String, Arc<dyn CatalogProvider>)>> {+ match &self.catalog_filter {+ Some(catalog_name) => {+ let catalog =+ self.catalog_list.catalog(catalog_name).ok_or_else(|| {+ plan_datafusion_err!("catalog '{catalog_name}' not found")+ })?;+ Ok(vec![(catalog_name.clone(), catalog)])+ }+ // Skip catalogs that were deregistered after `catalog_names` was called+ None => Ok(self+ .catalog_list+ .catalog_names()+ .into_iter()+ .filter_map(|name| {+ let catalog = self.catalog_list.catalog(&name)?;+ Some((name, catalog))+ })+ .collect()),+ }+ }+/// Construct the `information_schema.tables` virtual tableasync fn make_tables(&self,builder: &mut InformationSchemaTablesBuilder,) -> Result<(), DataFusionError> {- // create a mem table with the names of tables- let catalog_names = self.catalog_list.catalog_names();-- for catalog_name in &catalog_names {- let catalog = self.catalog_list.catalog(catalog_name).unwrap();-- for schema_name in catalog.schema_names() {- if schema_name != INFORMATION_SCHEMA {- // schema name may not exist in the catalog, so we need to check- if let Some(schema) = catalog.schema(&schema_name) {- for table_name in schema.table_names() {- if let Some(table_type) =- schema.table_type(&table_name).await?- {- builder.add_table(- catalog_name,- &schema_name,- &table_name,- table_type,- );- }+ for (catalog_name, catalog) in self.catalogs()? {+ self.add_tables(builder, &catalog_name, catalog.as_ref())+ .await?;+ }+ Ok(())+ }++ async fn add_tables(+ &self,+ builder: &mut InformationSchemaTablesBuilder,+ catalog_name: &str,+ catalog: &dyn CatalogProvider,+ ) -> Result<(), DataFusionError> {+ for schema_name in catalog.schema_names() {+ if schema_name != INFORMATION_SCHEMA {+ // schema name may not exist in the catalog, so we need to check+ if let Some(schema) = catalog.schema(&schema_name) {+ for table_name in schema.table_names() {+ if let Some(table_type) = schema.table_type(&table_name).await? {+ builder.add_table(+ catalog_name,+ &schema_name,+ &table_name,+ table_type,+ );}}}}-+ }+ if self.information_schema_enabled {// Add a final list for the information schema tables themselvesfor table_name in INFORMATION_SCHEMA_TABLES {builder.add_table(@@ -192,34 +479,30 @@ impl InformationSchemaConfig {);}}+ Ok(())+ }- if let Some(system_catalog_name) = &self.system_catalog_name- && !catalog_names.contains(system_catalog_name)- {- for table_name in INFORMATION_SCHEMA_TABLES {- builder.add_table(- system_catalog_name,- INFORMATION_SCHEMA,- table_name,- TableType::View,- );- }+ fn make_schemata(+ &self,+ builder: &mut InformationSchemataBuilder,+ ) -> Result<(), DataFusionError> {+ for (catalog_name, catalog) in self.catalogs()? {+ Self::add_schemata(builder, &catalog_name, catalog.as_ref());}-Ok(())}- fn make_schemata(&self, builder: &mut InformationSchemataBuilder) {- for catalog_name in self.catalog_list.catalog_names() {- let catalog = self.catalog_list.catalog(&catalog_name).unwrap();-- for schema_name in catalog.schema_names() {- if schema_name != INFORMATION_SCHEMA- && let Some(schema) = catalog.schema(&schema_name)- {- let schema_owner = schema.owner_name();- builder.add_schemata(&catalog_name, &schema_name, schema_owner);- }+ fn add_schemata(+ builder: &mut InformationSchemataBuilder,+ catalog_name: &str,+ catalog: &dyn CatalogProvider,+ ) {+ for schema_name in catalog.schema_names() {+ if schema_name != INFORMATION_SCHEMA+ && let Some(schema) = catalog.schema(&schema_name)+ {+ let schema_owner = schema.owner_name();+ builder.add_schemata(catalog_name, &schema_name, schema_owner);}}}@@ -228,42 +511,32 @@ impl InformationSchemaConfig {&self,builder: &mut InformationSchemaViewBuilder,) -> Result<(), DataFusionError> {- for catalog_name in self.catalog_list.catalog_names() {- let catalog = self.catalog_list.catalog(&catalog_name).unwrap();-- for schema_name in catalog.schema_names() {- if schema_name != INFORMATION_SCHEMA {- // schema name may not exist in the catalog, so we need to check- if let Some(schema) = catalog.schema(&schema_name) {- Self::add_views(- builder,- &catalog_name,- &schema_name,- schema.as_ref(),- )- .await?;- }- }- }+ for (catalog_name, catalog) in self.catalogs()? {+ Self::add_views(builder, &catalog_name, catalog.as_ref()).await?;}-Ok(())}async fn add_views(builder: &mut InformationSchemaViewBuilder,catalog_name: &str,- schema_name: &str,- schema: &dyn SchemaProvider,+ catalog: &dyn CatalogProvider,) -> Result<(), DataFusionError> {- for table_name in schema.table_names() {- if let Some(table) = schema.table(&table_name).await? {- builder.add_view(- catalog_name,- schema_name,- &table_name,- table.get_table_definition(),- )+ for schema_name in catalog.schema_names() {+ if schema_name != INFORMATION_SCHEMA {+ // schema name may not exist in the catalog, so we need to check+ if let Some(schema) = catalog.schema(&schema_name) {+ for table_name in schema.table_names() {+ if let Some(table) = schema.table(&table_name).await? {+ builder.add_view(+ catalog_name,+ &schema_name,+ &table_name,+ table.get_table_definition(),+ )+ }+ }+ }}}Ok(())@@ -274,63 +547,60 @@ impl InformationSchemaConfig {&self,builder: &mut InformationSchemaColumnsBuilder,) -> Result<(), DataFusionError> {- for catalog_name in self.catalog_list.catalog_names() {- let catalog = self.catalog_list.catalog(&catalog_name).unwrap();-- for schema_name in catalog.schema_names() {- if schema_name != INFORMATION_SCHEMA {- // schema name may not exist in the catalog, so we need to check- if let Some(schema) = catalog.schema(&schema_name) {- Self::add_columns(- builder,- &catalog_name,- &schema_name,- schema.as_ref(),- )- .await?;- }- }- }+ for (catalog_name, catalog) in self.catalogs()? {+ Self::add_columns(builder, &catalog_name, catalog.as_ref()).await?;}-Ok(())}async fn add_columns(builder: &mut InformationSchemaColumnsBuilder,catalog_name: &str,- schema_name: &str,- schema: &dyn SchemaProvider,+ catalog: &dyn CatalogProvider,) -> Result<(), DataFusionError> {- for table_name in schema.table_names() {- if let Some(table) = schema.table(&table_name).await? {- for (field_position, field) in table.schema().fields().iter().enumerate()- {- builder.add_column(- catalog_name,- schema_name,- &table_name,- field_position,- field,- )+ for schema_name in catalog.schema_names() {+ if schema_name != INFORMATION_SCHEMA {+ // schema name may not exist in the catalog, so we need to check+ if let Some(schema) = catalog.schema(&schema_name) {+ for table_name in schema.table_names() {+ if let Some(table) = schema.table(&table_name).await? {+ for (field_position, field) in+ table.schema().fields().iter().enumerate()+ {+ builder.add_column(+ catalog_name,+ &schema_name,+ &table_name,+ field_position,+ field,+ )+ }+ }+ }}}}Ok(())}- /// Construct the `information_schema.df_settings` virtual table+ /// Construct the `information_schema.df_settings` virtual table, restricted+ /// to the setting `name` if given. All settings are ordered by name.fn make_df_settings(&self,config_options: &ConfigOptions,runtime_env: &Arc<RuntimeEnv>,+ name: Option<&str>,builder: &mut InformationSchemaDfSettingsBuilder,) {- for entry in config_options.entries() {- builder.add_setting(entry);+ let mut entries = config_options.entries();+ entries.extend(runtime_env.config_entries());++ match name {+ Some(name) => entries.retain(|entry| entry.key == name),+ None => entries.sort_unstable_by(|a, b| a.key.cmp(&b.key)),}- // Add runtime configuration entries- for entry in runtime_env.config_entries() {++ for entry in entries {builder.add_setting(entry);}}@@ -406,10 +676,7 @@ impl InformationSchemaConfig {}}- // Table functions (UDTFs) don't have scalar signatures; their return- // type is always a table, so emit a single row per UDTF with- // routine_type = "FUNCTION", function_type = "TABLE" and- // data_type = "TABLE".+ // UDTFs have no signature; report a single `TABLE`-typed row for eachfor name in self.table_functions.keys() {builder.add_routine(catalog_name,@@ -524,13 +791,8 @@ impl InformationSchemaConfig {}}- // UDTFs deliberately do NOT appear in `information_schema.parameters`.- // A same-named scalar UDF (e.g. `generate_series` exists as both a- // scalar UDF in functions-nested and a UDTF in functions-table) would- // cross-join with a UDTF row keyed only by (name, rid) and produce- // spurious `TABLE`-typed variants of every scalar signature in- // SHOW FUNCTIONS. `show_functions_to_plan` sources UDTFs directly- // from `information_schema.routines` via a UNION branch instead.+ // UDTFs are deliberately omitted: a scalar UDF with the same name (e.g.+ // `generate_series`) would be indistinguishable from it here.Ok(())}@@ -648,24 +910,6 @@ fn remove_native_type_prefix(native_type: &NativeType) -> String {#[async_trait]impl SchemaProvider for InformationSchemaProvider {- fn table_names(&self) -> Vec<String> {- self.config.table_names()- }-- async fn table(- &self,- name: &str,- ) -> Result<Option<Arc<dyn TableProvider>>, DataFusionError> {- self.config.get_table(name)- }-- fn table_exist(&self, name: &str) -> bool {- self.config.table_exist(name)- }-}--#[async_trait]-impl SchemaProvider for InformationSchemaConfig {fn table_names(&self) -> Vec<String> {INFORMATION_SCHEMA_TABLES.iter()@@ -677,7 +921,7 @@ impl SchemaProvider for InformationSchemaConfig {&self,name: &str,) -> Result<Option<Arc<dyn TableProvider>>, DataFusionError> {- self.get_table(name)+ self.config.get_table(name)}fn table_exist(&self, name: &str) -> bool {@@ -716,23 +960,9 @@ struct InformationSchemaTables {impl InformationSchemaTables {fn new(config: InformationSchemaConfig) -> Self {- let schema = Arc::new(Schema::new(vec![- Field::new("table_catalog", DataType::Utf8, false),- Field::new("table_schema", DataType::Utf8, false),- Field::new("table_name", DataType::Utf8, false),- Field::new("table_type", DataType::Utf8, false),- ]));-- Self { schema, config }- }-- fn builder(&self) -> InformationSchemaTablesBuilder {- InformationSchemaTablesBuilder {- catalog_names: StringBuilder::new(),- schema_names: StringBuilder::new(),- table_names: StringBuilder::new(),- table_types: StringBuilder::new(),- schema: Arc::clone(&self.schema),+ Self {+ schema: Arc::new(tables_schema()),+ config,}}}@@ -743,7 +973,7 @@ impl PartitionStream for InformationSchemaTables {}fn execute(&self, _ctx: Arc<TaskContext>) -> SendableRecordBatchStream {- let mut builder = self.builder();+ let mut builder = InformationSchemaTablesBuilder::new();let config = self.config.clone();Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&self.schema),@@ -756,50 +986,6 @@ impl PartitionStream for InformationSchemaTables {}}-/// Builds the `information_schema.TABLE` table row by row-///-/// Columns are based on <https://www.postgresql.org/docs/current/infoschema-columns.html>-struct InformationSchemaTablesBuilder {- schema: SchemaRef,- catalog_names: StringBuilder,- schema_names: StringBuilder,- table_names: StringBuilder,- table_types: StringBuilder,-}--impl InformationSchemaTablesBuilder {- fn add_table(- &mut self,- catalog_name: impl AsRef<str>,- schema_name: impl AsRef<str>,- table_name: impl AsRef<str>,- table_type: TableType,- ) {- // Note: append_value is actually infallible.- self.catalog_names.append_value(catalog_name.as_ref());- self.schema_names.append_value(schema_name.as_ref());- self.table_names.append_value(table_name.as_ref());- self.table_types.append_value(match table_type {- TableType::Base => "BASE TABLE",- TableType::View => "VIEW",- TableType::Temporary => "LOCAL TEMPORARY",- });- }-- fn finish(&mut self) -> RecordBatch {- RecordBatch::try_new(- Arc::clone(&self.schema),- vec![- Arc::new(self.catalog_names.finish()),- Arc::new(self.schema_names.finish()),- Arc::new(self.table_names.finish()),- Arc::new(self.table_types.finish()),- ],- )- .unwrap()- }-}-#[derive(Debug)]struct InformationSchemaViews {schema: SchemaRef,@@ -808,23 +994,9 @@ struct InformationSchemaViews {impl InformationSchemaViews {fn new(config: InformationSchemaConfig) -> Self {- let schema = Arc::new(Schema::new(vec![- Field::new("table_catalog", DataType::Utf8, false),- Field::new("table_schema", DataType::Utf8, false),- Field::new("table_name", DataType::Utf8, false),- Field::new("definition", DataType::Utf8, true),- ]));-- Self { schema, config }- }-- fn builder(&self) -> InformationSchemaViewBuilder {- InformationSchemaViewBuilder {- catalog_names: StringBuilder::new(),- schema_names: StringBuilder::new(),- table_names: StringBuilder::new(),- definitions: StringBuilder::new(),- schema: Arc::clone(&self.schema),+ Self {+ schema: Arc::new(views_schema()),+ config,}}}@@ -835,7 +1007,7 @@ impl PartitionStream for InformationSchemaViews {}fn execute(&self, _ctx: Arc<TaskContext>) -> SendableRecordBatchStream {- let mut builder = self.builder();+ let mut builder = InformationSchemaViewBuilder::new();let config = self.config.clone();Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&self.schema),@@ -848,46 +1020,6 @@ impl PartitionStream for InformationSchemaViews {}}-/// Builds the `information_schema.VIEWS` table row by row-///-/// Columns are based on <https://www.postgresql.org/docs/current/infoschema-columns.html>-struct InformationSchemaViewBuilder {- schema: SchemaRef,- catalog_names: StringBuilder,- schema_names: StringBuilder,- table_names: StringBuilder,- definitions: StringBuilder,-}--impl InformationSchemaViewBuilder {- fn add_view(- &mut self,- catalog_name: impl AsRef<str>,- schema_name: impl AsRef<str>,- table_name: impl AsRef<str>,- definition: Option<&(impl AsRef<str> + ?Sized)>,- ) {- // Note: append_value is actually infallible.- self.catalog_names.append_value(catalog_name.as_ref());- self.schema_names.append_value(schema_name.as_ref());- self.table_names.append_value(table_name.as_ref());- self.definitions.append_option(definition.as_ref());- }-- fn finish(&mut self) -> RecordBatch {- RecordBatch::try_new(- Arc::clone(&self.schema),- vec![- Arc::new(self.catalog_names.finish()),- Arc::new(self.schema_names.finish()),- Arc::new(self.table_names.finish()),- Arc::new(self.definitions.finish()),- ],- )- .unwrap()- }-}-#[derive(Debug)]struct InformationSchemaColumns {schema: SchemaRef,@@ -896,50 +1028,9 @@ struct InformationSchemaColumns {impl InformationSchemaColumns {fn new(config: InformationSchemaConfig) -> Self {- let schema = Arc::new(Schema::new(vec![- Field::new("table_catalog", DataType::Utf8, false),- Field::new("table_schema", DataType::Utf8, false),- Field::new("table_name", DataType::Utf8, false),- Field::new("column_name", DataType::Utf8, false),- Field::new("ordinal_position", DataType::UInt64, false),- Field::new("column_default", DataType::Utf8, true),- Field::new("is_nullable", DataType::Utf8, false),- Field::new("data_type", DataType::Utf8, false),- Field::new("character_maximum_length", DataType::UInt64, true),- Field::new("character_octet_length", DataType::UInt64, true),- Field::new("numeric_precision", DataType::UInt64, true),- Field::new("numeric_precision_radix", DataType::UInt64, true),- Field::new("numeric_scale", DataType::UInt64, true),- Field::new("datetime_precision", DataType::UInt64, true),- Field::new("interval_type", DataType::Utf8, true),- ]));-- Self { schema, config }- }-- fn builder(&self) -> InformationSchemaColumnsBuilder {- // StringBuilder requires providing an initial capacity, so- // pick 10 here arbitrarily as this is not performance- // critical code and the number of tables is unavailable here.- let default_capacity = 10;-- InformationSchemaColumnsBuilder {- catalog_names: StringBuilder::new(),- schema_names: StringBuilder::new(),- table_names: StringBuilder::new(),- column_names: StringBuilder::new(),- ordinal_positions: UInt64Builder::with_capacity(default_capacity),- column_defaults: StringBuilder::new(),- is_nullables: StringBuilder::new(),- data_types: StringBuilder::new(),- character_maximum_lengths: UInt64Builder::with_capacity(default_capacity),- character_octet_lengths: UInt64Builder::with_capacity(default_capacity),- numeric_precisions: UInt64Builder::with_capacity(default_capacity),- numeric_precision_radixes: UInt64Builder::with_capacity(default_capacity),- numeric_scales: UInt64Builder::with_capacity(default_capacity),- datetime_precisions: UInt64Builder::with_capacity(default_capacity),- interval_types: StringBuilder::new(),- schema: Arc::clone(&self.schema),+ Self {+ schema: Arc::new(columns_schema(ColumnsDetail::Full)),+ config,}}}@@ -950,7 +1041,7 @@ impl PartitionStream for InformationSchemaColumns {}fn execute(&self, _ctx: Arc<TaskContext>) -> SendableRecordBatchStream {- let mut builder = self.builder();+ let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Full);let config = self.config.clone();Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&self.schema),@@ -963,281 +1054,19 @@ impl PartitionStream for InformationSchemaColumns {}}-/// Builds the `information_schema.COLUMNS` table row by row-///-/// Columns are based on <https://www.postgresql.org/docs/current/infoschema-columns.html>-struct InformationSchemaColumnsBuilder {- schema: SchemaRef,- catalog_names: StringBuilder,- schema_names: StringBuilder,- table_names: StringBuilder,- column_names: StringBuilder,- ordinal_positions: UInt64Builder,- column_defaults: StringBuilder,- is_nullables: StringBuilder,- data_types: StringBuilder,- character_maximum_lengths: UInt64Builder,- character_octet_lengths: UInt64Builder,- numeric_precisions: UInt64Builder,- numeric_precision_radixes: UInt64Builder,- numeric_scales: UInt64Builder,- datetime_precisions: UInt64Builder,- interval_types: StringBuilder,-}--impl InformationSchemaColumnsBuilder {- fn add_column(- &mut self,- catalog_name: &str,- schema_name: &str,- table_name: &str,- field_position: usize,- field: &Field,- ) {- use DataType::*;-- // Note: append_value is actually infallible.- self.catalog_names.append_value(catalog_name);- self.schema_names.append_value(schema_name);- self.table_names.append_value(table_name);-- self.column_names.append_value(field.name());-- self.ordinal_positions.append_value(field_position as u64);-- // DataFusion does not support column default values, so null- self.column_defaults.append_null();-- // "YES if the column is possibly nullable, NO if it is known not nullable. "- let nullable_str = if field.is_nullable() { "YES" } else { "NO" };- self.is_nullables.append_value(nullable_str);-- // "System supplied type" --> Use debug format of the datatype- self.data_types.append_value(field.data_type().to_string());-- // "If data_type identifies a character or bit string type, the- // declared maximum length; null for all other data types or- // if no maximum length was declared."- //- // Arrow has no equivalent of VARCHAR(20), so we leave this as Null- let max_chars = None;- self.character_maximum_lengths.append_option(max_chars);-- // "Maximum length, in bytes, for binary data, character data,- // or text and image data."- let char_len: Option<u64> = match field.data_type() {- Utf8 | Binary => Some(i32::MAX as u64),- LargeBinary | LargeUtf8 => Some(i64::MAX as u64),- _ => None,- };- self.character_octet_lengths.append_option(char_len);-- // numeric_precision: "If data_type identifies a numeric type, this column- // contains the (declared or implicit) precision of the type- // for this column. The precision indicates the number of- // significant digits. It can be expressed in decimal (base- // 10) or binary (base 2) terms, as specified in the column- // numeric_precision_radix. For all other data types, this- // column is null."- //- // numeric_radix: If data_type identifies a numeric type, this- // column indicates in which base the values in the columns- // numeric_precision and numeric_scale are expressed. The- // value is either 2 or 10. For all other data types, this- // column is null.- //- // numeric_scale: If data_type identifies an exact numeric- // type, this column contains the (declared or implicit) scale- // of the type for this column. The scale indicates the number- // of significant digits to the right of the decimal point. It- // can be expressed in decimal (base 10) or binary (base 2)- // terms, as specified in the column- // numeric_precision_radix. For all other data types, this- // column is null.- let (numeric_precision, numeric_radix, numeric_scale) = match field.data_type() {- Int8 | UInt8 => (Some(8), Some(2), None),- Int16 | UInt16 => (Some(16), Some(2), None),- Int32 | UInt32 => (Some(32), Some(2), None),- // From max value of 65504 as explained on- // https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Exponent_encoding- Float16 => (Some(15), Some(2), None),- // Numbers from postgres `real` type- Float32 => (Some(24), Some(2), None),- // Numbers from postgres `double` type- Float64 => (Some(24), Some(2), None),- Decimal128(precision, scale) => {- (Some(*precision as u64), Some(10), Some(*scale as u64))- }- _ => (None, None, None),- };-- self.numeric_precisions.append_option(numeric_precision);- self.numeric_precision_radixes.append_option(numeric_radix);- self.numeric_scales.append_option(numeric_scale);-- self.datetime_precisions.append_option(None);- self.interval_types.append_null();- }-- fn finish(&mut self) -> RecordBatch {- RecordBatch::try_new(- Arc::clone(&self.schema),- vec![- Arc::new(self.catalog_names.finish()),- Arc::new(self.schema_names.finish()),- Arc::new(self.table_names.finish()),- Arc::new(self.column_names.finish()),- Arc::new(self.ordinal_positions.finish()),- Arc::new(self.column_defaults.finish()),- Arc::new(self.is_nullables.finish()),- Arc::new(self.data_types.finish()),- Arc::new(self.character_maximum_lengths.finish()),- Arc::new(self.character_octet_lengths.finish()),- Arc::new(self.numeric_precisions.finish()),- Arc::new(self.numeric_precision_radixes.finish()),- Arc::new(self.numeric_scales.finish()),- Arc::new(self.datetime_precisions.finish()),- Arc::new(self.interval_types.finish()),- ],- )- .unwrap()- }-}-#[derive(Debug)]struct InformationSchemata {schema: SchemaRef,config: InformationSchemaConfig,}-/// The Arrow schema of [`information_schema.schemata`] rows.-///-/// Useful for downstream catalog implementations that want to declare a-/// `TableProvider` for `schemata` before populating any rows via-/// [`InformationSchemataBuilder`].-///-/// Columns and nullability match-/// <https://www.postgresql.org/docs/current/infoschema-schemata.html>.-///-/// [`information_schema.schemata`]: https://www.postgresql.org/docs/current/infoschema-schemata.html-pub fn schemata_schema() -> SchemaRef {- Arc::new(Schema::new(vec![- Field::new("catalog_name", DataType::Utf8, false),- Field::new("schema_name", DataType::Utf8, false),- Field::new("schema_owner", DataType::Utf8, true),- Field::new("default_character_set_catalog", DataType::Utf8, true),- Field::new("default_character_set_schema", DataType::Utf8, true),- Field::new("default_character_set_name", DataType::Utf8, true),- Field::new("sql_path", DataType::Utf8, true),- ]))-}-impl InformationSchemata {fn new(config: InformationSchemaConfig) -> Self {Self {- schema: schemata_schema(),+ schema: Arc::new(schemata_schema()),config,}}-- fn builder(&self) -> InformationSchemataBuilder {- InformationSchemataBuilder {- schema: Arc::clone(&self.schema),- catalog_name: StringBuilder::new(),- schema_name: StringBuilder::new(),- schema_owner: StringBuilder::new(),- default_character_set_catalog: StringBuilder::new(),- default_character_set_schema: StringBuilder::new(),- default_character_set_name: StringBuilder::new(),- sql_path: StringBuilder::new(),- }- }-}--/// Builder that produces [`RecordBatch`] values matching the schema of-/// `information_schema.schemata` (see [`schemata_schema`]).-///-/// Intended for downstream catalog implementations that need to emit-/// `schemata` rows from their own metadata source rather than going-/// through DataFusion's `InformationSchemaProvider`, which enumerates-/// schemas synchronously via `CatalogProviderList` and so is unsuitable-/// for catalog backends that resolve asynchronously.-#[derive(Debug)]-pub struct InformationSchemataBuilder {- schema: SchemaRef,- catalog_name: StringBuilder,- schema_name: StringBuilder,- schema_owner: StringBuilder,- default_character_set_catalog: StringBuilder,- default_character_set_schema: StringBuilder,- default_character_set_name: StringBuilder,- sql_path: StringBuilder,-}--impl Default for InformationSchemataBuilder {- fn default() -> Self {- Self::new()- }-}--impl InformationSchemataBuilder {- /// Construct an empty builder.- pub fn new() -> Self {- Self {- schema: schemata_schema(),- catalog_name: StringBuilder::new(),- schema_name: StringBuilder::new(),- schema_owner: StringBuilder::new(),- default_character_set_catalog: StringBuilder::new(),- default_character_set_schema: StringBuilder::new(),- default_character_set_name: StringBuilder::new(),- sql_path: StringBuilder::new(),- }- }-- /// Append one row to the builder. `schema_owner` is the optional SQL- /// schema owner; the three `default_character_set_*` columns and- /// `sql_path` are written as null (DataFusion does not model those- /// concepts; see the PostgreSQL docs link on [`schemata_schema`]).- pub fn add_schemata(- &mut self,- catalog_name: &str,- schema_name: &str,- schema_owner: Option<&str>,- ) {- self.catalog_name.append_value(catalog_name);- self.schema_name.append_value(schema_name);- match schema_owner {- Some(owner) => self.schema_owner.append_value(owner),- None => self.schema_owner.append_null(),- }- self.default_character_set_catalog.append_null();- self.default_character_set_schema.append_null();- self.default_character_set_name.append_null();- self.sql_path.append_null();- }-- /// Finalize the builder into a [`RecordBatch`].- ///- /// Returns an error only if Arrow buffer construction fails, which- /// the builder's column-count and type invariants make unreachable- /// under normal use. The `Result` return type preserves room to add- /// validation in the future without a breaking API change.- pub fn finish(&mut self) -> Result<RecordBatch> {- RecordBatch::try_new(- Arc::clone(&self.schema),- vec![- Arc::new(self.catalog_name.finish()),- Arc::new(self.schema_name.finish()),- Arc::new(self.schema_owner.finish()),- Arc::new(self.default_character_set_catalog.finish()),- Arc::new(self.default_character_set_schema.finish()),- Arc::new(self.default_character_set_name.finish()),- Arc::new(self.sql_path.finish()),- ],- )- .map_err(DataFusionError::from)- }}impl PartitionStream for InformationSchemata {@@ -1246,13 +1075,13 @@ impl PartitionStream for InformationSchemata {}fn execute(&self, _ctx: Arc<TaskContext>) -> SendableRecordBatchStream {- let mut builder = self.builder();+ let mut builder = InformationSchemataBuilder::new();let config = self.config.clone();Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&self.schema),// TODO: Stream thisfutures::stream::once(async move {- config.make_schemata(&mut builder);+ config.make_schemata(&mut builder)?;builder.finish()}),))@@ -1267,21 +1096,9 @@ struct InformationSchemaDfSettings {impl InformationSchemaDfSettings {fn new(config: InformationSchemaConfig) -> Self {- let schema = Arc::new(Schema::new(vec![- Field::new("name", DataType::Utf8, false),- Field::new("value", DataType::Utf8, true),- Field::new("description", DataType::Utf8, true),- ]));-- Self { schema, config }- }-- fn builder(&self) -> InformationSchemaDfSettingsBuilder {- InformationSchemaDfSettingsBuilder {- names: StringBuilder::new(),- values: StringBuilder::new(),- descriptions: StringBuilder::new(),- schema: Arc::clone(&self.schema),+ Self {+ schema: Arc::new(df_settings_schema(true)),+ config,}}}@@ -1293,7 +1110,7 @@ impl PartitionStream for InformationSchemaDfSettings {fn execute(&self, ctx: Arc<TaskContext>) -> SendableRecordBatchStream {let config = self.config.clone();- let mut builder = self.builder();+ let mut builder = InformationSchemaDfSettingsBuilder::new(true);Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&self.schema),// TODO: Stream this@@ -1303,6 +1120,7 @@ impl PartitionStream for InformationSchemaDfSettings {config.make_df_settings(ctx.session_config().options(),&runtime_env,+ None,&mut builder,);Ok(builder.finish())@@ -1311,33 +1129,6 @@ impl PartitionStream for InformationSchemaDfSettings {}}-struct InformationSchemaDfSettingsBuilder {- schema: SchemaRef,- names: StringBuilder,- values: StringBuilder,- descriptions: StringBuilder,-}--impl InformationSchemaDfSettingsBuilder {- fn add_setting(&mut self, entry: ConfigEntry) {- self.names.append_value(entry.key);- self.values.append_option(entry.value);- self.descriptions.append_value(entry.description);- }-- fn finish(&mut self) -> RecordBatch {- RecordBatch::try_new(- Arc::clone(&self.schema),- vec![- Arc::new(self.names.finish()),- Arc::new(self.values.finish()),- Arc::new(self.descriptions.finish()),- ],- )- .unwrap()- }-}-#[derive(Debug)]struct InformationSchemaRoutines {schema: SchemaRef,@@ -1346,109 +1137,13 @@ struct InformationSchemaRoutines {impl InformationSchemaRoutines {fn new(config: InformationSchemaConfig) -> Self {- let schema = Arc::new(Schema::new(vec![- Field::new("specific_catalog", DataType::Utf8, false),- Field::new("specific_schema", DataType::Utf8, false),- Field::new("specific_name", DataType::Utf8, false),- Field::new("routine_catalog", DataType::Utf8, false),- Field::new("routine_schema", DataType::Utf8, false),- Field::new("routine_name", DataType::Utf8, false),- Field::new("routine_type", DataType::Utf8, false),- Field::new("is_deterministic", DataType::Boolean, true),- Field::new("data_type", DataType::Utf8, true),- Field::new("function_type", DataType::Utf8, true),- Field::new("description", DataType::Utf8, true),- Field::new("syntax_example", DataType::Utf8, true),- ]));-- Self { schema, config }- }-- fn builder(&self) -> InformationSchemaRoutinesBuilder {- InformationSchemaRoutinesBuilder {- schema: Arc::clone(&self.schema),- specific_catalog: StringBuilder::new(),- specific_schema: StringBuilder::new(),- specific_name: StringBuilder::new(),- routine_catalog: StringBuilder::new(),- routine_schema: StringBuilder::new(),- routine_name: StringBuilder::new(),- routine_type: StringBuilder::new(),- is_deterministic: BooleanBuilder::new(),- data_type: StringBuilder::new(),- function_type: StringBuilder::new(),- description: StringBuilder::new(),- syntax_example: StringBuilder::new(),+ Self {+ schema: Arc::new(routines_schema()),+ config,}}}-struct InformationSchemaRoutinesBuilder {- schema: SchemaRef,- specific_catalog: StringBuilder,- specific_schema: StringBuilder,- specific_name: StringBuilder,- routine_catalog: StringBuilder,- routine_schema: StringBuilder,- routine_name: StringBuilder,- routine_type: StringBuilder,- is_deterministic: BooleanBuilder,- data_type: StringBuilder,- function_type: StringBuilder,- description: StringBuilder,- syntax_example: StringBuilder,-}--impl InformationSchemaRoutinesBuilder {- #[expect(clippy::too_many_arguments)]- fn add_routine(- &mut self,- catalog_name: impl AsRef<str>,- schema_name: impl AsRef<str>,- routine_name: impl AsRef<str>,- routine_type: impl AsRef<str>,- is_deterministic: bool,- data_type: Option<&impl AsRef<str>>,- function_type: impl AsRef<str>,- description: Option<impl AsRef<str>>,- syntax_example: Option<impl AsRef<str>>,- ) {- self.specific_catalog.append_value(catalog_name.as_ref());- self.specific_schema.append_value(schema_name.as_ref());- self.specific_name.append_value(routine_name.as_ref());- self.routine_catalog.append_value(catalog_name.as_ref());- self.routine_schema.append_value(schema_name.as_ref());- self.routine_name.append_value(routine_name.as_ref());- self.routine_type.append_value(routine_type.as_ref());- self.is_deterministic.append_value(is_deterministic);- self.data_type.append_option(data_type.as_ref());- self.function_type.append_value(function_type.as_ref());- self.description.append_option(description);- self.syntax_example.append_option(syntax_example);- }-- fn finish(&mut self) -> RecordBatch {- RecordBatch::try_new(- Arc::clone(&self.schema),- vec![- Arc::new(self.specific_catalog.finish()),- Arc::new(self.specific_schema.finish()),- Arc::new(self.specific_name.finish()),- Arc::new(self.routine_catalog.finish()),- Arc::new(self.routine_schema.finish()),- Arc::new(self.routine_name.finish()),- Arc::new(self.routine_type.finish()),- Arc::new(self.is_deterministic.finish()),- Arc::new(self.data_type.finish()),- Arc::new(self.function_type.finish()),- Arc::new(self.description.finish()),- Arc::new(self.syntax_example.finish()),- ],- )- .unwrap()- }-}-impl PartitionStream for InformationSchemaRoutines {fn schema(&self) -> &SchemaRef {&self.schema@@ -1456,7 +1151,7 @@ impl PartitionStream for InformationSchemaRoutines {fn execute(&self, ctx: Arc<TaskContext>) -> SendableRecordBatchStream {let config = self.config.clone();- let mut builder = self.builder();+ let mut builder = InformationSchemaRoutinesBuilder::new();Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&self.schema),futures::stream::once(async move {@@ -1481,106 +1176,13 @@ struct InformationSchemaParameters {impl InformationSchemaParameters {fn new(config: InformationSchemaConfig) -> Self {- let schema = Arc::new(Schema::new(vec![- Field::new("specific_catalog", DataType::Utf8, false),- Field::new("specific_schema", DataType::Utf8, false),- Field::new("specific_name", DataType::Utf8, false),- Field::new("ordinal_position", DataType::UInt64, false),- Field::new("parameter_mode", DataType::Utf8, false),- Field::new("parameter_name", DataType::Utf8, true),- Field::new("data_type", DataType::Utf8, false),- Field::new("parameter_default", DataType::Utf8, true),- Field::new("is_variadic", DataType::Boolean, false),- // `rid` (short for `routine id`) is used to differentiate parameters from different signatures- // (It serves as the group-by key when generating the `SHOW FUNCTIONS` query).- // For example, the following signatures have different `rid` values:- // - `datetrunc(Utf8, Timestamp(Microsecond, Some("+TZ"))) -> Timestamp(Microsecond, Some("+TZ"))`- // - `datetrunc(Utf8View, Timestamp(Nanosecond, None)) -> Timestamp(Nanosecond, None)`- Field::new("rid", DataType::UInt8, false),- ]));-- Self { schema, config }- }-- fn builder(&self) -> InformationSchemaParametersBuilder {- InformationSchemaParametersBuilder {- schema: Arc::clone(&self.schema),- specific_catalog: StringBuilder::new(),- specific_schema: StringBuilder::new(),- specific_name: StringBuilder::new(),- ordinal_position: UInt64Builder::new(),- parameter_mode: StringBuilder::new(),- parameter_name: StringBuilder::new(),- data_type: StringBuilder::new(),- parameter_default: StringBuilder::new(),- is_variadic: BooleanBuilder::new(),- rid: UInt8Builder::new(),+ Self {+ schema: Arc::new(parameters_schema()),+ config,}}}-struct InformationSchemaParametersBuilder {- schema: SchemaRef,- specific_catalog: StringBuilder,- specific_schema: StringBuilder,- specific_name: StringBuilder,- ordinal_position: UInt64Builder,- parameter_mode: StringBuilder,- parameter_name: StringBuilder,- data_type: StringBuilder,- parameter_default: StringBuilder,- is_variadic: BooleanBuilder,- rid: UInt8Builder,-}--impl InformationSchemaParametersBuilder {- #[expect(clippy::too_many_arguments)]- fn add_parameter(- &mut self,- specific_catalog: impl AsRef<str>,- specific_schema: impl AsRef<str>,- specific_name: impl AsRef<str>,- ordinal_position: u64,- parameter_mode: impl AsRef<str>,- parameter_name: Option<&(impl AsRef<str> + ?Sized)>,- data_type: impl AsRef<str>,- parameter_default: Option<impl AsRef<str>>,- is_variadic: bool,- rid: u8,- ) {- self.specific_catalog- .append_value(specific_catalog.as_ref());- self.specific_schema.append_value(specific_schema.as_ref());- self.specific_name.append_value(specific_name.as_ref());- self.ordinal_position.append_value(ordinal_position);- self.parameter_mode.append_value(parameter_mode.as_ref());- self.parameter_name.append_option(parameter_name.as_ref());- self.data_type.append_value(data_type.as_ref());- self.parameter_default.append_option(parameter_default);- self.is_variadic.append_value(is_variadic);- self.rid.append_value(rid);- }-- fn finish(&mut self) -> RecordBatch {- RecordBatch::try_new(- Arc::clone(&self.schema),- vec![- Arc::new(self.specific_catalog.finish()),- Arc::new(self.specific_schema.finish()),- Arc::new(self.specific_name.finish()),- Arc::new(self.ordinal_position.finish()),- Arc::new(self.parameter_mode.finish()),- Arc::new(self.parameter_name.finish()),- Arc::new(self.data_type.finish()),- Arc::new(self.parameter_default.finish()),- Arc::new(self.is_variadic.finish()),- Arc::new(self.rid.finish()),- ],- )- .unwrap()- }-}-impl PartitionStream for InformationSchemaParameters {fn schema(&self) -> &SchemaRef {&self.schema@@ -1588,7 +1190,7 @@ impl PartitionStream for InformationSchemaParameters {fn execute(&self, ctx: Arc<TaskContext>) -> SendableRecordBatchStream {let config = self.config.clone();- let mut builder = self.builder();+ let mut builder = InformationSchemaParametersBuilder::new();Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&self.schema),futures::stream::once(async move {@@ -1614,74 +1216,26 @@ mod tests {use datafusion_common::ScalarValue;use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};- #[test]- fn schemata_builder_emits_canonical_schema_and_rows() {- // Construct via `Default` so the test exercises both `new()` (via- // the `Default` impl) and the public column-layout contract.- let mut builder = InformationSchemataBuilder::default();- builder.add_schemata("cat", "schema_one", Some("alice"));- builder.add_schemata("cat", "schema_two", None);- let batch = builder.finish().expect("finish should not fail");-- assert_eq!(batch.schema(), schemata_schema());- assert_eq!(batch.num_rows(), 2);-- let col = |name: &str| {- batch- .column_by_name(name)- .unwrap_or_else(|| panic!("missing column {name}"))- };- let string_col = |name: &str| {- col(name)- .as_any()- .downcast_ref::<arrow::array::StringArray>()- .unwrap_or_else(|| panic!("{name} should be a StringArray"))- };-- let catalog = string_col("catalog_name");- assert_eq!(catalog.value(0), "cat");- assert_eq!(catalog.value(1), "cat");-- let schema = string_col("schema_name");- assert_eq!(schema.value(0), "schema_one");- assert_eq!(schema.value(1), "schema_two");-- let owner = string_col("schema_owner");- assert_eq!(owner.value(0), "alice");- assert!(owner.is_null(1));-- // The three character-set columns and sql_path are unconditionally- // null — they exist for SQL-standard column-layout compatibility.- for name in [- "default_character_set_catalog",- "default_character_set_schema",- "default_character_set_name",- "sql_path",- ] {- let c = string_col(name);- assert!(c.is_null(0), "{name} row 0 should be null");- assert!(c.is_null(1), "{name} row 1 should be null");- }- }-#[tokio::test]async fn make_tables_uses_table_type() {let config = InformationSchemaConfig {- system_catalog_name: None,+ information_schema_enabled: true,catalog_list: Arc::new(Fixture),+ catalog_filter: None,table_functions: HashMap::new(),};- let mut builder = InformationSchemaTablesBuilder {- catalog_names: StringBuilder::new(),- schema_names: StringBuilder::new(),- table_names: StringBuilder::new(),- table_types: StringBuilder::new(),- schema: Arc::new(Schema::empty()),- };+ let mut builder = InformationSchemaTablesBuilder::new();assert!(config.make_tables(&mut builder).await.is_ok());- assert_eq!("BASE TABLE", builder.table_types.finish().value(0));+ let batch = builder.finish();+ let table_types = batch+ .column_by_name("table_type")+ .unwrap()+ .as_any()+ .downcast_ref::<StringArray>()+ .unwrap();+ assert_eq!("BASE TABLE", table_types.value(0));}// UDFdiff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex ee74faa5bd..c825dfb2a8 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -229,11 +229,10 @@ config_namespace! {/// virtual tables for displaying schema informationpub information_schema: bool, default = false- /// The system catalog name - when set together with `information_schema`, the- /// `information_schema` for each catalog will only describe objects contained in the- /// catalog itself. The global, cross-catalog `information_schema` is available under- /// the virtual catalog named by this option.- pub system_catalog: Option<String>, default = None+ /// When set together with `information_schema`, the `information_schema`+ /// of each catalog only describes the objects in that catalog. When not+ /// set, the `information_schema` of each catalog describes all catalogs.+ pub partition_information_schema: bool, default = false/// Location scanned to load tables for `default` schemapub location: Option<String>, default = Nonediff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex 2f3d202260..b008542b47 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -2848,7 +2848,7 @@ mod tests {let ctx = SessionContext::new_with_config(SessionConfig::new().with_information_schema(true)- .with_system_catalog(Some(String::from("system"))),+ .with_partition_information_schema(true),);// Create catalog@@ -2864,7 +2864,7 @@ mod tests {.await?;// Check table exists in schema- let results = ctx.sql("SELECT * FROM system.information_schema.tables WHERE table_catalog='test' AND table_schema='abc' AND table_name = 'y'").await.unwrap().collect().await.unwrap();+ let results = ctx.sql("SELECT * FROM test.information_schema.tables WHERE table_catalog='test' AND table_schema='abc' AND table_name = 'y'").await.unwrap().collect().await.unwrap();assert_eq!(results[0].num_rows(), 1);Ok(())diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex 687e08234c..0149b38a1f 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -23,8 +23,6 @@ use crate::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner};use arrow_schema::DataType;use arrow_schema::FieldRef;use datafusion_catalog::MemoryCatalogProviderList;-#[cfg(feature = "sql")]-use datafusion_catalog::information_schema::INFORMATION_SCHEMA_TABLES;use datafusion_catalog::information_schema::{INFORMATION_SCHEMA, InformationSchemaProvider,};@@ -72,7 +70,7 @@ use datafusion_session::{};#[cfg(feature = "sql")]use datafusion_sql::{- parser::{DFParserBuilder, ExplainStatement, Statement},+ parser::{DFParserBuilder, Statement},planner::{ContextProvider, ParserOptions, PlannerContext, SqlToRel},};@@ -84,9 +82,7 @@ use log::{debug, info};use object_store::ObjectStore;#[cfg(feature = "sql")]use sqlparser::{- ast::{- Expr as SQLExpr, ExprWithAlias as SQLExprWithAlias, Statement as SQLStatement,- },+ ast::{Expr as SQLExpr, ExprWithAlias as SQLExprWithAlias},dialect::dialect_from_str,};use url::Url;@@ -416,45 +412,26 @@ impl SessionState {if *resolved_ref.schema == *INFORMATION_SCHEMA&& self.inner.config.information_schema(){- match self.inner.config.system_catalog() {- Some(system_catalog) => {- if system_catalog == catalog_name {- // Global catalog configured, catalog name references it- // => information schema provider for full catalog list- return Ok(Arc::new(- InformationSchemaProvider::new(Arc::clone(- &self.inner.catalog_list,- ))- .with_table_functions(self.inner.table_functions.clone())- .with_system_catalog(String::from(system_catalog)),- ));- } else {- // Global catalog configured, catalog name is other catalog- // => information schema provider for named catalog- let catalog = self.resolve_catalog(catalog_name)?;- return Ok(Arc::new(- InformationSchemaProvider::new_from_catalog(- catalog_name.to_string(),- catalog,- )- .with_table_functions(self.inner.table_functions.clone()),- ));- }- }- None => {- // Global catalog not configured- // => information schema provider for full catalog list- // Resolve the catalog (even though it's not needed for the InformationSchemaProvider)- // so references to non-existent catalogs report an error.- self.resolve_catalog(catalog_name)?;- return Ok(Arc::new(- InformationSchemaProvider::new(Arc::clone(- &self.inner.catalog_list,- ))- .with_table_functions(self.inner.table_functions.clone()),- ));- }- }+ // Resolve catalog to ensure it exists+ let _ = self.resolve_catalog(catalog_name)?;++ // Create the global information schema+ let information_schema_provider =+ InformationSchemaProvider::new(Arc::clone(self.catalog_list()))+ .with_information_schema_enabled(true)+ .with_table_functions(self.inner.table_functions.clone());++ // If partitioned information schema is enabled, restrict the information schema to+ // the containing catalog+ let information_schema_provider =+ if self.config().partition_information_schema() {+ information_schema_provider+ .with_catalog_filter(Some(catalog_name.to_string()))+ } else {+ information_schema_provider+ };++ return Ok(Arc::new(information_schema_provider));}let catalog = self.resolve_catalog(catalog_name)?;@@ -660,23 +637,7 @@ impl SessionState {&self,statement: Statement,) -> datafusion_common::Result<LogicalPlan> {- let mut references = self.resolve_table_references(&statement)?;-- if Self::requires_information_schema(&statement) {- let info_tables = INFORMATION_SCHEMA_TABLES.iter();- let info_table_refs: Vec<_> = if let Some(system_catalog) =- self.config().system_catalog()- {- info_tables- .map(|n| TableReference::full(system_catalog, INFORMATION_SCHEMA, *n))- .collect()- } else {- info_tables- .map(|n| TableReference::partial(INFORMATION_SCHEMA, *n))- .collect()- };- references.extend(info_table_refs);- }+ let references = self.resolve_table_references(&statement)?;let mut provider = SessionContextProvider {state: self,@@ -699,24 +660,6 @@ impl SessionState {query.statement_to_plan(statement)}- #[cfg(feature = "sql")]- fn requires_information_schema(statement: &Statement) -> bool {- match &statement {- Statement::Explain(ExplainStatement { statement, .. }) => {- Self::requires_information_schema(statement)- }- Statement::Statement(s) => matches!(- s.as_ref(),- SQLStatement::ShowCreate { .. }- | SQLStatement::ShowTables { .. }- | SQLStatement::ShowColumns { .. }- | SQLStatement::ShowVariable { .. }- | SQLStatement::ShowFunctions { .. }- ),- _ => false,- }- }-#[cfg(feature = "sql")]fn get_parser_options(&self) -> ParserOptions {let sql_parser_options = &self.config().options().sql_parser;diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 9ad5d3868b..bcabd26922 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -14,6 +14,7 @@ use crate::datasource::physical_plan::{FileOutputMode, FileSinkConfig};use crate::datasource::{DefaultTableSource, source_as_provider};use crate::error::{DataFusionError, Result};use crate::execution::context::ExecutionProps;+use crate::execution::session_state::SessionState;use crate::logical_expr::utils::generate_sort_key;use crate::logical_expr::{Aggregate, EmptyRelation, Join, Projection, Sort, TableScan, Unnest, Values, Window,@@ -47,11 +48,12 @@ use crate::physical_plan::{};use crate::schema_equivalence::schema_satisfied_by;-use arrow::array::{ArrayRef, RecordBatch, UInt64Array, builder::StringBuilder};+use arrow::array::{ArrayRef, RecordBatch, UInt64Array};use arrow::compute::SortOptions;-use arrow::datatypes::Schema;+use arrow::datatypes::{DataType, Schema};use arrow_schema::Field;use datafusion_catalog::ScanArgs;+use datafusion_catalog::information_schema::InformationSchemaProvider;use datafusion_common::Column;use datafusion_common::HashMap as DFHashMap;use datafusion_common::config::ConfigOptions;@@ -80,10 +82,10 @@ use datafusion_expr::physical_planning_context::{};use datafusion_expr::utils::{expr_to_columns, split_conjunction};use datafusion_expr::{- Analyze, BinaryExpr, DescribeTable, DmlStatement, Explain, ExplainFormat, Extension,- FetchType, Filter, JoinType, MaterializedCte, MaterializedCteId, MaterializedCtePlan,- MaterializedCteScan, Operator, RecursiveQuery, SkipType, StringifiedPlan,- WindowFrame, WindowFrameBound, WriteOp,+ Analyze, BinaryExpr, DmlStatement, Explain, ExplainFormat, Extension, FetchType,+ Filter, JoinType, MaterializedCte, MaterializedCteId, MaterializedCtePlan,+ MaterializedCteScan, Operator, RecursiveQuery, Show, ShowKind, SkipType,+ StringifiedPlan, WindowFrame, WindowFrameBound, WriteOp,};use datafusion_physical_expr::aggregate::{AggregateFunctionExpr, LoweredAggregate, LoweredAggregateBuilder,@@ -780,11 +782,11 @@ impl DefaultPhysicalPlanner {async fn map_logical_node_to_physical(&self,node: &LogicalPlan,- session_state: &dyn Session,+ session: &dyn Session,planning_ctx: &PhysicalPlanningContext,children: ChildrenContainer,) -> Result<Arc<dyn ExecutionPlan>> {- let execution_props = session_state.execution_props();+ let execution_props = session.execution_props();let exec_node: Arc<dyn ExecutionPlan> = match node {// Leaves (no children)LogicalPlan::TableScan(scan) => {@@ -813,7 +815,7 @@ impl DefaultPhysicalPlanner {.with_limit(*fetch).with_skip(*offset).with_statistics_requests(&stats_requests);- let res = source.scan_with_args(session_state, opts).await?;+ let res = source.scan_with_args(session, opts).await?;Arc::clone(res.plan())} else {let mut maybe_plan = None;@@ -823,7 +825,7 @@ impl DefaultPhysicalPlanner {}maybe_plan = planner- .plan_table_scan(self, scan, session_state, planning_ctx)+ .plan_table_scan(self, scan, session, planning_ctx).await?;}@@ -868,12 +870,71 @@ impl DefaultPhysicalPlanner {produce_one_row: true,schema,}) => Arc::new(PlaceholderRowExec::new(Arc::clone(schema.inner()))),- LogicalPlan::DescribeTable(DescribeTable {- schema,- output_schema,+ LogicalPlan::Show(Show {+ kind: ShowKind::Tables,+ ..+ }) => create_information_schema_provider(session).show_tables()?,+ LogicalPlan::Show(Show {+ kind: ShowKind::Describe { schema },+ ..+ }) => create_information_schema_provider(session).describe_schema(schema)?,+ LogicalPlan::Show(Show {+ kind: ShowKind::Columns { table_ref, detail },+ ..+ }) => {+ let catalog_options = &session.config_options().catalog;+ let resolved = table_ref.clone().resolve(+ &catalog_options.default_catalog,+ &catalog_options.default_schema,+ );+ create_information_schema_provider(session)+ .show_columns(&resolved, *detail)+ .await?+ }+ LogicalPlan::Show(Show {+ kind: ShowKind::CreateTable { table_ref },+ ..+ }) => {+ let catalog_options = &session.config_options().catalog;+ let resolved = table_ref.clone().resolve(+ &catalog_options.default_catalog,+ &catalog_options.default_schema,+ );++ create_information_schema_provider(session)+ .show_create_table(&resolved)+ .await?+ }+ LogicalPlan::Show(Show {+ kind: ShowKind::Variables { name, verbose },+ ..+ }) => create_information_schema_provider(session).show_variables(+ session.config_options(),+ session.runtime_env(),+ name.as_deref(),+ *verbose,+ )?,+ LogicalPlan::Show(Show {+ kind: ShowKind::Functions { filter },+ ..}) => {- let output_schema = Arc::clone(output_schema.inner());- self.plan_describe(&Arc::clone(schema), output_schema)?+ // Table functions (UDTFs) are only reachable through the+ // concrete `SessionState` (the `Session` trait has no+ // accessor for them); a custom `Session` implementor will+ // simply see no `TABLE`-typed routines in `SHOW FUNCTIONS`.+ let table_functions = session+ .as_any()+ .downcast_ref::<SessionState>()+ .map(|state| state.table_functions().clone())+ .unwrap_or_default();+ create_information_schema_provider(session)+ .with_table_functions(table_functions)+ .show_functions(+ session.scalar_functions(),+ session.aggregate_functions(),+ session.window_functions(),+ filter.as_deref(),+ )?}LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => {@@ -882,7 +943,7 @@ impl DefaultPhysicalPlanner {scan.id.as_u64(),&scan.name,Arc::clone(scan.schema.inner()),- session_state.config().target_partitions(),+ session.config().target_partitions(),))}@@ -907,7 +968,7 @@ impl DefaultPhysicalPlanner {// the column name rather than column name + explicit data type.let table_partition_cols = partition_by.iter()- .map(|s| (s.to_string(), arrow::datatypes::DataType::Null))+ .map(|s| (s.to_string(), DataType::Null)).collect::<Vec<_>>();let keep_partition_by_columns = match source_option_tuples@@ -915,7 +976,7 @@ impl DefaultPhysicalPlanner {.map(|v| v.trim()){None => {- session_state+ session.config().options().execution@@ -952,8 +1013,8 @@ impl DefaultPhysicalPlanner {.map(|(k, v)| (k.clone(), v.clone())).collect();- let sink_format = file_type_to_format(file_type)?- .create(session_state, &format_options)?;+ let sink_format =+ file_type_to_format(file_type)?.create(session, &format_options)?;// Determine extension based on format extension and compressionlet file_extension = match sink_format.compression_type() {@@ -982,7 +1043,7 @@ impl DefaultPhysicalPlanner {sink_format.create_writer_physical_plan(input_exec,- session_state,+ session,config,ordering.map(Into::into),)@@ -997,7 +1058,7 @@ impl DefaultPhysicalPlanner {let input_exec = children.one()?;provider.table_provider- .insert_into(session_state, input_exec, *insert_op)+ .insert_into(session, input_exec, *insert_op).await?} else {return exec_err!(@@ -1024,7 +1085,7 @@ impl DefaultPhysicalPlanner {extract_dml_filters(input, table_name, &allowed_refs)?;provider.table_provider- .delete_from(session_state, filters)+ .delete_from(session, filters).await.map_err(|e| {e.context(format!(@@ -1062,7 +1123,7 @@ impl DefaultPhysicalPlanner {let assignments = extract_update_assignments(input)?;provider.table_provider- .update(session_state, assignments, filters)+ .update(session, assignments, filters).await.map_err(|e| {e.context(format!(@@ -1086,7 +1147,7 @@ impl DefaultPhysicalPlanner {if let Some(provider) = target.downcast_ref::<DefaultTableSource>() {provider.table_provider- .truncate(session_state)+ .truncate(session).await.map_err(|e| {e.context(format!(@@ -1115,7 +1176,7 @@ impl DefaultPhysicalPlanner {);provider.merge_into(- session_state,+ session,input_exec,merge_schema,merge_op.on.clone(),@@ -1182,8 +1243,8 @@ impl DefaultPhysicalPlanner {}).collect::<Result<Vec<_>>>()?;- let can_repartition = session_state.config().target_partitions() > 1- && session_state.config().repartition_window_functions();+ let can_repartition = session.config().target_partitions() > 1+ && session.config().repartition_window_functions();let uses_bounded_memory =window_expr.iter().all(|e| e.uses_bounded_memory());@@ -1206,7 +1267,7 @@ impl DefaultPhysicalPlanner {}LogicalPlan::Aggregate(aggregate) => self.plan_aggregate(aggregate,- session_state,+ session,planning_ctx,execution_props,children,@@ -1243,7 +1304,7 @@ impl DefaultPhysicalPlanner {Arc::clone(&runtime_expr[0]),physical_input,)- .with_batch_size(session_state.config().batch_size())+ .with_batch_size(session.config().batch_size()).build()?}PlanAsyncExpr::Async(@@ -1263,7 +1324,7 @@ impl DefaultPhysicalPlanner {.apply_projection(Some((0..input.schema().fields().len()).collect::<Vec<_>>(),))?- .with_batch_size(session_state.config().batch_size())+ .with_batch_size(session.config().batch_size()).build()?}_ => {@@ -1273,7 +1334,7 @@ impl DefaultPhysicalPlanner {}};- let selectivity = session_state+ let selectivity = session.config().options().optimizer@@ -1655,12 +1716,12 @@ impl DefaultPhysicalPlanner {};let prefer_hash_join =- session_state.config_options().optimizer.prefer_hash_join;+ session.config_options().optimizer.prefer_hash_join;// Null-aware joins are pinned to CollectLeft hash joins (see// `HashJoinExec::null_aware`): never repartition them, and// never route them to the sort-merge path below.- let can_repartition_join = session_state.config().target_partitions() > 1- && session_state.config().repartition_joins()+ let can_repartition_join = session.config().target_partitions() > 1+ && session.config().repartition_joins()&& !*null_aware;// Only `HashJoinExec` implements null-aware semantics, and it@@ -1681,7 +1742,7 @@ impl DefaultPhysicalPlanner {Arc::new(CrossJoinExec::new(physical_left, physical_right))} else if num_range_filters == 1&& total_filters == 1- && session_state+ && session.config_options().optimizer.enable_piecewise_merge_join@@ -1786,7 +1847,7 @@ impl DefaultPhysicalPlanner {(on_left, on_right),op,*join_type,- session_state.config().target_partitions(),+ session.config().target_partitions(),)?)} else {// there is no equal join condition, use the nested loop join@@ -1798,8 +1859,8 @@ impl DefaultPhysicalPlanner {None,)?)}- } else if session_state.config().target_partitions() > 1- && session_state.config().repartition_joins()+ } else if session.config().target_partitions() > 1+ && session.config().repartition_joins()&& !prefer_hash_join&& !*null_aware// Null-aware joins (e.g. `NOT IN` with a nullable subquery) must@@ -1955,7 +2016,7 @@ impl DefaultPhysicalPlanner {node.as_ref(),&logical_input,&children,- session_state,+ session,planning_ctx,).await?;@@ -2283,6 +2344,15 @@ impl DefaultPhysicalPlanner {}}+fn create_information_schema_provider(+ session: &dyn Session,+) -> InformationSchemaProvider {+ // Do not set a catalog filter. `SHOW` statements describe all catalogs,+ // regardless of `SessionConfig::partition_information_schema`+ let provider = InformationSchemaProvider::new(session.catalog_list());+ provider.with_information_schema_enabled(session.config().information_schema())+}+/// Expand and align a GROUPING SET expression./// (see <https://www.postgresql.org/docs/current/queries-table-expressions.html#QUERIES-GROUPING-SETS>)///@@ -2735,7 +2805,7 @@ fn extract_dml_filters(// Plans without filter informationLogicalPlan::EmptyRelation(_)| LogicalPlan::Values(_)- | LogicalPlan::DescribeTable(_)+ | LogicalPlan::Show(_)| LogicalPlan::Explain(_)| LogicalPlan::Analyze(_)| LogicalPlan::Distinct(_)@@ -3431,43 +3501,6 @@ impl DefaultPhysicalPlanner {Ok(new_plan)}- // return an record_batch which describes a table's schema.- fn plan_describe(- &self,- table_schema: &Arc<Schema>,- output_schema: Arc<Schema>,- ) -> Result<Arc<dyn ExecutionPlan>> {- let mut column_names = StringBuilder::new();- let mut data_types = StringBuilder::new();- let mut is_nullables = StringBuilder::new();- for field in table_schema.fields() {- column_names.append_value(field.name());-- // "System supplied type" --> Use debug format of the datatype- let data_type = field.data_type();- data_types.append_value(format!("{data_type}"));-- // "YES if the column is possibly nullable, NO if it is known not nullable. "- let nullable_str = if field.is_nullable() { "YES" } else { "NO" };- is_nullables.append_value(nullable_str);- }-- let record_batch = RecordBatch::try_new(- output_schema,- vec![- Arc::new(column_names.finish()),- Arc::new(data_types.finish()),- Arc::new(is_nullables.finish()),- ],- )?;-- let schema = record_batch.schema();- let partitions = vec![vec![record_batch]];- let projection = None;- let mem_exec = MemorySourceConfig::try_new_exec(&partitions, schema, projection)?;- Ok(mem_exec)- }-/// Build physical plans for scalar subqueries and assign each an ordinal/// `SubqueryIndex`. Returns the links (plan + index) and a map from logical/// `Subquery` to its index.diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rsindex 0b642327cc..d36edb3dda 100644--- a/datafusion/execution/src/config.rs+++ b/datafusion/execution/src/config.rs@@ -214,13 +214,6 @@ impl SessionConfig {self.options.execution.target_partitions}- /// Get [`system_catalog`]- ///- /// [`system_catalog`]: datafusion_common::config::CatalogOptions::system_catalog- pub fn system_catalog(&self) -> Option<&str> {- self.options.catalog.system_catalog.as_deref()- }-/// Get [`information_schema`]////// [`information_schema`]: datafusion_common::config::CatalogOptions::information_schema@@ -228,6 +221,13 @@ impl SessionConfig {self.options.catalog.information_schema}+ /// Get [`partition_information_schema`]+ ///+ /// [`partition_information_schema`]: datafusion_common::config::CatalogOptions::partition_information_schema+ pub fn partition_information_schema(&self) -> bool {+ self.options.catalog.partition_information_schema+ }+/// Get [`create_default_catalog_and_schema`]////// [`create_default_catalog_and_schema`]: datafusion_common::config::CatalogOptions::create_default_catalog_and_schema@@ -306,11 +306,11 @@ impl SessionConfig {self}- /// Set [`system_catalog`]+ /// Set [`partition_information_schema`]///- /// [`system_catalog`]: datafusion_common::config::CatalogOptions::system_catalog- pub fn with_system_catalog(mut self, catalog_name: Option<String>) -> Self {- self.options_mut().catalog.system_catalog = catalog_name;+ /// [`partition_information_schema`]: datafusion_common::config::CatalogOptions::partition_information_schema+ pub fn with_partition_information_schema(mut self, partition: bool) -> Self {+ self.options_mut().catalog.partition_information_schema = partition;self}diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex b56e0da642..3c523f6014 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -20,8 +20,8 @@ use crate::expr_rewriter::{use crate::logical_plan::{Aggregate, Analyze, AsOfJoin, AsOfMatch, Distinct, DistinctOn, EmptyRelation,Explain, Filter, Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning,- PlanType, Prepare, Projection, Repartition, Sort, SubqueryAlias, TableScanBuilder,- Union, Unnest, Values, Window,+ PlanType, Prepare, Projection, Repartition, Show, ShowKind, Sort, SubqueryAlias,+ TableScanBuilder, Union, Unnest, Values, Window,};use crate::select_expr::SelectExpr;use crate::utils::{@@ -37,6 +37,7 @@ use crate::{};use super::dml::InsertOp;+use crate::logical_plan::information_schema::ColumnsDetail;use arrow::compute::can_cast_types;use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef};use datafusion_common::display::ToStringifiedPlan;@@ -251,6 +252,63 @@ impl LogicalPlanBuilder {}))}+ /// Create a `DESCRIBE <query>` plan, where `schema` is the output schema of the query.+ pub fn describe(schema: Arc<Schema>) -> Result<Self> {+ Self::show(+ ShowKind::Describe { schema },+ crate::information_schema::columns_schema(ColumnsDetail::Describe),+ )+ }++ /// Create a `SHOW TABLES` plan.+ pub fn show_tables() -> Result<Self> {+ Self::show(ShowKind::Tables, crate::information_schema::tables_schema())+ }++ /// Create a `DESCRIBE <table>` or `SHOW [FULL|EXTENDED] COLUMNS FROM <table>` plan.+ pub fn show_columns(+ table_ref: TableReference,+ detail: ColumnsDetail,+ ) -> Result<Self> {+ Self::show(+ ShowKind::Columns { table_ref, detail },+ crate::information_schema::columns_schema(detail),+ )+ }++ /// Create a `SHOW CREATE TABLE <table>` plan.+ pub fn show_create_table(table_ref: TableReference) -> Result<Self> {+ Self::show(+ ShowKind::CreateTable { table_ref },+ crate::information_schema::views_schema(),+ )+ }++ /// Create a `SHOW <variable>` / `SHOW ALL` plan. `name` is `None` for+ /// `SHOW ALL`.+ pub fn show_variables(name: Option<String>, verbose: bool) -> Result<Self> {+ Self::show(+ ShowKind::Variables { name, verbose },+ crate::information_schema::df_settings_schema(verbose),+ )+ }++ /// Create a `SHOW FUNCTIONS [LIKE <pattern>]` plan.+ pub fn show_functions(filter: Option<String>) -> Result<Self> {+ Self::show(+ ShowKind::Functions { filter },+ crate::information_schema::show_functions_schema(),+ )+ }++ /// Shared helper for the `SHOW`-class factory methods above.+ fn show(kind: ShowKind, output_schema: Schema) -> Result<Self> {+ Ok(Self::new(LogicalPlan::Show(Show {+ kind,+ output_schema: Arc::new(DFSchema::try_from(output_schema)?),+ })))+ }+/// Convert a regular plan into a recursive query./// `is_distinct` indicates whether the recursive term should be de-duplicated (`UNION`) after each iteration or not (`UNION ALL`).pub fn to_recursive_query(diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rsindex f92d306842..bc3ee91612 100644--- a/datafusion/expr/src/logical_plan/display.rs+++ b/datafusion/expr/src/logical_plan/display.rs@@ -8,14 +8,15 @@ use std::collections::HashMap;use std::fmt;use crate::{- Aggregate, AsOfJoin, DescribeTable, Distinct, DistinctOn, DmlStatement, Expr, Filter,- Join, Limit, LogicalPlan, MaterializedCte, MaterializedCtePlan, MaterializedCteScan,- Partitioning, Projection, RecursiveQuery, Repartition, Sort, Subquery, SubqueryAlias,- TableProviderFilterPushDown, TableScan, Unnest, Values, Window, WriteOp,- expr_vec_fmt,+ Aggregate, AsOfJoin, Distinct, DistinctOn, DmlStatement, Expr, Filter, Join, Limit,+ LogicalPlan, MaterializedCte, MaterializedCtePlan, MaterializedCteScan, Partitioning,+ Projection, RecursiveQuery, Repartition, Show, ShowKind, Sort, Subquery,+ SubqueryAlias, TableProviderFilterPushDown, TableScan, Unnest, Values, Window,+ WriteOp, expr_vec_fmt,};use crate::dml::CopyTo;+use crate::logical_plan::information_schema::ColumnsDetail;use arrow::datatypes::Schema;use datafusion_common::display::GraphvizBuilder;use datafusion_common::tree_node::{TreeNodeRecursion, TreeNodeVisitor};@@ -712,11 +713,59 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {"Name": name,)}- LogicalPlan::DescribeTable(DescribeTable { .. }) => {- pg_fields!(- "Node Type": "DescribeTable"- )- }+ LogicalPlan::Show(Show { kind, .. }) => match kind {+ ShowKind::Describe { .. } => {+ pg_fields!(+ "Node Type": "Describe"+ )+ }+ ShowKind::Tables => {+ pg_fields!(+ "Node Type": "ShowTables"+ )+ }+ ShowKind::Columns {+ table_ref,+ detail: ColumnsDetail::Describe,+ } => {+ pg_fields!(+ "Node Type": "DescribeTable",+ "Relation Name": table_ref.table()+ )+ }+ ShowKind::Columns { table_ref, detail } => {+ pg_fields!(+ "Node Type": "ShowColumns",+ "Relation Name": table_ref.table(),+ "Detail": format!("{detail:?}")+ )+ }+ ShowKind::CreateTable { table_ref } => {+ pg_fields!(+ "Node Type": "ShowCreateTable",+ "Relation Name": table_ref.table()+ )+ }+ ShowKind::Variables { name, verbose } => {+ let mut object = pg_fields!(+ "Node Type": "ShowVariables",+ "Verbose": verbose+ );+ if let Some(name) = name {+ object.push("Variable", serde_json::Value::String(name.clone()));+ }+ object+ }+ ShowKind::Functions { filter } => {+ let mut object = pg_fields!(+ "Node Type": "ShowFunctions"+ );+ if let Some(filter) = filter {+ object.push("Filter", serde_json::Value::String(filter.clone()));+ }+ object+ }+ },LogicalPlan::Unnest(Unnest {input: plan,list_type_columns: list_col_indices,diff --git a/datafusion/expr/src/logical_plan/information_schema.rs b/datafusion/expr/src/logical_plan/information_schema.rsnew file mode 100644index 0000000000..fa201f618e--- /dev/null+++ b/datafusion/expr/src/logical_plan/information_schema.rs@@ -0,0 +1,829 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++//! Schema and [`RecordBatch`] builders for the `information_schema` virtual+//! tables and `SHOW` statements.++use crate::TableType;+use arrow::array::ArrayRef;+use arrow::array::builder::{BooleanBuilder, StringBuilder, UInt8Builder, UInt64Builder};+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use arrow::record_batch::RecordBatch;+use datafusion_common::Result;+use datafusion_common::config::ConfigEntry;+use std::sync::Arc;++/// Returns the Arrow schema of `information_schema.tables` rows.+///+/// Columns are based on <https://www.postgresql.org/docs/current/infoschema-tables.html>.+pub fn tables_schema() -> Schema {+ Schema::new(vec![+ Field::new("table_catalog", DataType::Utf8, false),+ Field::new("table_schema", DataType::Utf8, false),+ Field::new("table_name", DataType::Utf8, false),+ Field::new("table_type", DataType::Utf8, false),+ ])+}++/// Builds `information_schema.tables` rows.+#[derive(Debug)]+pub struct InformationSchemaTablesBuilder {+ schema: SchemaRef,+ catalog_names: StringBuilder,+ schema_names: StringBuilder,+ table_names: StringBuilder,+ table_types: StringBuilder,+}++impl Default for InformationSchemaTablesBuilder {+ fn default() -> Self {+ Self::new()+ }+}++impl InformationSchemaTablesBuilder {+ /// Construct an empty builder.+ pub fn new() -> Self {+ Self {+ schema: Arc::new(tables_schema()),+ catalog_names: StringBuilder::new(),+ schema_names: StringBuilder::new(),+ table_names: StringBuilder::new(),+ table_types: StringBuilder::new(),+ }+ }++ /// Append one row.+ pub fn add_table(+ &mut self,+ catalog_name: impl AsRef<str>,+ schema_name: impl AsRef<str>,+ table_name: impl AsRef<str>,+ table_type: TableType,+ ) {+ // Note: append_value is actually infallible.+ self.catalog_names.append_value(catalog_name.as_ref());+ self.schema_names.append_value(schema_name.as_ref());+ self.table_names.append_value(table_name.as_ref());+ self.table_types.append_value(match table_type {+ TableType::Base => "BASE TABLE",+ TableType::View => "VIEW",+ TableType::Temporary => "LOCAL TEMPORARY",+ });+ }++ /// Finalize the builder into a [`RecordBatch`].+ pub fn finish(&mut self) -> RecordBatch {+ RecordBatch::try_new(+ Arc::clone(&self.schema),+ vec![+ Arc::new(self.catalog_names.finish()),+ Arc::new(self.schema_names.finish()),+ Arc::new(self.table_names.finish()),+ Arc::new(self.table_types.finish()),+ ],+ )+ .unwrap()+ }+}++/// Returns the Arrow schema of `information_schema.views` rows.+///+/// Columns are based on <https://www.postgresql.org/docs/current/infoschema-views.html>.+pub fn views_schema() -> Schema {+ Schema::new(vec![+ Field::new("table_catalog", DataType::Utf8, false),+ Field::new("table_schema", DataType::Utf8, false),+ Field::new("table_name", DataType::Utf8, false),+ Field::new("definition", DataType::Utf8, true),+ ])+}++/// Builds `information_schema.views` rows.+#[derive(Debug)]+pub struct InformationSchemaViewBuilder {+ schema: SchemaRef,+ catalog_names: StringBuilder,+ schema_names: StringBuilder,+ table_names: StringBuilder,+ definitions: StringBuilder,+}++impl Default for InformationSchemaViewBuilder {+ fn default() -> Self {+ Self::new()+ }+}++impl InformationSchemaViewBuilder {+ /// Construct an empty builder.+ pub fn new() -> Self {+ Self {+ schema: Arc::new(views_schema()),+ catalog_names: StringBuilder::new(),+ schema_names: StringBuilder::new(),+ table_names: StringBuilder::new(),+ definitions: StringBuilder::new(),+ }+ }++ /// Append one row. `definition` is the statement that created the table, if known.+ pub fn add_view(+ &mut self,+ catalog_name: impl AsRef<str>,+ schema_name: impl AsRef<str>,+ table_name: impl AsRef<str>,+ definition: Option<&(impl AsRef<str> + ?Sized)>,+ ) {+ // Note: append_value is actually infallible.+ self.catalog_names.append_value(catalog_name.as_ref());+ self.schema_names.append_value(schema_name.as_ref());+ self.table_names.append_value(table_name.as_ref());+ self.definitions.append_option(definition.as_ref());+ }++ /// Finalize the builder into a [`RecordBatch`].+ pub fn finish(&mut self) -> RecordBatch {+ RecordBatch::try_new(+ Arc::clone(&self.schema),+ vec![+ Arc::new(self.catalog_names.finish()),+ Arc::new(self.schema_names.finish()),+ Arc::new(self.table_names.finish()),+ Arc::new(self.definitions.finish()),+ ],+ )+ .unwrap()+ }+}++/// Which subset of the `information_schema.columns` columns to produce.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum ColumnsDetail {+ /// `column_name`, `data_type` and `is_nullable`, as output by `DESCRIBE`.+ Describe,+ /// [`Self::Describe`] plus the table's catalog, schema and name, as output+ /// by `SHOW COLUMNS`.+ Basic,+ /// All columns, as output by `SHOW FULL COLUMNS` and `information_schema.columns`.+ Full,+}++/// Returns the Arrow schema of `information_schema.columns` rows at the+/// given [`ColumnsDetail`] level.+///+/// The `Full` columns are based on+/// <https://www.postgresql.org/docs/current/infoschema-columns.html>.+pub fn columns_schema(detail: ColumnsDetail) -> Schema {+ match detail {+ ColumnsDetail::Describe => Schema::new(vec![+ Field::new("column_name", DataType::Utf8, false),+ Field::new("data_type", DataType::Utf8, false),+ Field::new("is_nullable", DataType::Utf8, false),+ ]),+ ColumnsDetail::Basic => Schema::new(vec![+ Field::new("table_catalog", DataType::Utf8, false),+ Field::new("table_schema", DataType::Utf8, false),+ Field::new("table_name", DataType::Utf8, false),+ Field::new("column_name", DataType::Utf8, false),+ Field::new("data_type", DataType::Utf8, false),+ Field::new("is_nullable", DataType::Utf8, false),+ ]),+ ColumnsDetail::Full => Schema::new(vec![+ Field::new("table_catalog", DataType::Utf8, false),+ Field::new("table_schema", DataType::Utf8, false),+ Field::new("table_name", DataType::Utf8, false),+ Field::new("column_name", DataType::Utf8, false),+ Field::new("ordinal_position", DataType::UInt64, false),+ Field::new("column_default", DataType::Utf8, true),+ Field::new("is_nullable", DataType::Utf8, false),+ Field::new("data_type", DataType::Utf8, false),+ Field::new("character_maximum_length", DataType::UInt64, true),+ Field::new("character_octet_length", DataType::UInt64, true),+ Field::new("numeric_precision", DataType::UInt64, true),+ Field::new("numeric_precision_radix", DataType::UInt64, true),+ Field::new("numeric_scale", DataType::UInt64, true),+ Field::new("datetime_precision", DataType::UInt64, true),+ Field::new("interval_type", DataType::Utf8, true),+ ]),+ }+}++/// Builds `information_schema.columns` rows at a given [`ColumnsDetail`] level.+#[derive(Debug)]+pub struct InformationSchemaColumnsBuilder {+ schema: SchemaRef,+ detail: ColumnsDetail,+ catalog_names: Option<StringBuilder>,+ schema_names: Option<StringBuilder>,+ table_names: Option<StringBuilder>,+ column_names: StringBuilder,+ ordinal_positions: Option<UInt64Builder>,+ column_defaults: Option<StringBuilder>,+ is_nullables: StringBuilder,+ data_types: StringBuilder,+ character_maximum_lengths: Option<UInt64Builder>,+ character_octet_lengths: Option<UInt64Builder>,+ numeric_precisions: Option<UInt64Builder>,+ numeric_precision_radixes: Option<UInt64Builder>,+ numeric_scales: Option<UInt64Builder>,+ datetime_precisions: Option<UInt64Builder>,+ interval_types: Option<StringBuilder>,+}++impl InformationSchemaColumnsBuilder {+ /// Construct an empty builder that will only retain the columns needed+ /// for `detail`.+ pub fn new(detail: ColumnsDetail) -> Self {+ // StringBuilder requires providing an initial capacity, so+ // pick 10 here arbitrarily as this is not performance+ // critical code and the number of tables is unavailable here.+ let default_capacity = 10;++ let with_table_qualifiers =+ matches!(detail, ColumnsDetail::Basic | ColumnsDetail::Full);+ let full = detail == ColumnsDetail::Full;++ Self {+ schema: Arc::new(columns_schema(detail)),+ detail,+ catalog_names: with_table_qualifiers.then(StringBuilder::new),+ schema_names: with_table_qualifiers.then(StringBuilder::new),+ table_names: with_table_qualifiers.then(StringBuilder::new),+ column_names: StringBuilder::new(),+ ordinal_positions: full+ .then(|| UInt64Builder::with_capacity(default_capacity)),+ column_defaults: full.then(StringBuilder::new),+ is_nullables: StringBuilder::new(),+ data_types: StringBuilder::new(),+ character_maximum_lengths: full+ .then(|| UInt64Builder::with_capacity(default_capacity)),+ character_octet_lengths: full+ .then(|| UInt64Builder::with_capacity(default_capacity)),+ numeric_precisions: full+ .then(|| UInt64Builder::with_capacity(default_capacity)),+ numeric_precision_radixes: full+ .then(|| UInt64Builder::with_capacity(default_capacity)),+ numeric_scales: full.then(|| UInt64Builder::with_capacity(default_capacity)),+ datetime_precisions: full+ .then(|| UInt64Builder::with_capacity(default_capacity)),+ interval_types: full.then(StringBuilder::new),+ }+ }++ /// Append one row describing `field`. Values not needed at this builder's+ /// detail level are discarded.+ pub fn add_column(+ &mut self,+ catalog_name: &str,+ schema_name: &str,+ table_name: &str,+ field_position: usize,+ field: &Field,+ ) {+ use DataType::*;++ // Note: append_value is actually infallible.+ if let Some(b) = &mut self.catalog_names {+ b.append_value(catalog_name);+ }+ if let Some(b) = &mut self.schema_names {+ b.append_value(schema_name);+ }+ if let Some(b) = &mut self.table_names {+ b.append_value(table_name);+ }++ self.column_names.append_value(field.name());++ if let Some(b) = &mut self.ordinal_positions {+ b.append_value(field_position as u64);+ }++ // DataFusion does not support column default values, so null+ if let Some(b) = &mut self.column_defaults {+ b.append_null();+ }++ // "YES if the column is possibly nullable, NO if it is known not nullable. "+ let nullable_str = if field.is_nullable() { "YES" } else { "NO" };+ self.is_nullables.append_value(nullable_str);++ // "System supplied type" --> Use display format of the datatype+ self.data_types.append_value(field.data_type().to_string());++ // "If data_type identifies a character or bit string type, the+ // declared maximum length; null for all other data types or+ // if no maximum length was declared."+ //+ // Arrow has no equivalent of VARCHAR(20), so we leave this as Null+ if let Some(b) = &mut self.character_maximum_lengths {+ b.append_option(None);+ }++ // "Maximum length, in bytes, for binary data, character data,+ // or text and image data."+ if let Some(b) = &mut self.character_octet_lengths {+ let char_len: Option<u64> = match field.data_type() {+ Utf8 | Binary => Some(i32::MAX as u64),+ LargeBinary | LargeUtf8 => Some(i64::MAX as u64),+ _ => None,+ };+ b.append_option(char_len);+ }++ // numeric_precision: "If data_type identifies a numeric type, this column+ // contains the (declared or implicit) precision of the type+ // for this column. The precision indicates the number of+ // significant digits. It can be expressed in decimal (base+ // 10) or binary (base 2) terms, as specified in the column+ // numeric_precision_radix. For all other data types, this+ // column is null."+ //+ // numeric_radix: If data_type identifies a numeric type, this+ // column indicates in which base the values in the columns+ // numeric_precision and numeric_scale are expressed. The+ // value is either 2 or 10. For all other data types, this+ // column is null.+ //+ // numeric_scale: If data_type identifies an exact numeric+ // type, this column contains the (declared or implicit) scale+ // of the type for this column. The scale indicates the number+ // of significant digits to the right of the decimal point. It+ // can be expressed in decimal (base 10) or binary (base 2)+ // terms, as specified in the column+ // numeric_precision_radix. For all other data types, this+ // column is null.+ if let (Some(precisions), Some(radixes), Some(scales)) = (+ &mut self.numeric_precisions,+ &mut self.numeric_precision_radixes,+ &mut self.numeric_scales,+ ) {+ let (numeric_precision, numeric_radix, numeric_scale) =+ match field.data_type() {+ Int8 | UInt8 => (Some(8), Some(2), None),+ Int16 | UInt16 => (Some(16), Some(2), None),+ Int32 | UInt32 => (Some(32), Some(2), None),+ // From max value of 65504 as explained on+ // https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Exponent_encoding+ Float16 => (Some(15), Some(2), None),+ // Numbers from postgres `real` type+ Float32 => (Some(24), Some(2), None),+ // Numbers from postgres `double` type+ Float64 => (Some(24), Some(2), None),+ Decimal128(precision, scale) => {+ (Some(*precision as u64), Some(10), Some(*scale as u64))+ }+ _ => (None, None, None),+ };++ precisions.append_option(numeric_precision);+ radixes.append_option(numeric_radix);+ scales.append_option(numeric_scale);+ }++ if let Some(b) = &mut self.datetime_precisions {+ b.append_option(None);+ }+ if let Some(b) = &mut self.interval_types {+ b.append_null();+ }+ }++ /// Finalize the builder into a [`RecordBatch`].+ pub fn finish(&mut self) -> RecordBatch {+ let mut columns: Vec<ArrayRef> = Vec::new();+ if self.detail != ColumnsDetail::Describe {+ columns.push(Arc::new(self.catalog_names.as_mut().unwrap().finish()));+ columns.push(Arc::new(self.schema_names.as_mut().unwrap().finish()));+ columns.push(Arc::new(self.table_names.as_mut().unwrap().finish()));+ }+ columns.push(Arc::new(self.column_names.finish()));+ if self.detail == ColumnsDetail::Full {+ columns.push(Arc::new(self.ordinal_positions.as_mut().unwrap().finish()));+ columns.push(Arc::new(self.column_defaults.as_mut().unwrap().finish()));+ }+ if self.detail == ColumnsDetail::Full {+ columns.push(Arc::new(self.is_nullables.finish()));+ columns.push(Arc::new(self.data_types.finish()));+ } else {+ // `Describe`/`Basic` list `data_type` before `is_nullable`,+ // unlike the SQL-standard `information_schema.columns` order+ // used by `Full`.+ columns.push(Arc::new(self.data_types.finish()));+ columns.push(Arc::new(self.is_nullables.finish()));+ }+ if self.detail == ColumnsDetail::Full {+ columns.push(Arc::new(+ self.character_maximum_lengths.as_mut().unwrap().finish(),+ ));+ columns.push(Arc::new(+ self.character_octet_lengths.as_mut().unwrap().finish(),+ ));+ columns.push(Arc::new(self.numeric_precisions.as_mut().unwrap().finish()));+ columns.push(Arc::new(+ self.numeric_precision_radixes.as_mut().unwrap().finish(),+ ));+ columns.push(Arc::new(self.numeric_scales.as_mut().unwrap().finish()));+ columns.push(Arc::new(+ self.datetime_precisions.as_mut().unwrap().finish(),+ ));+ columns.push(Arc::new(self.interval_types.as_mut().unwrap().finish()));+ }+ RecordBatch::try_new(Arc::clone(&self.schema), columns).unwrap()+ }+}++/// Returns the Arrow schema of `information_schema.schemata` rows.+///+/// Columns and nullability match+/// <https://www.postgresql.org/docs/current/infoschema-schemata.html>.+pub fn schemata_schema() -> Schema {+ Schema::new(vec![+ Field::new("catalog_name", DataType::Utf8, false),+ Field::new("schema_name", DataType::Utf8, false),+ Field::new("schema_owner", DataType::Utf8, true),+ Field::new("default_character_set_catalog", DataType::Utf8, true),+ Field::new("default_character_set_schema", DataType::Utf8, true),+ Field::new("default_character_set_name", DataType::Utf8, true),+ Field::new("sql_path", DataType::Utf8, true),+ ])+}++/// Builds `information_schema.schemata` rows.+#[derive(Debug)]+pub struct InformationSchemataBuilder {+ schema: SchemaRef,+ catalog_name: StringBuilder,+ schema_name: StringBuilder,+ schema_owner: StringBuilder,+ default_character_set_catalog: StringBuilder,+ default_character_set_schema: StringBuilder,+ default_character_set_name: StringBuilder,+ sql_path: StringBuilder,+}++impl Default for InformationSchemataBuilder {+ fn default() -> Self {+ Self::new()+ }+}++impl InformationSchemataBuilder {+ /// Construct an empty builder.+ pub fn new() -> Self {+ Self {+ schema: Arc::new(schemata_schema()),+ catalog_name: StringBuilder::new(),+ schema_name: StringBuilder::new(),+ schema_owner: StringBuilder::new(),+ default_character_set_catalog: StringBuilder::new(),+ default_character_set_schema: StringBuilder::new(),+ default_character_set_name: StringBuilder::new(),+ sql_path: StringBuilder::new(),+ }+ }++ /// Append one row. The `default_character_set_*` and `sql_path` columns are+ /// always null.+ pub fn add_schemata(+ &mut self,+ catalog_name: &str,+ schema_name: &str,+ schema_owner: Option<&str>,+ ) {+ self.catalog_name.append_value(catalog_name);+ self.schema_name.append_value(schema_name);+ match schema_owner {+ Some(owner) => self.schema_owner.append_value(owner),+ None => self.schema_owner.append_null(),+ }+ self.default_character_set_catalog.append_null();+ self.default_character_set_schema.append_null();+ self.default_character_set_name.append_null();+ self.sql_path.append_null();+ }++ /// Finalize the builder into a [`RecordBatch`].+ pub fn finish(&mut self) -> Result<RecordBatch> {+ Ok(RecordBatch::try_new(+ Arc::clone(&self.schema),+ vec![+ Arc::new(self.catalog_name.finish()),+ Arc::new(self.schema_name.finish()),+ Arc::new(self.schema_owner.finish()),+ Arc::new(self.default_character_set_catalog.finish()),+ Arc::new(self.default_character_set_schema.finish()),+ Arc::new(self.default_character_set_name.finish()),+ Arc::new(self.sql_path.finish()),+ ],+ )?)+ }+}++/// Returns the Arrow schema of `information_schema.df_settings` rows.+/// The `description` column is only included when `verbose` is set.+pub fn df_settings_schema(verbose: bool) -> Schema {+ let mut fields = vec![+ Field::new("name", DataType::Utf8, false),+ Field::new("value", DataType::Utf8, true),+ ];+ if verbose {+ fields.push(Field::new("description", DataType::Utf8, true));+ }+ Schema::new(fields)+}++/// Builds `information_schema.df_settings` rows.+#[derive(Debug)]+pub struct InformationSchemaDfSettingsBuilder {+ schema: SchemaRef,+ names: StringBuilder,+ values: StringBuilder,+ descriptions: Option<StringBuilder>,+}++impl InformationSchemaDfSettingsBuilder {+ /// Construct an empty builder producing the columns of+ /// [`df_settings_schema`]`(verbose)`.+ pub fn new(verbose: bool) -> Self {+ Self {+ schema: Arc::new(df_settings_schema(verbose)),+ names: StringBuilder::new(),+ values: StringBuilder::new(),+ descriptions: verbose.then(StringBuilder::new),+ }+ }++ /// Append one row.+ pub fn add_setting(&mut self, entry: ConfigEntry) {+ self.names.append_value(entry.key);+ self.values.append_option(entry.value);+ if let Some(descriptions) = &mut self.descriptions {+ descriptions.append_value(entry.description);+ }+ }++ /// Finalize the builder into a [`RecordBatch`].+ pub fn finish(&mut self) -> RecordBatch {+ let mut columns: Vec<ArrayRef> = vec![+ Arc::new(self.names.finish()),+ Arc::new(self.values.finish()),+ ];+ if let Some(descriptions) = &mut self.descriptions {+ columns.push(Arc::new(descriptions.finish()));+ }+ RecordBatch::try_new(Arc::clone(&self.schema), columns).unwrap()+ }+}++/// Returns the Arrow schema of `information_schema.routines` rows.+pub fn routines_schema() -> Schema {+ Schema::new(vec![+ Field::new("specific_catalog", DataType::Utf8, false),+ Field::new("specific_schema", DataType::Utf8, false),+ Field::new("specific_name", DataType::Utf8, false),+ Field::new("routine_catalog", DataType::Utf8, false),+ Field::new("routine_schema", DataType::Utf8, false),+ Field::new("routine_name", DataType::Utf8, false),+ Field::new("routine_type", DataType::Utf8, false),+ Field::new("is_deterministic", DataType::Boolean, true),+ Field::new("data_type", DataType::Utf8, true),+ Field::new("function_type", DataType::Utf8, true),+ Field::new("description", DataType::Utf8, true),+ Field::new("syntax_example", DataType::Utf8, true),+ ])+}++/// Builds `information_schema.routines` rows.+#[derive(Debug)]+pub struct InformationSchemaRoutinesBuilder {+ schema: SchemaRef,+ specific_catalog: StringBuilder,+ specific_schema: StringBuilder,+ specific_name: StringBuilder,+ routine_catalog: StringBuilder,+ routine_schema: StringBuilder,+ routine_name: StringBuilder,+ routine_type: StringBuilder,+ is_deterministic: BooleanBuilder,+ data_type: StringBuilder,+ function_type: StringBuilder,+ description: StringBuilder,+ syntax_example: StringBuilder,+}++impl Default for InformationSchemaRoutinesBuilder {+ fn default() -> Self {+ Self::new()+ }+}++impl InformationSchemaRoutinesBuilder {+ /// Construct an empty builder.+ pub fn new() -> Self {+ Self {+ schema: Arc::new(routines_schema()),+ specific_catalog: StringBuilder::new(),+ specific_schema: StringBuilder::new(),+ specific_name: StringBuilder::new(),+ routine_catalog: StringBuilder::new(),+ routine_schema: StringBuilder::new(),+ routine_name: StringBuilder::new(),+ routine_type: StringBuilder::new(),+ is_deterministic: BooleanBuilder::new(),+ data_type: StringBuilder::new(),+ function_type: StringBuilder::new(),+ description: StringBuilder::new(),+ syntax_example: StringBuilder::new(),+ }+ }++ /// Append one row.+ #[expect(clippy::too_many_arguments)]+ pub fn add_routine(+ &mut self,+ catalog_name: impl AsRef<str>,+ schema_name: impl AsRef<str>,+ routine_name: impl AsRef<str>,+ routine_type: impl AsRef<str>,+ is_deterministic: bool,+ data_type: Option<&impl AsRef<str>>,+ function_type: impl AsRef<str>,+ description: Option<impl AsRef<str>>,+ syntax_example: Option<impl AsRef<str>>,+ ) {+ self.specific_catalog.append_value(catalog_name.as_ref());+ self.specific_schema.append_value(schema_name.as_ref());+ self.specific_name.append_value(routine_name.as_ref());+ self.routine_catalog.append_value(catalog_name.as_ref());+ self.routine_schema.append_value(schema_name.as_ref());+ self.routine_name.append_value(routine_name.as_ref());+ self.routine_type.append_value(routine_type.as_ref());+ self.is_deterministic.append_value(is_deterministic);+ self.data_type.append_option(data_type.as_ref());+ self.function_type.append_value(function_type.as_ref());+ self.description.append_option(description);+ self.syntax_example.append_option(syntax_example);+ }++ /// Finalize the builder into a [`RecordBatch`].+ pub fn finish(&mut self) -> RecordBatch {+ RecordBatch::try_new(+ Arc::clone(&self.schema),+ vec![+ Arc::new(self.specific_catalog.finish()),+ Arc::new(self.specific_schema.finish()),+ Arc::new(self.specific_name.finish()),+ Arc::new(self.routine_catalog.finish()),+ Arc::new(self.routine_schema.finish()),+ Arc::new(self.routine_name.finish()),+ Arc::new(self.routine_type.finish()),+ Arc::new(self.is_deterministic.finish()),+ Arc::new(self.data_type.finish()),+ Arc::new(self.function_type.finish()),+ Arc::new(self.description.finish()),+ Arc::new(self.syntax_example.finish()),+ ],+ )+ .unwrap()+ }+}++/// Returns the Arrow schema of `information_schema.parameters` rows.+pub fn parameters_schema() -> Schema {+ Schema::new(vec![+ Field::new("specific_catalog", DataType::Utf8, false),+ Field::new("specific_schema", DataType::Utf8, false),+ Field::new("specific_name", DataType::Utf8, false),+ Field::new("ordinal_position", DataType::UInt64, false),+ Field::new("parameter_mode", DataType::Utf8, false),+ Field::new("parameter_name", DataType::Utf8, true),+ Field::new("data_type", DataType::Utf8, false),+ Field::new("parameter_default", DataType::Utf8, true),+ Field::new("is_variadic", DataType::Boolean, false),+ // `rid` (short for `routine id`) is used to differentiate parameters from different signatures+ // (It serves as the group-by key when generating the `SHOW FUNCTIONS` query).+ // For example, the following signatures have different `rid` values:+ // - `datetrunc(Utf8, Timestamp(Microsecond, Some("+TZ"))) -> Timestamp(Microsecond, Some("+TZ"))`+ // - `datetrunc(Utf8View, Timestamp(Nanosecond, None)) -> Timestamp(Nanosecond, None)`+ Field::new("rid", DataType::UInt8, false),+ ])+}++/// Builds `information_schema.parameters` rows.+#[derive(Debug)]+pub struct InformationSchemaParametersBuilder {+ schema: SchemaRef,+ specific_catalog: StringBuilder,+ specific_schema: StringBuilder,+ specific_name: StringBuilder,+ ordinal_position: UInt64Builder,+ parameter_mode: StringBuilder,+ parameter_name: StringBuilder,+ data_type: StringBuilder,+ parameter_default: StringBuilder,+ is_variadic: BooleanBuilder,+ rid: UInt8Builder,+}++impl Default for InformationSchemaParametersBuilder {+ fn default() -> Self {+ Self::new()+ }+}++impl InformationSchemaParametersBuilder {+ /// Construct an empty builder.+ pub fn new() -> Self {+ Self {+ schema: Arc::new(parameters_schema()),+ specific_catalog: StringBuilder::new(),+ specific_schema: StringBuilder::new(),+ specific_name: StringBuilder::new(),+ ordinal_position: UInt64Builder::new(),+ parameter_mode: StringBuilder::new(),+ parameter_name: StringBuilder::new(),+ data_type: StringBuilder::new(),+ parameter_default: StringBuilder::new(),+ is_variadic: BooleanBuilder::new(),+ rid: UInt8Builder::new(),+ }+ }++ /// Append one row.+ #[expect(clippy::too_many_arguments)]+ pub fn add_parameter(+ &mut self,+ specific_catalog: impl AsRef<str>,+ specific_schema: impl AsRef<str>,+ specific_name: impl AsRef<str>,+ ordinal_position: u64,+ parameter_mode: impl AsRef<str>,+ parameter_name: Option<&(impl AsRef<str> + ?Sized)>,+ data_type: impl AsRef<str>,+ parameter_default: Option<impl AsRef<str>>,+ is_variadic: bool,+ rid: u8,+ ) {+ self.specific_catalog+ .append_value(specific_catalog.as_ref());+ self.specific_schema.append_value(specific_schema.as_ref());+ self.specific_name.append_value(specific_name.as_ref());+ self.ordinal_position.append_value(ordinal_position);+ self.parameter_mode.append_value(parameter_mode.as_ref());+ self.parameter_name.append_option(parameter_name.as_ref());+ self.data_type.append_value(data_type.as_ref());+ self.parameter_default.append_option(parameter_default);+ self.is_variadic.append_value(is_variadic);+ self.rid.append_value(rid);+ }++ /// Finalize the builder into a [`RecordBatch`].+ pub fn finish(&mut self) -> RecordBatch {+ RecordBatch::try_new(+ Arc::clone(&self.schema),+ vec![+ Arc::new(self.specific_catalog.finish()),+ Arc::new(self.specific_schema.finish()),+ Arc::new(self.specific_name.finish()),+ Arc::new(self.ordinal_position.finish()),+ Arc::new(self.parameter_mode.finish()),+ Arc::new(self.parameter_name.finish()),+ Arc::new(self.data_type.finish()),+ Arc::new(self.parameter_default.finish()),+ Arc::new(self.is_variadic.finish()),+ Arc::new(self.rid.finish()),+ ],+ )+ .unwrap()+ }+}++/// Returns the Arrow schema of `SHOW FUNCTIONS` output, with one row per function signature:+///+/// - `function_name`+/// - `return_type`+/// - `parameters`: parameter names, in order+/// - `parameter_types`: parameter types, in order+/// - `function_type`: `SCALAR`, `AGGREGATE`, `WINDOW` or `TABLE`+/// - `description` and `syntax_example`: from the function's documentation+///+/// Unlike the other schemas in this module, this does not correspond to an+/// `information_schema` table.+pub fn show_functions_schema() -> Schema {+ Schema::new(vec![+ Field::new("function_name", DataType::Utf8, false),+ Field::new("return_type", DataType::Utf8, true),+ Field::new("parameters", DataType::new_list(DataType::Utf8, true), true),+ Field::new(+ "parameter_types",+ DataType::new_list(DataType::Utf8, true),+ true,+ ),+ Field::new("function_type", DataType::Utf8, true),+ Field::new("description", DataType::Utf8, true),+ Field::new("syntax_example", DataType::Utf8, true),+ ])+}diff --git a/datafusion/expr/src/logical_plan/mod.rs b/datafusion/expr/src/logical_plan/mod.rsindex 17114f0f3d..5191b78392 100644--- a/datafusion/expr/src/logical_plan/mod.rs+++ b/datafusion/expr/src/logical_plan/mod.rs@@ -10,6 +10,7 @@ mod extension;pub(crate) mod invariants;pub mod materialized_cte;pub use invariants::{InvariantLevel, assert_expected_schema, check_subquery_expr};+pub mod information_schema;mod plan;mod statement;pub mod tree_node;@@ -30,10 +31,10 @@ pub use dml::{WriteOp,};pub use plan::{- Aggregate, Analyze, AsOfJoin, AsOfMatch, ColumnUnnestList, DescribeTable, Distinct,- DistinctOn, EmptyRelation, Explain, ExplainOption, Extension, FetchType, Filter,- Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType,- Projection, RangePartitioning, RecursiveQuery, Repartition, SkipType, Sort,+ Aggregate, Analyze, AsOfJoin, AsOfMatch, ColumnUnnestList, Distinct, DistinctOn,+ EmptyRelation, Explain, ExplainOption, Extension, FetchType, Filter, Join,+ JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType, Projection,+ RangePartitioning, RecursiveQuery, Repartition, Show, ShowKind, SkipType, Sort,StringifiedPlan, Subquery, SubqueryAlias, TableScan, TableScanBuilder,ToStringifiedPlan, Union, Unnest, Values, Window, projection_schema,};diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex 26626d4d32..0fac69e7a0 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -10,12 +10,12 @@ use std::fmt::{self, Debug, Display, Formatter};use std::hash::{Hash, Hasher};use std::sync::{Arc, LazyLock};-use super::DdlStatement;use super::dml::CopyTo;use super::invariants::{InvariantLevel, assert_always_invariants_at_current_node,assert_executable_invariants,};+use super::{DdlStatement, information_schema};use crate::builder::{unique_field_aliases, unnest_with_options};use crate::expr::{Alias, Placeholder, Sort as SortExpr, WindowFunction, WindowFunctionParams,@@ -42,6 +42,7 @@ use crate::{requalify_sides_if_needed,};+use crate::logical_plan::information_schema::ColumnsDetail;use crate::statistics::StatisticsRequest;use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, FieldRef, Metadata, Schema, SchemaRef};@@ -342,9 +343,10 @@ pub enum LogicalPlan {Ddl(DdlStatement),/// `COPY TO` for writing plan results to filesCopy(CopyTo),- /// Describe the schema of the table. This is used to implement the- /// SQL `DESCRIBE` command from MySQL.- DescribeTable(DescribeTable),+ /// A `SHOW`-class SQL statement: `SHOW TABLES`, `SHOW COLUMNS`,+ /// `SHOW FUNCTIONS`, `SHOW <variable>`, `SHOW CREATE TABLE`, and+ /// `DESCRIBE` (MySQL's alias for `SHOW COLUMNS FROM`).+ Show(Show),/// Unnest a column that contains a nested list type such as an/// ARRAY. This is used to implement SQL `UNNEST`Unnest(Unnest),@@ -415,9 +417,7 @@ impl LogicalPlan {LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => cte.schema(),LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => &scan.schema,LogicalPlan::Union(Union { schema, .. }) => schema,- LogicalPlan::DescribeTable(DescribeTable { output_schema, .. }) => {- output_schema- }+ LogicalPlan::Show(Show { output_schema, .. }) => output_schema,LogicalPlan::Dml(DmlStatement { output_schema, .. }) => output_schema,LogicalPlan::Copy(CopyTo { output_schema, .. }) => output_schema,LogicalPlan::Ddl(ddl) => ddl.schema(),@@ -454,11 +454,7 @@ impl LogicalPlan {/// Returns the (fixed) output schema for `DESCRIBE` planspub fn describe_schema() -> Schema {- Schema::new(vec![- Field::new("column_name", DataType::Utf8, false),- Field::new("data_type", DataType::Utf8, false),- Field::new("is_nullable", DataType::Utf8, false),- ])+ information_schema::columns_schema(ColumnsDetail::Describe)}/// Returns all expressions (non-recursively) evaluated by the current@@ -560,7 +556,7 @@ impl LogicalPlan {LogicalPlan::TableScan { .. }| LogicalPlan::EmptyRelation { .. }| LogicalPlan::Values { .. }- | LogicalPlan::DescribeTable(_) => vec![],+ | LogicalPlan::Show(_) => vec![],}}@@ -688,7 +684,7 @@ impl LogicalPlan {| LogicalPlan::Dml(_)| LogicalPlan::Copy(_)| LogicalPlan::Ddl(_)- | LogicalPlan::DescribeTable(_)+ | LogicalPlan::Show(_)| LogicalPlan::Unnest(_) => Ok(None),}}@@ -871,7 +867,7 @@ impl LogicalPlan {LogicalPlan::TableScan(_) => Ok(self),LogicalPlan::EmptyRelation(_) => Ok(self),LogicalPlan::Statement(_) => Ok(self),- LogicalPlan::DescribeTable(_) => Ok(self),+ LogicalPlan::Show(_) => Ok(self),LogicalPlan::Unnest(Unnest {input,exec_columns,@@ -1353,7 +1349,7 @@ impl LogicalPlan {LogicalPlan::EmptyRelation(_)| LogicalPlan::Ddl(_)| LogicalPlan::Statement(_)- | LogicalPlan::DescribeTable(_) => {+ | LogicalPlan::Show(_) => {// All of these plan types have no inputs / exprs so should not be calledself.assert_no_expressions(expr)?;self.assert_no_inputs(inputs)?;@@ -1617,7 +1613,7 @@ impl LogicalPlan {| LogicalPlan::Analyze(_)| LogicalPlan::Dml(_)| LogicalPlan::Copy(_)- | LogicalPlan::DescribeTable(_)+ | LogicalPlan::Show(_)| LogicalPlan::Statement(_)| LogicalPlan::Extension(_)| LogicalPlan::MaterializedCte(_) => None,@@ -1695,7 +1691,7 @@ impl LogicalPlan {| LogicalPlan::Analyze(_)| LogicalPlan::Dml(_)| LogicalPlan::Copy(_)- | LogicalPlan::DescribeTable(_)+ | LogicalPlan::Show(_)| LogicalPlan::Statement(_)| LogicalPlan::Extension(_)| LogicalPlan::MaterializedCte(_) => 0,@@ -1736,7 +1732,7 @@ impl LogicalPlan {LogicalPlan::Dml(_) => Ok(None),LogicalPlan::Ddl(_) => Ok(None),LogicalPlan::Copy(_) => Ok(None),- LogicalPlan::DescribeTable(_) => Ok(None),+ LogicalPlan::Show(_) => Ok(None),LogicalPlan::Unnest(_) => Ok(None),LogicalPlan::RecursiveQuery(_) => Ok(None),}@@ -1776,7 +1772,7 @@ impl LogicalPlan {LogicalPlan::Dml(_) => Ok(None),LogicalPlan::Ddl(_) => Ok(None),LogicalPlan::Copy(_) => Ok(None),- LogicalPlan::DescribeTable(_) => Ok(None),+ LogicalPlan::Show(_) => Ok(None),LogicalPlan::Unnest(_) => Ok(None),LogicalPlan::RecursiveQuery(_) => Ok(None),}@@ -2528,9 +2524,18 @@ impl LogicalPlan {)) => {write!(f, "MaterializedCteScan: name={name}")}- LogicalPlan::DescribeTable(DescribeTable { .. }) => {- write!(f, "DescribeTable")- }+ LogicalPlan::Show(Show { kind, .. }) => match kind {+ ShowKind::Describe { .. }+ | ShowKind::Columns {+ detail: ColumnsDetail::Describe,+ ..+ } => write!(f, "DescribeTable"),+ ShowKind::Tables => write!(f, "ShowTables"),+ ShowKind::Columns { .. } => write!(f, "ShowColumns"),+ ShowKind::CreateTable { .. } => write!(f, "ShowCreateTable"),+ ShowKind::Variables { .. } => write!(f, "ShowVariables"),+ ShowKind::Functions { .. } => write!(f, "ShowFunctions"),+ },LogicalPlan::Unnest(Unnest {input: plan,list_type_columns: list_col_indices,@@ -3770,9 +3775,10 @@ impl PartialOrd for Union {}}-/// Describe the schema of table+/// A `SHOW`-class SQL statement: `SHOW TABLES`, `SHOW COLUMNS`, `SHOW+/// FUNCTIONS`, `SHOW <variable>`, `SHOW CREATE TABLE`, and `DESCRIBE`.///-/// # Example output:+/// # Example output (`DESCRIBE`):////// ```sql/// > describe traces;@@ -3793,16 +3799,52 @@ impl PartialOrd for Union {/// +--------------------+-----------------------------+-------------+/// ```#[derive(Debug, Clone, PartialEq, Eq, Hash)]-pub struct DescribeTable {- /// Table schema- pub schema: Arc<Schema>,- /// schema of describe table output+pub struct Show {+ /// Which `SHOW`-class statement this is, and any parameters it carries+ pub kind: ShowKind,+ /// schema of the output produced by this statementpub output_schema: DFSchemaRef,}+/// The specific `SHOW`-class statement carried by a [`Show`] plan.+#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub enum ShowKind {+ /// `DESCRIBE <query>`+ Describe {+ /// The schema to describe+ schema: Arc<Schema>,+ },+ /// `SHOW TABLES`+ Tables,+ /// `SHOW [FULL|EXTENDED] COLUMNS FROM <table>` and `DESCRIBE <table>`+ Columns {+ /// Table to describe+ table_ref: TableReference,+ /// How detailed the output should be+ detail: ColumnsDetail,+ },+ /// `SHOW CREATE TABLE <table>`+ CreateTable {+ /// Table to describe+ table_ref: TableReference,+ },+ /// `SHOW <variable>` / `SHOW ALL`+ Variables {+ /// The name of the variable to show, or `None` if all variables should be shown.+ name: Option<String>,+ /// Whether to include the `description` column (`SHOW ... VERBOSE`)+ verbose: bool,+ },+ /// `SHOW FUNCTIONS [LIKE <pattern>]`+ Functions {+ /// The `LIKE` pattern to filter function names by, if any+ filter: Option<String>,+ },+}+// Manual implementation of `PartialOrd`, returning none since there are no comparable types in-// `DescribeTable`. This allows `LogicalPlan` to derive `PartialOrd`.-impl PartialOrd for DescribeTable {+// `Show`. This allows `LogicalPlan` to derive `PartialOrd`.+impl PartialOrd for Show {fn partial_cmp(&self, _other: &Self) -> Option<Ordering> {// There is no relevant comparison for schemasNone@@ -6906,21 +6948,25 @@ mod tests {schema: Arc::new(DFSchema::empty()),});- let describe_table = LogicalPlan::DescribeTable(DescribeTable {- schema: Arc::new(Schema::new(vec![Field::new(- "foo",- DataType::Int32,- false,- )])),+ let describe_table = LogicalPlan::Show(Show {+ kind: ShowKind::Describe {+ schema: Arc::new(Schema::new(vec![Field::new(+ "foo",+ DataType::Int32,+ false,+ )])),+ },output_schema: DFSchemaRef::new(DFSchema::empty()),});- let describe_table_clone = LogicalPlan::DescribeTable(DescribeTable {- schema: Arc::new(Schema::new(vec![Field::new(- "foo",- DataType::Int32,- false,- )])),+ let describe_table_clone = LogicalPlan::Show(Show {+ kind: ShowKind::Describe {+ schema: Arc::new(Schema::new(vec![Field::new(+ "foo",+ DataType::Int32,+ false,+ )])),+ },output_schema: DFSchemaRef::new(DFSchema::empty()),});diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rsindex 555b057a64..29093d3352 100644--- a/datafusion/expr/src/logical_plan/tree_node.rs+++ b/datafusion/expr/src/logical_plan/tree_node.rs@@ -396,7 +396,7 @@ impl TreeNode for LogicalPlan {| LogicalPlan::EmptyRelation { .. }| LogicalPlan::Values { .. }| LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))- | LogicalPlan::DescribeTable(_) => Transformed::no(self),+ | LogicalPlan::Show(_) => Transformed::no(self),})}}@@ -533,7 +533,7 @@ impl LogicalPlan {| LogicalPlan::Dml(_)| LogicalPlan::Ddl(_)| LogicalPlan::Copy(_)- | LogicalPlan::DescribeTable(_) => Ok(TreeNodeRecursion::Continue),+ | LogicalPlan::Show(_) => Ok(TreeNodeRecursion::Continue),}}@@ -821,7 +821,7 @@ impl LogicalPlan {| LogicalPlan::Dml(_)| LogicalPlan::Ddl(_)| LogicalPlan::Copy(_)- | LogicalPlan::DescribeTable(_) => Transformed::no(self),+ | LogicalPlan::Show(_) => Transformed::no(self),})}diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rsindex ef73988813..89f0cd30dc 100644--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs@@ -567,7 +567,7 @@ impl OptimizerRule for CommonSubexprEliminate {| LogicalPlan::Explain(_)| LogicalPlan::Analyze(_)| LogicalPlan::Statement(_)- | LogicalPlan::DescribeTable(_)+ | LogicalPlan::Show(_)| LogicalPlan::Distinct(_)| LogicalPlan::Extension(_)| LogicalPlan::Dml(_)diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex aee031d33e..e1c6235f31 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -350,7 +350,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {| LogicalPlan::Dml(_)| LogicalPlan::Ddl(_)| LogicalPlan::Copy(_)- | LogicalPlan::DescribeTable(_) => {+ | LogicalPlan::Show(_) => {if holds_outer_reference(&plan) {Ok(self.stop_pull_up(plan))} else {diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rsindex fd59587872..0b402795d3 100644--- a/datafusion/optimizer/src/optimize_projections/mod.rs+++ b/datafusion/optimizer/src/optimize_projections/mod.rs@@ -375,7 +375,7 @@ fn optimize_projections(LogicalPlan::EmptyRelation(_)| LogicalPlan::Values(_)| LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))- | LogicalPlan::DescribeTable(_) => {+ | LogicalPlan::Show(_) => {// These operators have no inputs, so stop the optimization process.return Ok(Transformed::no(plan));}diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rsindex c54c55790f..0e6822ed9f 100644--- a/datafusion/optimizer/src/optimizer.rs+++ b/datafusion/optimizer/src/optimizer.rs@@ -468,7 +468,7 @@ fn map_children_mut<F: FnMut(&mut LogicalPlan) -> Result<bool>>(| LogicalPlan::EmptyRelation { .. }| LogicalPlan::Values { .. }| LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))- | LogicalPlan::DescribeTable(_)+ | LogicalPlan::Show(_)| LogicalPlan::Ddl(DdlStatement::CreateExternalTable(_))| LogicalPlan::Ddl(DdlStatement::CreateCatalogSchema(_))| LogicalPlan::Ddl(DdlStatement::CreateCatalog(_))diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex a43093c5a8..592d56cb44 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -2395,8 +2395,8 @@ impl AsLogicalPlan for LogicalPlanNode {))),})}- LogicalPlan::DescribeTable(_) => Err(proto_error(- "LogicalPlan serde is not yet implemented for DescribeTable",+ LogicalPlan::Show(_) => Err(proto_error(+ "LogicalPlan serde is not yet implemented for Show",)),LogicalPlan::MaterializedCte(_) => {not_impl_err!(diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rsindex a6564512d3..e3701bc1a3 100644--- a/datafusion/sql/src/statement.rs+++ b/datafusion/sql/src/statement.rs@@ -3,18 +3,15 @@// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreementsuse std::collections::{BTreeMap, HashMap, HashSet};-use std::ops::Deref;use std::path::Path;use std::str::FromStr;use std::sync::Arc;use crate::parser::{- CopyToSource, CopyToStatement, CreateExternalCatalog, CreateExternalTable, DFParser,+ CopyToSource, CopyToStatement, CreateExternalCatalog, CreateExternalTable,ExplainStatement, LexOrdering, ResetStatement, Statement as DFStatement,};-use crate::planner::{- ContextProvider, PlannerContext, SqlToRel, object_name_to_qualifier,-};+use crate::planner::{ContextProvider, PlannerContext, SqlToRel};use crate::utils::normalize_ident;use arrow::datatypes::{Field, FieldRef, Fields, Metadata};@@ -33,15 +30,16 @@ use datafusion_expr::dml::{use datafusion_expr::expr_rewriter::normalize_col_with_schemas_and_ambiguity_check;use datafusion_expr::logical_plan::DdlStatement;use datafusion_expr::logical_plan::builder::project;+use datafusion_expr::logical_plan::information_schema::ColumnsDetail;use datafusion_expr::utils::expr_to_columns;use datafusion_expr::{Analyze, Cast, CreateCatalog, CreateCatalogSchema,CreateExternalCatalog as PlanCreateExternalCatalog,CreateExternalTable as PlanCreateExternalTable, CreateFunction, CreateFunctionBody,CreateIndex as PlanCreateIndex, CreateMemoryTable, CreateView, Deallocate,- DescribeTable, DmlStatement, DropCatalog, DropCatalogSchema, DropFunction, DropTable,- DropView, EmptyRelation, Execute, Explain, ExplainFormat, Expr, ExprSchemable,- Filter, LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare,+ DmlStatement, DropCatalog, DropCatalogSchema, DropFunction, DropTable, DropView,+ EmptyRelation, Execute, Explain, ExplainFormat, Expr, ExprSchemable, Filter,+ LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare,ResetVariable, SetVariable, SortExpr, Statement as PlanStatement, ToStringifiedPlan,TransactionAccessMode, TransactionConclusion, TransactionEnd,TransactionIsolationLevel, TransactionStart, Volatility, WriteOp, cast,@@ -1663,31 +1661,15 @@ impl<S: ContextProvider> SqlToRel<'_, S> {/// Generate a logical plan from a "SHOW TABLES" queryfn show_tables_to_plan(&self) -> Result<LogicalPlan> {- let Some(tables_table_ref) = self.resolve_info_table("tables") else {- return plan_err!(- "SHOW TABLES is not supported unless information_schema is enabled"- );- };-- let query = format!("SELECT * FROM {tables_table_ref};");- let mut rewrite = DFParser::parse_sql(&query)?;- assert_eq!(rewrite.len(), 1);- self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1+ LogicalPlanBuilder::show_tables()?.build()}fn describe_table_to_plan(&self, table_name: ObjectName) -> Result<LogicalPlan> {let table_ref = self.object_name_to_table_reference(table_name)?;+ // Do a table lookup to verify the table exists+ let _ = self.context_provider.get_table_source(table_ref.clone())?;- let table_source = self.context_provider.get_table_source(table_ref)?;-- let schema = table_source.schema();-- let output_schema = DFSchema::try_from(LogicalPlan::describe_schema()).unwrap();-- Ok(LogicalPlan::DescribeTable(DescribeTable {- schema,- output_schema: Arc::new(output_schema),- }))+ LogicalPlanBuilder::show_columns(table_ref, ColumnsDetail::Describe)?.build()}fn describe_query_to_plan(&self, query: Query) -> Result<LogicalPlan> {@@ -1695,12 +1677,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let schema = Arc::new(plan.schema().as_arrow().clone());- let output_schema = DFSchema::try_from(LogicalPlan::describe_schema()).unwrap();-- Ok(LogicalPlan::DescribeTable(DescribeTable {- schema,- output_schema: Arc::new(output_schema),- }))+ LogicalPlanBuilder::describe(schema)?.build()}fn copy_to_plan(&self, statement: CopyToStatement) -> Result<LogicalPlan> {@@ -2200,59 +2177,25 @@ impl<S: ContextProvider> SqlToRel<'_, S> {}fn show_variable_to_plan(&self, variable: &[Ident]) -> Result<LogicalPlan> {- let Some(df_settings_table_ref) = self.resolve_info_table("df_settings") else {- return plan_err!(- "SHOW [VARIABLE] is not supported unless information_schema is enabled"- );- };-let verbose = variable.last().map(|s| ident_to_string(s) == "verbose").unwrap_or(false);let mut variable_vec = variable.to_vec();- let mut columns: String = "name, value".to_owned();if verbose {- columns = format!("{columns}, description");variable_vec = variable_vec.split_at(variable_vec.len() - 1).0.to_vec();}let variable = object_name_to_string(&ObjectName::from(variable_vec));- let base_query = format!("SELECT {columns} FROM {df_settings_table_ref}");- let query = if variable == "all" {- // Add an ORDER BY so the output comes out in a consistent order- format!("{base_query} ORDER BY name")- } else if variable == "timezone" || variable == "time.zone" {- // we could introduce alias in OptionDefinition if this string matching thing grows- format!("{base_query} WHERE name = 'datafusion.execution.time_zone'")+ let name = if variable == "all" {+ None} else {- // These values are what are used to make the information_schema table, so we just- // check here, before actually planning or executing the query, if it would produce no- // results, and error preemptively if it would (for a better UX)- let is_valid_variable = self- .context_provider- .options()- .entries()- .iter()- .any(|opt| opt.key == variable);-- // Check if it's a runtime variable- let is_runtime_variable = variable.starts_with("datafusion.runtime.");-- if !is_valid_variable && !is_runtime_variable {- return plan_err!(- "'{variable}' is not a variable which can be viewed with 'SHOW'"- );- }-- format!("{base_query} WHERE name = '{variable}'")+ // No need to validate if the variable exists here; that is handled at execution time.+ Some(variable)};- let mut rewrite = DFParser::parse_sql(&query)?;- assert_eq!(rewrite.len(), 1);-- self.statement_to_plan(rewrite.pop_front().unwrap())+ LogicalPlanBuilder::show_variables(name, verbose)?.build()}fn set_statement_to_plan(&self, statement: Set) -> Result<LogicalPlan> {@@ -2971,219 +2914,43 @@ impl<S: ContextProvider> SqlToRel<'_, S> {full: bool,sql_table_name: ObjectName,) -> Result<LogicalPlan> {- // Figure out the where clause- let where_clause = object_name_to_qualifier(- &sql_table_name,- self.options.enable_ident_normalization,- )?;-- let Some(columns_table_ref) = self.resolve_info_table("columns") else {- return plan_err!(- "SHOW COLUMNS is not supported unless information_schema is enabled"- );- };-- // Do a table lookup to verify the table existslet table_ref = self.object_name_to_table_reference(sql_table_name)?;- let _ = self.context_provider.get_table_source(table_ref)?;+ // Do a table lookup to verify the table exists+ let _ = self.context_provider.get_table_source(table_ref.clone())?;// Treat both FULL and EXTENDED as the same- let select_list = if full || extended {- "*"+ let detail = if full || extended {+ ColumnsDetail::Full} else {- "table_catalog, table_schema, table_name, column_name, data_type, is_nullable"+ ColumnsDetail::Basic};- let query =- format!("SELECT {select_list} FROM {columns_table_ref} WHERE {where_clause}");-- let mut rewrite = DFParser::parse_sql(&query)?;- assert_eq!(rewrite.len(), 1);- self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1+ LogicalPlanBuilder::show_columns(table_ref, detail)?.build()}- /// Rewrite `SHOW FUNCTIONS` to another SQL query- /// The query is based on the `information_schema.routines` and `information_schema.parameters` tables- ///- /// The output columns:- /// - function_name: The name of function- /// - return_type: The return type of the function- /// - parameters: The name of parameters (ordered by the ordinal position)- /// - parameter_types: The type of parameters (ordered by the ordinal position)- /// - description: The description of the function (the description defined in the document)- /// - syntax_example: The syntax_example of the function (the syntax_example defined in the document)+ /// `SHOW FUNCTIONS [LIKE <pattern>]`.fn show_functions_to_plan(&self,filter: Option<ShowStatementFilter>,) -> Result<LogicalPlan> {- let (Some(parameters_table_ref), Some(routines_table_ref)) = (- self.resolve_info_table("parameters"),- self.resolve_info_table("routines"),- ) else {- return plan_err!(- "SHOW FUNCTIONS is not supported unless information_schema is enabled"- );- };-- let where_clause = if let Some(filter) = filter {- match filter {- ShowStatementFilter::Like(like) => {- format!("WHERE p.function_name like '{like}'")- }- _ => return plan_err!("Unsupported SHOW FUNCTIONS filter"),- }- } else {- "".to_string()+ let filter = match filter {+ Some(ShowStatementFilter::Like(like)) => Some(like),+ Some(_) => return plan_err!("Unsupported SHOW FUNCTIONS filter"),+ None => None,};- // Scalar / aggregate / window functions are resolved by joining- // parameters (IN rows aggregated per OUT row) with routines.- // Table functions (UDTFs) don't have parameter rows, so they are- // sourced directly from routines via a UNION branch. Restricting- // the JOIN to non-TABLE routines prevents same-named scalar+UDTF- // pairs (e.g. `generate_series`) from cross-joining.- let where_clause = where_clause.replace("p.function_name", "sc.function_name");- let query = format!(- r#"-SELECT DISTINCT- sc.function_name,- sc.return_type,- sc.parameters,- sc.parameter_types,- sc.function_type,- sc.description,- sc.syntax_example-FROM (- SELECT- p.function_name,- p.return_type,- p.parameters,- p.parameter_types,- r.function_type function_type,- r.description description,- r.syntax_example syntax_example- FROM (- SELECT- o.specific_name function_name,- o.data_type return_type,- array_agg(i.parameter_name ORDER BY i.ordinal_position ASC) parameters,- array_agg(i.data_type ORDER BY i.ordinal_position ASC) parameter_types- FROM (- SELECT- specific_catalog,- specific_schema,- specific_name,- ordinal_position,- parameter_name,- data_type,- rid- FROM- {parameters_table_ref}- WHERE- parameter_mode = 'OUT'- ) o- LEFT JOIN- (- SELECT- specific_catalog,- specific_schema,- specific_name,- ordinal_position,- parameter_name,- data_type,- rid- FROM- {parameters_table_ref}- WHERE- parameter_mode = 'IN'- ) i- ON i.specific_catalog = o.specific_catalog- AND i.specific_schema = o.specific_schema- AND i.specific_name = o.specific_name- AND i.rid = o.rid- GROUP BY 1, 2, o.rid- ) as p- JOIN {routines_table_ref} r- ON p.function_name = r.routine_name- AND r.function_type <> 'TABLE'-- UNION ALL-- SELECT- routine_name function_name,- data_type return_type,- array_agg(NULL) FILTER (WHERE FALSE) parameters,- array_agg(NULL) FILTER (WHERE FALSE) parameter_types,- function_type,- description,- syntax_example- FROM {routines_table_ref}- WHERE function_type = 'TABLE'- GROUP BY routine_name, data_type, function_type, description, syntax_example-) sc-{where_clause}- "#- );- let mut rewrite = DFParser::parse_sql(&query)?;- assert_eq!(rewrite.len(), 1);- self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1+ LogicalPlanBuilder::show_functions(filter)?.build()}fn show_create_table_to_plan(&self,sql_table_name: ObjectName,) -> Result<LogicalPlan> {- let Some(views_table_ref) = self.resolve_info_table("views") else {- return plan_err!(- "SHOW CREATE TABLE is not supported unless information_schema is enabled"- );- };-- // Figure out the where clause- let where_clause = object_name_to_qualifier(- &sql_table_name,- self.options.enable_ident_normalization,- )?;-- // Do a table lookup to verify the table existslet table_ref = self.object_name_to_table_reference(sql_table_name)?;- let _ = self.context_provider.get_table_source(table_ref)?;-- let query = format!(- "SELECT table_catalog, table_schema, table_name, definition FROM {views_table_ref} WHERE {where_clause}"- );-- let mut rewrite = DFParser::parse_sql(&query)?;- assert_eq!(rewrite.len(), 1);- self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1- }-- /// Return true if there is a table provider available for "schema.table"- fn resolve_info_table(&self, table: &str) -> Option<TableReference> {- let table_reference = if let Some(system_catalog) =- &self.context_provider.options().catalog.system_catalog- {- TableReference::Full {- catalog: system_catalog.deref().into(),- schema: "information_schema".into(),- table: table.into(),- }- } else {- TableReference::Partial {- schema: "information_schema".into(),- table: table.into(),- }- };+ // Do a table lookup to verify the table exists+ let _ = self.context_provider.get_table_source(table_ref.clone())?;- if self- .context_provider- .get_table_source(table_reference.clone())- .is_ok()- {- Some(table_reference)- } else {- None- }+ LogicalPlanBuilder::show_create_table(table_ref)?.build()}fn validate_transaction_kind(diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rsindex 7f5ffaecd4..564de3a281 100644--- a/datafusion/sql/src/unparser/plan.rs+++ b/datafusion/sql/src/unparser/plan.rs@@ -191,7 +191,7 @@ impl Unparser<'_> {| LogicalPlan::Analyze(_)| LogicalPlan::Ddl(_)| LogicalPlan::Copy(_)- | LogicalPlan::DescribeTable(_)+ | LogicalPlan::Show(_)| LogicalPlan::RecursiveQuery(_)| LogicalPlan::Unnest(_) => not_impl_err!("Unsupported plan: {plan:?}"),LogicalPlan::MaterializedCte(_) => {diff --git a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rsindex d3597e1c44..91caab4a48 100644--- a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs+++ b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs@@ -160,7 +160,7 @@ async fn run_query_substrait_round_trip(| LogicalPlan::Explain(_)| LogicalPlan::Dml(_)| LogicalPlan::Copy(_)- | LogicalPlan::DescribeTable(_)+ | LogicalPlan::Show(_)| LogicalPlan::Statement(_) => logical_plan,// For any other plan, convert to Substraitlogical_plan => {diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex 2ec9dece06..c7939b29d8 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -10,8 +10,12 @@ SELECT * from information_schema.tablesstatement error DataFusion error: Error during planning: table 'datafusion.information_schema.schemata' not foundSELECT * from information_schema.schemata-statement error DataFusion error: Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled-show all+# SHOW resolves its content directly (like DESCRIBE/SHOW COLUMNS), so it+# works regardless of whether information_schema is enabled.+query TT+SHOW datafusion.catalog.information_schema+----+datafusion.catalog.information_schema false# Turn it on@@ -200,7 +204,7 @@ datafusion.catalog.has_header truedatafusion.catalog.information_schema truedatafusion.catalog.location NULLdatafusion.catalog.newlines_in_values false-datafusion.catalog.system_catalog NULL+datafusion.catalog.partition_information_schema falsedatafusion.execution.batch_size 8192datafusion.execution.coalesce_batches truedatafusion.execution.collect_statistics true@@ -366,7 +370,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CRdatafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema informationdatafusion.catalog.location NULL Location scanned to load tables for `default` schemadatafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance.-datafusion.catalog.system_catalog NULL The system catalog name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option.+datafusion.catalog.partition_information_schema false When set together with `information_schema`, the `information_schema` of each catalog only describes the objects in that catalog. When not set, the `information_schema` of each catalog describes all catalogs.datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumptiondatafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration settingdatafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true.@@ -635,17 +639,22 @@ datafusion information_schema views VIEWquery TTEXPLAIN SHOW TABLES-----logical_plan TableScan: information_schema.tables projection=[table_catalog, table_schema, table_name, table_type]+logical_plan ShowTablesphysical_plan StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, table_type]# information_schema_show_tables_no_information_schema+#+# SHOW TABLES reuses the information_schema.tables machinery directly, so it+# works regardless of whether information_schema is enabled; it just won't+# list the information_schema tables themselves when disabled.statement okset datafusion.catalog.information_schema = false;-statement error Error during planning: SHOW TABLES is not supported unless information_schema is enabled+query TTTT rowsortSHOW TABLES+----statement okset datafusion.catalog.information_schema = true;@@ -683,12 +692,17 @@ SHOW EXTENDED COLUMNS FROM t;datafusion public t i 0 NULL NO Int32 NULL NULL 32 2 NULL NULL NULL# information_schema_show_columns_no_information_schema+#+# SHOW COLUMNS resolves the table directly (like DESCRIBE), so it works+# regardless of whether information_schema is enabled.statement okset datafusion.catalog.information_schema = false;-statement error Error during planning: SHOW COLUMNS is not supported unless information_schema is enabled+query TTTTTTSHOW COLUMNS FROM t+----+datafusion public t i Int32 NOstatement okset datafusion.catalog.information_schema = true;@@ -719,17 +733,26 @@ SHOW SOMETHING_UNKNOWN;statement okDROP TABLE t;-# show_unsupported_when_information_schema_off+# show_variable_when_information_schema_off+#+# SHOW resolves its content directly, so it still works (and still rejects+# unknown variable names) regardless of whether information_schema is+# enabled.statement okset datafusion.catalog.information_schema = false;-statement error Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled+statement error DataFusion error: Error during planning: 'something' is not a variable which can be viewed with 'SHOW'SHOW SOMETHING-statement error Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled+statement error DataFusion error: Error during planning: 'something' is not a variable which can be viewed with 'SHOW'SHOW SOMETHING VERBOSE+query TT+SHOW datafusion.catalog.information_schema+----+datafusion.catalog.information_schema false+statement okset datafusion.catalog.information_schema = true;diff --git a/datafusion/sqllogictest/test_files/information_schema_columns.slt b/datafusion/sqllogictest/test_files/information_schema_columns.sltindex 043c6bf41f..240f6fe955 100644--- a/datafusion/sqllogictest/test_files/information_schema_columns.slt+++ b/datafusion/sqllogictest/test_files/information_schema_columns.slt@@ -32,10 +32,35 @@ my_catalog my_schema table_with_many_types large_utf8_col 3 NULL NO LargeUtf8 NUmy_catalog my_schema table_with_many_types timestamp_nanos 6 NULL NO Timestamp(ns) NULL NULL NULL NULL NULL NULL NULLmy_catalog my_schema table_with_many_types utf8_col 2 NULL YES Utf8 NULL 2147483647 NULL NULL NULL NULL NULL+###########+# numeric_precision, numeric_precision_radix and numeric_scale for the+# remaining numeric types+###########++statement ok+CREATE TABLE t2 AS SELECT+ arrow_cast(1, 'Int8') AS int8_col,+ arrow_cast(1, 'Int16') AS int16_col,+ arrow_cast(1.0, 'Float32') AS float32_col,+ arrow_cast(1.5, 'Decimal128(10, 2)') AS decimal128_col,+ arrow_cast(1.5, 'Decimal256(40, 5)') AS decimal256_col;++query TTTTITTTIIIIIIT rowsort+SELECT * from information_schema.columns WHERE table_name = 't2';+----+my_catalog my_schema t2 decimal128_col 3 NULL NO Decimal128(10, 2) NULL NULL 10 10 2 NULL NULL+my_catalog my_schema t2 decimal256_col 4 NULL NO Decimal256(40, 5) NULL NULL NULL NULL NULL NULL NULL+my_catalog my_schema t2 float32_col 2 NULL NO Float32 NULL NULL 24 2 NULL NULL NULL+my_catalog my_schema t2 int16_col 1 NULL NO Int16 NULL NULL 16 2 NULL NULL NULL+my_catalog my_schema t2 int8_col 0 NULL NO Int8 NULL NULL 8 2 NULL NULL NULL+# Cleanupstatement okdrop table t1+statement ok+drop table t2+statement okdrop table table_with_many_typesdiff --git a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.sltindex 8efc053d42..38b0481c98 100644--- a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt+++ b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt@@ -6,8 +6,12 @@statement error DataFusion error: Error during planning: table 'datafusion.information_schema.tables' not foundSELECT * from information_schema.tables-statement error DataFusion error: Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled-show all+# SHOW resolves its content directly (like DESCRIBE/SHOW COLUMNS), so it+# works regardless of whether information_schema is enabled.+query TT+SHOW datafusion.catalog.information_schema+----+datafusion.catalog.information_schema false# Turn it on@@ -110,14 +114,11 @@ my_other_catalog information_schema views VIEWmy_other_catalog my_other_schema t3 BASE TABLE############-# Configure a system catalog-#-# When `datafusion.catalog.system_catalog` is set, the `information_schema`-# of each catalog only describes the objects in that catalog. Only the-# `information_schema` of the system catalog describes all catalogs.+# Enable information schema partitioning so that the `information_schema`+# of each catalog only describes the objects in that catalog.###########statement ok-set datafusion.catalog.system_catalog = system;+set datafusion.catalog.partition_information_schema = true;query TTTT rowsortSELECT * FROM my_catalog.information_schema.tables;@@ -168,41 +169,6 @@ my_other_catalog information_schema tables VIEWmy_other_catalog information_schema views VIEWmy_other_catalog my_other_schema t3 BASE TABLE-query TTTT rowsort-SELECT * FROM system.information_schema.tables;------datafusion information_schema columns VIEW-datafusion information_schema df_settings VIEW-datafusion information_schema parameters VIEW-datafusion information_schema routines VIEW-datafusion information_schema schemata VIEW-datafusion information_schema tables VIEW-datafusion information_schema views VIEW-my_catalog information_schema columns VIEW-my_catalog information_schema df_settings VIEW-my_catalog information_schema parameters VIEW-my_catalog information_schema routines VIEW-my_catalog information_schema schemata VIEW-my_catalog information_schema tables VIEW-my_catalog information_schema views VIEW-my_catalog my_schema t1 BASE TABLE-my_catalog my_schema t2 BASE TABLE-my_other_catalog information_schema columns VIEW-my_other_catalog information_schema df_settings VIEW-my_other_catalog information_schema parameters VIEW-my_other_catalog information_schema routines VIEW-my_other_catalog information_schema schemata VIEW-my_other_catalog information_schema tables VIEW-my_other_catalog information_schema views VIEW-my_other_catalog my_other_schema t3 BASE TABLE-system information_schema columns VIEW-system information_schema df_settings VIEW-system information_schema parameters VIEW-system information_schema routines VIEW-system information_schema schemata VIEW-system information_schema tables VIEW-system information_schema views VIEW-query TTTTTTT rowsortSELECT * FROM my_catalog.information_schema.schemata;----@@ -213,13 +179,6 @@ SELECT * FROM my_other_catalog.information_schema.schemata;----my_other_catalog my_other_schema NULL NULL NULL NULL NULL-query TTTTTTT rowsort-SELECT * FROM system.information_schema.schemata;------datafusion public NULL NULL NULL NULL NULL-my_catalog my_schema NULL NULL NULL NULL NULL-my_other_catalog my_other_schema NULL NULL NULL NULL NULL-query TTTT rowsortSELECT table_catalog, table_schema, table_name, column_nameFROM my_catalog.information_schema.columns;@@ -233,14 +192,6 @@ FROM my_other_catalog.information_schema.columns;----my_other_catalog my_other_schema t3 column1-query TTTT rowsort-SELECT table_catalog, table_schema, table_name, column_name-FROM system.information_schema.columns;------my_catalog my_schema t1 column1-my_catalog my_schema t2 column1-my_other_catalog my_other_schema t3 column1-query TTT rowsortSELECT table_catalog, table_schema, table_nameFROM my_catalog.information_schema.views@@ -249,16 +200,7 @@ WHERE table_schema <> 'information_schema';my_catalog my_schema t1my_catalog my_schema t2-query TTT rowsort-SELECT table_catalog, table_schema, table_name-FROM system.information_schema.views-WHERE table_schema <> 'information_schema';------my_catalog my_schema t1-my_catalog my_schema t2-my_other_catalog my_other_schema t3--# SHOW statements use the system catalog's information schema+# SHOW statements are not affected by partition_information_schema and describe all catalogsquery TTTT rowsortSHOW TABLES;----@@ -286,13 +228,6 @@ my_other_catalog information_schema schemata VIEWmy_other_catalog information_schema tables VIEWmy_other_catalog information_schema views VIEWmy_other_catalog my_other_schema t3 BASE TABLE-system information_schema columns VIEW-system information_schema df_settings VIEW-system information_schema parameters VIEW-system information_schema routines VIEW-system information_schema schemata VIEW-system information_schema tables VIEW-system information_schema views VIEWquery TTTTTTSHOW COLUMNS FROM my_catalog.my_schema.t1;@@ -303,30 +238,9 @@ my_catalog my_schema t1 column1 Int64 NOstatement error DataFusion error: Error during planning: table 'no_such_catalog.information_schema.tables' not foundSELECT * FROM no_such_catalog.information_schema.tables;-# A system catalog can be an existing catalog. Its information schema then-# describes all catalogs-statement ok-set datafusion.catalog.system_catalog = my_catalog;--query TTT rowsort-SELECT table_catalog, table_schema, table_name-FROM my_catalog.information_schema.tables-WHERE table_schema <> 'information_schema';------my_catalog my_schema t1-my_catalog my_schema t2-my_other_catalog my_other_schema t3--query TTT rowsort-SELECT table_catalog, table_schema, table_name-FROM my_other_catalog.information_schema.tables-WHERE table_schema <> 'information_schema';------my_other_catalog my_other_schema t3--# Resetting the system catalog makes every information schema global again+# Resetting partition_information_schema makes every information schema global againstatement ok-RESET datafusion.catalog.system_catalog;+RESET datafusion.catalog.partition_information_schema;query TTT rowsortSELECT table_catalog, table_schema, table_name@@ -337,11 +251,6 @@ my_catalog my_schema t1my_catalog my_schema t2my_other_catalog my_other_schema t3-# Without a system catalog, the information schema of a catalog that does not-# exist cannot be queried either-statement error DataFusion error: Error during planning: table 'no_such_catalog.information_schema.tables' not found-SELECT * FROM no_such_catalog.information_schema.tables;-# Cleanupstatement okdiff --git a/datafusion/sqllogictest/test_files/show_functions.slt b/datafusion/sqllogictest/test_files/show_functions.sltnew file mode 100644index 0000000000..ee06a88a12--- /dev/null+++ b/datafusion/sqllogictest/test_files/show_functions.slt@@ -0,0 +1,34 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++# SHOW FUNCTIONS fetches it source data directly so it does not require information_schema.routines/.parameters to+# be present.+statement ok+set datafusion.catalog.information_schema = false;++# A LIKE filter that doesn't match any function returns the empty set+query TT??TTT+SHOW FUNCTIONS LIKE 'this_function_does_not_exist'+----++# Unsupported (non-LIKE) filter kinds are rejected+statement error DataFusion error: Error during planning: Unsupported SHOW FUNCTIONS filter+SHOW FUNCTIONS WHERE function_name = 'abs'++# A function with multiple overloads should report all signatures+query TT??TTT rowsort+SHOW FUNCTIONS LIKE 'string_agg'+----+string_agg String [expression, delimiter] [String, Null] AGGREGATE Concatenates the values of string expressions and places separator values between them. If ordering is required, strings are concatenated in the specified order. This aggregation function can only mix DISTINCT and ORDER BY if the ordering expression is exactly the same as the first argument expression. string_agg([DISTINCT] expression, delimiter [ORDER BY expression])+string_agg String [expression, delimiter] [String, String] AGGREGATE Concatenates the values of string expressions and places separator values between them. If ordering is required, strings are concatenated in the specified order. This aggregation function can only mix DISTINCT and ORDER BY if the ordering expression is exactly the same as the first argument expression. string_agg([DISTINCT] expression, delimiter [ORDER BY expression])++# Functions without parameters should be reported+query TT??TTT rowsort+SHOW FUNCTIONS LIKE 'pi'+----+pi NULL [] [] SCALAR Returns an approximate value of π. pi()++# Config reset+statement ok+reset datafusion.catalog.information_schema;diff --git a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs b/datafusion/substrait/src/logical_plan/producer/rel/mod.rsindex 197ed27a69..7b7a4cdde2 100644--- a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs+++ b/datafusion/substrait/src/logical_plan/producer/rel/mod.rs@@ -57,9 +57,7 @@ pub fn to_substrait_rel(LogicalPlan::Dml(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?,LogicalPlan::Ddl(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?,LogicalPlan::Copy(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?,- LogicalPlan::DescribeTable(plan) => {- not_impl_err!("Unsupported plan type: {plan:?}")?- }+ LogicalPlan::Show(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?,LogicalPlan::Unnest(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?,LogicalPlan::RecursiveQuery(plan) => {not_impl_err!("Unsupported plan type: {plan:?}")?diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex ba87076b3d..45899c2759 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -1242,6 +1242,23 @@ See [issue #24625](https://github.com/apache/datafusion/issues/24625) for detail`MaterializedCteScan::find_in` returns the scans of one CTE in a plan, also in subquery expressions.+### `LogicalPlan::DescribeTable` is replaced by `LogicalPlan::Show`++`SHOW TABLES`, `SHOW COLUMNS`, `SHOW CREATE TABLE`, `SHOW FUNCTIONS`, `SHOW <variable>` and `DESCRIBE` are now all+planned as the new `LogicalPlan::Show` variant `LogicalPlan::DescribeTable` and the `DescribeTable` struct are+removed.++**Migration guide:**++```rust+// 55.0.0+LogicalPlan::DescribeTable(DescribeTable { schema, output_schema }) => { ... }++// 56.0.0+LogicalPlan::Show(Show { kind: ShowKind::Describe { schema }, output_schema }) => { ... }+LogicalPlan::Show(_) => { ... } // other SHOW statements+```+### Extension `PhysicalExpr`s can decode by name instead of through a codec`PhysicalExtensionExprNode` has a new optional `expr_name` field. An extension expression that sets it is decoded by a decoder registered on the session, not by a `PhysicalExtensionCodec`. This is the same mechanism extension `ExecutionPlan`s use (see above), and it uses the same `ProtoDecoderRegistry`.diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.mdindex ca81398508..5029e0ac80 100644--- a/docs/source/user-guide/configs.md+++ b/docs/source/user-guide/configs.md@@ -69,7 +69,7 @@ The following configuration settings are available:| datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified || datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified || datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information |-| datafusion.catalog.system_catalog | NULL | The system catalog name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. |+| datafusion.catalog.partition_information_schema | false | When set together with `information_schema`, the `information_schema` of each catalog only describes the objects in that catalog. When not set, the `information_schema` of each catalog describes all catalogs. || datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema || datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema || datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. |