Apache DataFusion SQL Query Engine
feat(sql): Postgres-style `EXPLAIN (...)` option list (#21768)
Adrian Garcia Badaraccod5643aed9e1e
diff --git a/datafusion/common/src/format.rs b/datafusion/common/src/format.rsindex a6bd42be69..ea88eca4a6 100644--- a/datafusion/common/src/format.rs+++ b/datafusion/common/src/format.rs@@ -23,6 +23,8 @@ use arrow::util::display::{DurationFormat, FormatOptions};use crate::config::{ConfigField, Visit};use crate::error::{DataFusionError, Result};+#[cfg(feature = "sql")]+use sqlparser::ast::{Expr, UtilityOption, Value, ValueWithSpan};/// The default [`FormatOptions`] to use within DataFusion/// Also see [`crate::config::FormatOptions`]@@ -430,3 +432,470 @@ impl ConfigField for ExplainAnalyzeCategories {Ok(())}}++/// Normalized options for a single `EXPLAIN` statement.+///+/// This collects the knobs that can be set per-statement from either the+/// legacy keyword form (`EXPLAIN ANALYZE VERBOSE FORMAT tree ...`) or the+/// Postgres-style `EXPLAIN (option [arg], ...) ...` form supported on+/// dialects whose+/// [`Dialect::supports_explain_with_utility_options`](https://docs.rs/sqlparser/latest/sqlparser/dialect/trait.Dialect.html#method.supports_explain_with_utility_options)+/// returns `true`.+///+/// Fields that are `None` / `false` mean "not set at the statement level" —+/// the physical planner falls back to the corresponding session config+/// value.+#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]+pub struct ExplainStatementOptions {+ /// Whether to actually execute the plan and gather metrics.+ ///+ /// Corresponds to the `ANALYZE` keyword or the `ANALYZE` option.+ pub analyze: bool,+ /// Whether to include extra detail in the output.+ ///+ /// Corresponds to the `VERBOSE` keyword or the `VERBOSE` option.+ pub verbose: bool,+ /// Output format for the plan. When `None`, the session-config+ /// default (`datafusion.explain.format`) is used.+ pub format: Option<ExplainFormat>,+ /// Override for [`MetricType`] (summary / dev) when running+ /// `EXPLAIN ANALYZE`.+ pub analyze_level: Option<MetricType>,+ /// Override for [`ExplainAnalyzeCategories`] (rows / bytes / timing+ /// / uncategorized) when running `EXPLAIN ANALYZE`.+ pub analyze_categories: Option<ExplainAnalyzeCategories>,+ /// Override for `datafusion.explain.show_statistics`.+ pub show_statistics: Option<bool>,+}++#[cfg(feature = "sql")]+impl ExplainStatementOptions {+ /// Parse a list of [`UtilityOption`] values (produced by sqlparser's+ /// `parse_utility_options`) into a normalized [`ExplainStatementOptions`].+ ///+ /// Argument grammar accepted:+ /// - `OPTION` — bare, implies `TRUE` for boolean options.+ /// - `OPTION TRUE` / `OPTION FALSE`+ /// - `OPTION ON` / `OPTION OFF`+ /// - `OPTION 1` / `OPTION 0`+ /// - `OPTION <ident>` or `OPTION '<string>'` for format / level / metrics.+ ///+ /// Options recognized by DataFusion are: `ANALYZE`, `VERBOSE`, `FORMAT`,+ /// `METRICS`, `LEVEL`, `TIMING`, `SUMMARY`, `COSTS`.+ ///+ /// Postgres-only options (`BUFFERS`, `WAL`, `SETTINGS`, `GENERIC_PLAN`,+ /// `MEMORY`) return a helpful "not supported" error. Any other option+ /// name produces an `unknown EXPLAIN option` error.+ pub fn from_utility_options(opts: &[UtilityOption]) -> Result<Self> {+ let mut out = ExplainStatementOptions::default();+ // Track whether METRICS was explicitly set so TIMING can merge+ // into it rather than overwrite.+ let mut metrics_explicit = false;++ for opt in opts {+ let name = opt.name.value.to_ascii_lowercase();+ match name.as_str() {+ "analyze" => {+ out.analyze = parse_bool_arg(&opt.arg, &name)?;+ }+ "verbose" => {+ out.verbose = parse_bool_arg(&opt.arg, &name)?;+ }+ "format" => {+ let s = parse_ident_or_string_arg(&opt.arg, &name)?;+ out.format = Some(ExplainFormat::from_str(&s)?);+ }+ "metrics" => {+ let s = parse_ident_or_string_arg(&opt.arg, &name)?;+ out.analyze_categories =+ Some(ExplainAnalyzeCategories::from_str(&s)?);+ metrics_explicit = true;+ }+ "level" => {+ let s = parse_ident_or_string_arg(&opt.arg, &name)?;+ out.analyze_level = Some(MetricType::from_str(&s)?);+ }+ "timing" => {+ let enable = parse_bool_arg(&opt.arg, &name)?;+ out.analyze_categories = Some(adjust_timing(+ out.analyze_categories.take(),+ enable,+ metrics_explicit,+ ));+ }+ "summary" => {+ let summary = parse_bool_arg(&opt.arg, &name)?;+ out.analyze_level = Some(if summary {+ MetricType::Summary+ } else {+ MetricType::Dev+ });+ }+ "costs" => {+ out.show_statistics = Some(parse_bool_arg(&opt.arg, &name)?);+ }+ // Postgres options DataFusion does not model. Give a helpful+ // pointer rather than silently accepting them.+ "buffers" | "wal" | "settings" | "generic_plan" | "memory" => {+ let upper = name.to_ascii_uppercase();+ return Err(DataFusionError::NotImplemented(format!(+ "EXPLAIN option {upper} is not supported by DataFusion; \+ see METRICS for category filtering"+ )));+ }+ _ => {+ return Err(DataFusionError::Plan(format!(+ "unknown EXPLAIN option: {}",+ opt.name.value+ )));+ }+ }+ }++ Ok(out)+ }+}++/// Parse a boolean argument for an EXPLAIN option.+///+/// `None` (bare option, e.g. `ANALYZE`) is treated as `true`. Accepts+/// identifiers `TRUE`/`FALSE`/`ON`/`OFF` (case-insensitive) and the numeric+/// literals `0` / `1`.+#[cfg(feature = "sql")]+fn parse_bool_arg(arg: &Option<Expr>, name: &str) -> Result<bool> {+ let Some(expr) = arg else {+ return Ok(true);+ };+ match expr {+ Expr::Identifier(ident) => match ident.value.to_ascii_lowercase().as_str() {+ "true" | "on" => Ok(true),+ "false" | "off" => Ok(false),+ other => Err(DataFusionError::Plan(format!(+ "expected boolean for EXPLAIN option {name}, got '{other}'"+ ))),+ },+ Expr::Value(ValueWithSpan { value, .. }) => match value {+ Value::Boolean(b) => Ok(*b),+ Value::Number(n, _) => match n.as_str() {+ "0" => Ok(false),+ "1" => Ok(true),+ other => Err(DataFusionError::Plan(format!(+ "expected boolean (0 or 1) for EXPLAIN option {name}, got '{other}'"+ ))),+ },+ Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => {+ match s.to_ascii_lowercase().as_str() {+ "true" | "on" | "1" => Ok(true),+ "false" | "off" | "0" => Ok(false),+ other => Err(DataFusionError::Plan(format!(+ "expected boolean for EXPLAIN option {name}, got '{other}'"+ ))),+ }+ }+ other => Err(DataFusionError::Plan(format!(+ "expected boolean for EXPLAIN option {name}, got '{other}'"+ ))),+ },+ other => Err(DataFusionError::Plan(format!(+ "expected boolean for EXPLAIN option {name}, got '{other}'"+ ))),+ }+}++/// Parse an identifier-or-string argument (used for `FORMAT`, `METRICS`,+/// `LEVEL`).+#[cfg(feature = "sql")]+fn parse_ident_or_string_arg(arg: &Option<Expr>, name: &str) -> Result<String> {+ let expr = arg.as_ref().ok_or_else(|| {+ DataFusionError::Plan(format!(+ "EXPLAIN option {} requires an argument",+ name.to_ascii_uppercase()+ ))+ })?;+ match expr {+ Expr::Identifier(ident) => Ok(ident.value.clone()),+ Expr::Value(ValueWithSpan { value, .. }) => match value {+ Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => Ok(s.clone()),+ other => Err(DataFusionError::Plan(format!(+ "expected identifier or string for EXPLAIN option {name}, got '{other}'"+ ))),+ },+ other => Err(DataFusionError::Plan(format!(+ "expected identifier or string for EXPLAIN option {name}, got '{other}'"+ ))),+ }+}++/// Merge a `TIMING on/off` option into an existing `METRICS` selection.+///+/// If METRICS was already specified, we only add/remove the Timing category+/// within that selection. If METRICS was not specified, TIMING effectively+/// means "Only(Timing)" when on, or "show everything except timing" when off.+#[cfg(feature = "sql")]+fn adjust_timing(+ current: Option<ExplainAnalyzeCategories>,+ enable: bool,+ metrics_explicit: bool,+) -> ExplainAnalyzeCategories {+ // METRICS was not specified — TIMING alone shapes the selection.+ if !metrics_explicit {+ return if enable {+ ExplainAnalyzeCategories::All+ } else {+ ExplainAnalyzeCategories::Only(vec![+ MetricCategory::Rows,+ MetricCategory::Bytes,+ MetricCategory::Uncategorized,+ ])+ };+ }++ // METRICS was specified explicitly earlier — merge into its list. When+ // METRICS was explicit, `current` is always `Some(_)`; fall back to All+ // to be safe.+ match current.unwrap_or(ExplainAnalyzeCategories::All) {+ ExplainAnalyzeCategories::All if enable => ExplainAnalyzeCategories::All,+ ExplainAnalyzeCategories::All => {+ // Everything except timing: rows, bytes, uncategorized.+ ExplainAnalyzeCategories::Only(vec![+ MetricCategory::Rows,+ MetricCategory::Bytes,+ MetricCategory::Uncategorized,+ ])+ }+ ExplainAnalyzeCategories::Only(mut cats) if enable => {+ if !cats.contains(&MetricCategory::Timing) {+ cats.push(MetricCategory::Timing);+ }+ ExplainAnalyzeCategories::Only(cats)+ }+ ExplainAnalyzeCategories::Only(cats) => ExplainAnalyzeCategories::Only(+ cats.into_iter()+ .filter(|c| *c != MetricCategory::Timing)+ .collect(),+ ),+ }+}++#[cfg(all(test, feature = "sql"))]+mod explain_options_tests {+ use super::*;+ use sqlparser::ast::Ident;+ use sqlparser::tokenizer::Span;++ fn bare(name: &str) -> UtilityOption {+ UtilityOption {+ name: Ident {+ value: name.to_string(),+ quote_style: None,+ span: Span::empty(),+ },+ arg: None,+ }+ }++ fn with_ident_arg(name: &str, arg: &str) -> UtilityOption {+ UtilityOption {+ name: Ident {+ value: name.to_string(),+ quote_style: None,+ span: Span::empty(),+ },+ arg: Some(Expr::Identifier(Ident {+ value: arg.to_string(),+ quote_style: None,+ span: Span::empty(),+ })),+ }+ }++ fn with_string_arg(name: &str, arg: &str) -> UtilityOption {+ UtilityOption {+ name: Ident {+ value: name.to_string(),+ quote_style: None,+ span: Span::empty(),+ },+ arg: Some(Expr::Value(ValueWithSpan {+ value: Value::SingleQuotedString(arg.to_string()),+ span: Span::empty(),+ })),+ }+ }++ fn with_bool_arg(name: &str, b: bool) -> UtilityOption {+ UtilityOption {+ name: Ident {+ value: name.to_string(),+ quote_style: None,+ span: Span::empty(),+ },+ arg: Some(Expr::Value(ValueWithSpan {+ value: Value::Boolean(b),+ span: Span::empty(),+ })),+ }+ }++ fn with_number_arg(name: &str, n: &str) -> UtilityOption {+ UtilityOption {+ name: Ident {+ value: name.to_string(),+ quote_style: None,+ span: Span::empty(),+ },+ arg: Some(Expr::Value(ValueWithSpan {+ value: Value::Number(n.to_string(), false),+ span: Span::empty(),+ })),+ }+ }++ #[test]+ fn bare_analyze_and_verbose() {+ let opts = ExplainStatementOptions::from_utility_options(&[+ bare("ANALYZE"),+ bare("VERBOSE"),+ ])+ .unwrap();+ assert!(opts.analyze);+ assert!(opts.verbose);+ assert!(opts.format.is_none());+ }++ #[test]+ fn format_from_ident_and_string() {+ let opts = ExplainStatementOptions::from_utility_options(&[with_ident_arg(+ "FORMAT", "tree",+ )])+ .unwrap();+ assert_eq!(opts.format, Some(ExplainFormat::Tree));++ let opts = ExplainStatementOptions::from_utility_options(&[with_string_arg(+ "FORMAT", "pgjson",+ )])+ .unwrap();+ assert_eq!(opts.format, Some(ExplainFormat::PostgresJSON));+ }++ #[test]+ fn metrics_and_level() {+ let opts = ExplainStatementOptions::from_utility_options(&[+ with_string_arg("METRICS", "rows,bytes"),+ with_ident_arg("LEVEL", "dev"),+ ])+ .unwrap();+ assert_eq!(+ opts.analyze_categories,+ Some(ExplainAnalyzeCategories::Only(vec![+ MetricCategory::Rows,+ MetricCategory::Bytes,+ ]))+ );+ assert_eq!(opts.analyze_level, Some(MetricType::Dev));+ }++ #[test]+ fn on_off_numeric_bool() {+ let opts = ExplainStatementOptions::from_utility_options(&[+ with_ident_arg("ANALYZE", "ON"),+ with_ident_arg("VERBOSE", "off"),+ with_bool_arg("COSTS", true),+ ])+ .unwrap();+ assert!(opts.analyze);+ assert!(!opts.verbose);+ assert_eq!(opts.show_statistics, Some(true));++ let opts = ExplainStatementOptions::from_utility_options(&[+ with_number_arg("ANALYZE", "1"),+ with_number_arg("VERBOSE", "0"),+ ])+ .unwrap();+ assert!(opts.analyze);+ assert!(!opts.verbose);+ }++ #[test]+ fn summary_sugar_sets_level() {+ let opts = ExplainStatementOptions::from_utility_options(&[with_ident_arg(+ "SUMMARY", "ON",+ )])+ .unwrap();+ assert_eq!(opts.analyze_level, Some(MetricType::Summary));++ let opts = ExplainStatementOptions::from_utility_options(&[with_bool_arg(+ "SUMMARY", false,+ )])+ .unwrap();+ assert_eq!(opts.analyze_level, Some(MetricType::Dev));+ }++ #[test]+ fn timing_merges_with_metrics() {+ // METRICS then TIMING off → timing is removed from the list+ let opts = ExplainStatementOptions::from_utility_options(&[+ with_string_arg("METRICS", "rows,timing"),+ with_bool_arg("TIMING", false),+ ])+ .unwrap();+ assert_eq!(+ opts.analyze_categories,+ Some(ExplainAnalyzeCategories::Only(vec![MetricCategory::Rows]))+ );++ // METRICS 'rows' then TIMING on → timing is appended+ let opts = ExplainStatementOptions::from_utility_options(&[+ with_string_arg("METRICS", "rows"),+ with_bool_arg("TIMING", true),+ ])+ .unwrap();+ assert_eq!(+ opts.analyze_categories,+ Some(ExplainAnalyzeCategories::Only(vec![+ MetricCategory::Rows,+ MetricCategory::Timing,+ ]))+ );+ }++ #[test]+ fn timing_alone() {+ let opts = ExplainStatementOptions::from_utility_options(&[with_bool_arg(+ "TIMING", false,+ )])+ .unwrap();+ assert_eq!(+ opts.analyze_categories,+ Some(ExplainAnalyzeCategories::Only(vec![+ MetricCategory::Rows,+ MetricCategory::Bytes,+ MetricCategory::Uncategorized,+ ]))+ );+ }++ #[test]+ fn unknown_option_rejected() {+ let err =+ ExplainStatementOptions::from_utility_options(&[bare("FOO")]).unwrap_err();+ assert!(+ err.to_string().contains("unknown EXPLAIN option: FOO"),+ "got: {err}"+ );+ }++ #[test]+ fn postgres_only_options_rejected() {+ for pg_only in ["BUFFERS", "WAL", "SETTINGS", "GENERIC_PLAN", "MEMORY"] {+ let err = ExplainStatementOptions::from_utility_options(&[bare(pg_only)])+ .unwrap_err();+ let msg = err.to_string();+ assert!(+ msg.contains(pg_only),+ "msg did not include {pg_only}: {msg}"+ );+ assert!(msg.contains("not supported"), "msg: {msg}");+ }+ }+}diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex de5e6b97c1..c4c6f1889b 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -691,6 +691,7 @@ impl SessionState {stringified_plans,schema: Arc::clone(&e.schema),logical_optimization_succeeded: false,+ show_statistics: e.show_statistics,}));}Err(e) => return Err(e),@@ -728,6 +729,7 @@ impl SessionState {stringified_plans,schema: Arc::clone(&e.schema),logical_optimization_succeeded,+ show_statistics: e.show_statistics,}))} else {let analyzed_plan = self.analyzer.execute_and_check(diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex a00d07a09f..e5e1b34642 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -2597,6 +2597,8 @@ impl DefaultPhysicalPlanner {let config = &session_state.config_options().explain;let explain_format = &e.explain_format;+ // Statement-level override wins over session config for show_statistics.+ let show_statistics = e.show_statistics.unwrap_or(config.show_statistics);if !e.logical_optimization_succeeded {return Ok(Arc::new(ExplainExec::new(@@ -2669,7 +2671,7 @@ impl DefaultPhysicalPlanner {stringified_plans.push(StringifiedPlan::new(InitialPhysicalPlan,displayable(input.as_ref())- .set_show_statistics(config.show_statistics)+ .set_show_statistics(show_statistics).set_show_schema(config.show_schema).indent(e.verbose).to_string(),@@ -2678,7 +2680,7 @@ impl DefaultPhysicalPlanner {// Show statistics + schema in verbose output even if not// explicitly requestedif e.verbose {- if !config.show_statistics {+ if !show_statistics {stringified_plans.push(StringifiedPlan::new(InitialPhysicalPlanWithStats,displayable(input.as_ref())@@ -2707,7 +2709,7 @@ impl DefaultPhysicalPlanner {stringified_plans.push(StringifiedPlan::new(plan_type,displayable(plan)- .set_show_statistics(config.show_statistics)+ .set_show_statistics(show_statistics).set_show_schema(config.show_schema).indent(e.verbose).to_string(),@@ -2720,7 +2722,7 @@ impl DefaultPhysicalPlanner {stringified_plans.push(StringifiedPlan::new(FinalPhysicalPlan,displayable(input.as_ref())- .set_show_statistics(config.show_statistics)+ .set_show_statistics(show_statistics).set_show_schema(config.show_schema).indent(e.verbose).to_string(),@@ -2729,7 +2731,7 @@ impl DefaultPhysicalPlanner {// Show statistics + schema in verbose output even if not// explicitly requestedif e.verbose {- if !config.show_statistics {+ if !show_statistics {stringified_plans.push(StringifiedPlan::new(FinalPhysicalPlanWithStats,displayable(input.as_ref())@@ -2783,13 +2785,18 @@ impl DefaultPhysicalPlanner {let input = self.create_physical_plan(&a.input, session_state).await?;let schema = Arc::clone(a.schema.inner());let show_statistics = session_state.config_options().explain.show_statistics;- let analyze_level = session_state.config_options().explain.analyze_level;+ // Statement-level overrides take precedence over the session config.+ let analyze_level = a+ .analyze_level+ .unwrap_or(session_state.config_options().explain.analyze_level);let metric_types = analyze_level.included_types();- let analyze_categories = session_state- .config_options()- .explain- .analyze_categories- .clone();+ let analyze_categories = a.analyze_categories.clone().unwrap_or_else(|| {+ session_state+ .config_options()+ .explain+ .analyze_categories+ .clone()+ });let metric_categories = match analyze_categories {ExplainAnalyzeCategories::All => None,ExplainAnalyzeCategories::Only(cats) => Some(cats),@@ -4312,6 +4319,7 @@ mod tests {stringified_plans,schema: schema.to_dfschema_ref().unwrap(),logical_optimization_succeeded: false,+ show_statistics: None,};let plan = planner.handle_explain(&explain, &ctx.state())diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex b093563d9a..17e3dba14b 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -1267,3 +1267,115 @@ async fn explain_analyze_categories() {);}}++/// Returns a [`SessionContext`] configured with the PostgreSQL dialect so+/// that `EXPLAIN (option, ...)` utility-option syntax is accepted.+fn session_ctx_with_pg_dialect() -> SessionContext {+ use std::str::FromStr;+ let mut config = SessionConfig::new();+ let options = config.options_mut();+ options.sql_parser.dialect =+ datafusion::config::Dialect::from_str("PostgreSQL").unwrap();+ SessionContext::new_with_config(config)+}++async fn collect_explain(ctx: &SessionContext, sql: &str) -> String {+ let dataframe = ctx.sql(sql).await.unwrap();+ let batches = dataframe.collect().await.unwrap();+ arrow::util::pretty::pretty_format_batches(&batches)+ .unwrap()+ .to_string()+}++/// Verifies that the Postgres-style `EXPLAIN (METRICS '...')` form produces+/// the same category filtering as `SET datafusion.explain.analyze_categories`.+#[tokio::test]+async fn explain_analyze_paren_metrics_filtering() {+ let ctx = session_ctx_with_pg_dialect();+ let sql = "EXPLAIN (ANALYZE, METRICS 'rows') \+ SELECT * FROM generate_series(10) as t1(v1) ORDER BY v1 DESC";+ let plan = collect_explain(&ctx, sql).await;+ assert!(+ plan.contains("output_rows"),+ "rows category should include output_rows:\n{plan}"+ );+ assert!(+ !plan.contains("elapsed_compute"),+ "rows-only METRICS should exclude elapsed_compute:\n{plan}"+ );+ assert!(+ !plan.contains("output_bytes"),+ "rows-only METRICS should exclude output_bytes:\n{plan}"+ );+}++/// Verifies that a statement-level METRICS overrides the session config.+#[tokio::test]+async fn explain_analyze_paren_metrics_overrides_session_config() {+ let ctx = session_ctx_with_pg_dialect();+ // Session default: show only `rows` via config.+ {+ let state = ctx.state_ref();+ let mut state = state.write();+ state.config_mut().options_mut().explain.analyze_categories =+ ExplainAnalyzeCategories::Only(vec![MetricCategory::Rows]);+ }+ // Statement overrides with 'bytes' — we should see output_bytes but not+ // output_rows (except row-count metrics with the `output_bytes` substring+ // are avoided because the metric names are distinct).+ let sql = "EXPLAIN (ANALYZE, METRICS 'bytes') \+ SELECT * FROM generate_series(10) as t1(v1) ORDER BY v1 DESC";+ let plan = collect_explain(&ctx, sql).await;+ assert!(+ plan.contains("output_bytes"),+ "statement-level METRICS='bytes' should show output_bytes:\n{plan}"+ );+ assert!(+ !plan.contains("output_rows"),+ "statement-level METRICS='bytes' should hide output_rows:\n{plan}"+ );+}++/// Verifies that `EXPLAIN (ANALYZE, LEVEL summary)` only shows summary metrics,+/// overriding the session default of `dev`.+#[tokio::test]+async fn explain_analyze_paren_level_overrides_session_config() {+ let ctx = session_ctx_with_pg_dialect();+ // Session default: Dev+ {+ let state = ctx.state_ref();+ let mut state = state.write();+ state.config_mut().options_mut().explain.analyze_level = MetricType::Dev;+ }+ let sql = "EXPLAIN (ANALYZE, LEVEL summary) \+ SELECT * FROM generate_series(10) as t1(v1) ORDER BY v1 DESC";+ let plan = collect_explain(&ctx, sql).await;+ // `spill_count` is Dev-only; `output_rows` is Summary.+ assert!(+ plan.contains("output_rows"),+ "summary should still show output_rows:\n{plan}"+ );+ assert!(+ !plan.contains("spill_count"),+ "summary should hide Dev-only spill_count:\n{plan}"+ );+}++/// Verifies that `EXPLAIN (ANALYZE, BUFFERS)` returns a helpful error.+#[tokio::test]+async fn explain_paren_buffers_rejected() {+ let ctx = session_ctx_with_pg_dialect();+ let err = ctx+ .sql("EXPLAIN (ANALYZE, BUFFERS) SELECT 1")+ .await+ .unwrap_err();+ let msg = err.to_string();+ assert!(+ msg.contains("BUFFERS"),+ "error should mention BUFFERS: {msg}"+ );+ assert!(+ msg.contains("not supported"),+ "error should say not supported: {msg}"+ );+}diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rsindex 7bc705e0f4..e107d233b6 100644--- a/datafusion/expr/src/logical_plan/builder.rs+++ b/datafusion/expr/src/logical_plan/builder.rs@@ -1337,6 +1337,8 @@ impl LogicalPlanBuilder {verbose: explain_option.verbose,input: self.plan,schema,+ analyze_level: explain_option.analyze_level,+ analyze_categories: explain_option.analyze_categories,})))} else {let stringified_plans =@@ -1349,6 +1351,7 @@ impl LogicalPlanBuilder {stringified_plans,schema,logical_optimization_succeeded: false,+ show_statistics: explain_option.show_statistics,})))}}diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex e7e03bcac5..0a953e759c 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -53,7 +53,7 @@ use crate::{use crate::statistics::StatisticsRequest;use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef};use datafusion_common::cse::{NormalizeEq, Normalizeable};-use datafusion_common::format::ExplainFormat;+use datafusion_common::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};use datafusion_common::metadata::check_metadata_with_storage_equal;use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion,@@ -1095,6 +1095,8 @@ impl LogicalPlan {verbose: a.verbose,schema: Arc::clone(&a.schema),input: Arc::new(input),+ analyze_level: a.analyze_level,+ analyze_categories: a.analyze_categories.clone(),}))}LogicalPlan::Explain(e) => {@@ -1107,6 +1109,7 @@ impl LogicalPlan {stringified_plans: e.stringified_plans.clone(),schema: Arc::clone(&e.schema),logical_optimization_succeeded: e.logical_optimization_succeeded,+ show_statistics: e.show_statistics,}))}LogicalPlan::Statement(Statement::Prepare(Prepare {@@ -3329,6 +3332,15 @@ pub struct ExplainOption {pub analyze: bool,/// Output syntax/formatpub format: ExplainFormat,+ /// Statement-level override for `datafusion.explain.show_statistics`.+ /// `None` means "fall back to session config".+ pub show_statistics: Option<bool>,+ /// Statement-level override for `datafusion.explain.analyze_level`.+ /// `None` means "fall back to session config".+ pub analyze_level: Option<MetricType>,+ /// Statement-level override for `datafusion.explain.analyze_categories`.+ /// `None` means "fall back to session config".+ pub analyze_categories: Option<ExplainAnalyzeCategories>,}impl Default for ExplainOption {@@ -3337,6 +3349,9 @@ impl Default for ExplainOption {verbose: false,analyze: false,format: ExplainFormat::Indent,+ show_statistics: None,+ analyze_level: None,+ analyze_categories: None,}}}@@ -3359,6 +3374,30 @@ impl ExplainOption {self.format = format;self}++ /// Builder-style setter for a statement-level override of+ /// `datafusion.explain.show_statistics`.+ pub fn with_show_statistics(mut self, show_statistics: Option<bool>) -> Self {+ self.show_statistics = show_statistics;+ self+ }++ /// Builder-style setter for a statement-level override of+ /// `datafusion.explain.analyze_level`.+ pub fn with_analyze_level(mut self, analyze_level: Option<MetricType>) -> Self {+ self.analyze_level = analyze_level;+ self+ }++ /// Builder-style setter for a statement-level override of+ /// `datafusion.explain.analyze_categories`.+ pub fn with_analyze_categories(+ mut self,+ analyze_categories: Option<ExplainAnalyzeCategories>,+ ) -> Self {+ self.analyze_categories = analyze_categories;+ self+ }}/// Produces a relation with string representations of@@ -3382,6 +3421,9 @@ pub struct Explain {pub schema: DFSchemaRef,/// Used by physical planner to check if should proceed with planningpub logical_optimization_succeeded: bool,+ /// Statement-level override for `datafusion.explain.show_statistics`.+ /// When `None`, the session-config value is used.+ pub show_statistics: Option<bool>,}// Manual implementation needed because of `schema` field. Comparison excludes this field.@@ -3397,18 +3439,22 @@ impl PartialOrd for Explain {pub stringified_plans: &'a Vec<StringifiedPlan>,/// Used by physical planner to check if should proceed with planningpub logical_optimization_succeeded: &'a bool,+ /// Statement-level override for show_statistics+ pub show_statistics: &'a Option<bool>,}let comparable_self = ComparableExplain {verbose: &self.verbose,plan: &self.plan,stringified_plans: &self.stringified_plans,logical_optimization_succeeded: &self.logical_optimization_succeeded,+ show_statistics: &self.show_statistics,};let comparable_other = ComparableExplain {verbose: &other.verbose,plan: &other.plan,stringified_plans: &other.stringified_plans,logical_optimization_succeeded: &other.logical_optimization_succeeded,+ show_statistics: &other.show_statistics,};comparable_self.partial_cmp(&comparable_other)@@ -3427,9 +3473,18 @@ pub struct Analyze {pub input: Arc<LogicalPlan>,/// The output schema of the explain (2 columns of text)pub schema: DFSchemaRef,+ /// Statement-level override for `datafusion.explain.analyze_level`.+ /// When `None`, the session-config value is used.+ pub analyze_level: Option<MetricType>,+ /// Statement-level override for `datafusion.explain.analyze_categories`.+ /// When `None`, the session-config value is used.+ pub analyze_categories: Option<ExplainAnalyzeCategories>,}-// Manual implementation needed because of `schema` field. Comparison excludes this field.+// Manual implementation needed because of `schema` field and the lack of+// `PartialOrd` on `MetricType` / `ExplainAnalyzeCategories`. Ordering is+// defined over `(verbose, input)` and then falls back to `==` for the+// remaining statement-level override fields.impl PartialOrd for Analyze {fn partial_cmp(&self, other: &Self) -> Option<Ordering> {match self.verbose.partial_cmp(&other.verbose) {diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rsindex 801caddcd0..98ac27aa2b 100644--- a/datafusion/expr/src/logical_plan/tree_node.rs+++ b/datafusion/expr/src/logical_plan/tree_node.rs@@ -203,6 +203,7 @@ impl TreeNode for LogicalPlan {stringified_plans,schema,logical_optimization_succeeded,+ show_statistics,}) => plan.map_elements(f)?.update_data(|plan| {LogicalPlan::Explain(Explain {verbose,@@ -211,17 +212,22 @@ impl TreeNode for LogicalPlan {stringified_plans,schema,logical_optimization_succeeded,+ show_statistics,})}),LogicalPlan::Analyze(Analyze {verbose,input,schema,+ analyze_level,+ analyze_categories,}) => input.map_elements(f)?.update_data(|input| {LogicalPlan::Analyze(Analyze {verbose,input,schema,+ analyze_level,+ analyze_categories,})}),LogicalPlan::Dml(DmlStatement {diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.protoindex 684d9a2612..26b400f879 100644--- a/datafusion/proto-common/proto/datafusion_common.proto+++ b/datafusion/proto-common/proto/datafusion_common.proto@@ -683,4 +683,30 @@ enum ExplainFormat {EXPLAIN_FORMAT_TREE = 1;EXPLAIN_FORMAT_PGJSON = 2;EXPLAIN_FORMAT_GRAPHVIZ = 3;+}++// Verbosity level for `EXPLAIN ANALYZE`. Mirrors+// `datafusion_common::format::MetricType`.+enum MetricType {+ METRIC_TYPE_SUMMARY = 0;+ METRIC_TYPE_DEV = 1;+}++// Category of an `EXPLAIN ANALYZE` metric. Mirrors+// `datafusion_common::format::MetricCategory`.+enum MetricCategory {+ METRIC_CATEGORY_ROWS = 0;+ METRIC_CATEGORY_BYTES = 1;+ METRIC_CATEGORY_TIMING = 2;+ METRIC_CATEGORY_UNCATEGORIZED = 3;+}++// Wire encoding for `datafusion_common::format::ExplainAnalyzeCategories`.+//+// If `all` is true, every category is shown (the `only` list is ignored).+// If `all` is false, only the categories listed in `only` are shown — an+// empty `only` means "plan only", i.e. suppress all metrics.+message ExplainAnalyzeCategoriesNode {+ bool all = 1;+ repeated MetricCategory only = 2;}\ No newline at end of filediff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rsindex 0568982e97..3139553d5e 100644--- a/datafusion/proto-common/src/generated/pbjson.rs+++ b/datafusion/proto-common/src/generated/pbjson.rs@@ -4116,6 +4116,118 @@ impl<'de> serde::Deserialize<'de> for EmptyMessage {deserializer.deserialize_struct("datafusion_common.EmptyMessage", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for ExplainAnalyzeCategoriesNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if self.all {+ len += 1;+ }+ if !self.only.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion_common.ExplainAnalyzeCategoriesNode", len)?;+ if self.all {+ struct_ser.serialize_field("all", &self.all)?;+ }+ if !self.only.is_empty() {+ let v = self.only.iter().cloned().map(|v| {+ MetricCategory::try_from(v)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", v)))+ }).collect::<std::result::Result<Vec<_>, _>>()?;+ struct_ser.serialize_field("only", &v)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for ExplainAnalyzeCategoriesNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "all",+ "only",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ All,+ Only,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "all" => Ok(GeneratedField::All),+ "only" => Ok(GeneratedField::Only),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = ExplainAnalyzeCategoriesNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion_common.ExplainAnalyzeCategoriesNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<ExplainAnalyzeCategoriesNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut all__ = None;+ let mut only__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::All => {+ if all__.is_some() {+ return Err(serde::de::Error::duplicate_field("all"));+ }+ all__ = Some(map_.next_value()?);+ }+ GeneratedField::Only => {+ if only__.is_some() {+ return Err(serde::de::Error::duplicate_field("only"));+ }+ only__ = Some(map_.next_value::<Vec<MetricCategory>>()?.into_iter().map(|x| x as i32).collect());+ }+ }+ }+ Ok(ExplainAnalyzeCategoriesNode {+ all: all__.unwrap_or_default(),+ only: only__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion_common.ExplainAnalyzeCategoriesNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for ExplainFormat {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -5474,6 +5586,154 @@ impl<'de> serde::Deserialize<'de> for Map {deserializer.deserialize_struct("datafusion_common.Map", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for MetricCategory {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ let variant = match self {+ Self::Rows => "METRIC_CATEGORY_ROWS",+ Self::Bytes => "METRIC_CATEGORY_BYTES",+ Self::Timing => "METRIC_CATEGORY_TIMING",+ Self::Uncategorized => "METRIC_CATEGORY_UNCATEGORIZED",+ };+ serializer.serialize_str(variant)+ }+}+impl<'de> serde::Deserialize<'de> for MetricCategory {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "METRIC_CATEGORY_ROWS",+ "METRIC_CATEGORY_BYTES",+ "METRIC_CATEGORY_TIMING",+ "METRIC_CATEGORY_UNCATEGORIZED",+ ];++ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = MetricCategory;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ fn visit_i64<E>(self, v: i64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self)+ })+ }++ fn visit_u64<E>(self, v: u64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self)+ })+ }++ fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ match value {+ "METRIC_CATEGORY_ROWS" => Ok(MetricCategory::Rows),+ "METRIC_CATEGORY_BYTES" => Ok(MetricCategory::Bytes),+ "METRIC_CATEGORY_TIMING" => Ok(MetricCategory::Timing),+ "METRIC_CATEGORY_UNCATEGORIZED" => Ok(MetricCategory::Uncategorized),+ _ => Err(serde::de::Error::unknown_variant(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_any(GeneratedVisitor)+ }+}+impl serde::Serialize for MetricType {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ let variant = match self {+ Self::Summary => "METRIC_TYPE_SUMMARY",+ Self::Dev => "METRIC_TYPE_DEV",+ };+ serializer.serialize_str(variant)+ }+}+impl<'de> serde::Deserialize<'de> for MetricType {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "METRIC_TYPE_SUMMARY",+ "METRIC_TYPE_DEV",+ ];++ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = MetricType;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ fn visit_i64<E>(self, v: i64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self)+ })+ }++ fn visit_u64<E>(self, v: u64) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ i32::try_from(v)+ .ok()+ .and_then(|x| x.try_into().ok())+ .ok_or_else(|| {+ serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self)+ })+ }++ fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>+ where+ E: serde::de::Error,+ {+ match value {+ "METRIC_TYPE_SUMMARY" => Ok(MetricType::Summary),+ "METRIC_TYPE_DEV" => Ok(MetricType::Dev),+ _ => Err(serde::de::Error::unknown_variant(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_any(GeneratedVisitor)+ }+}impl serde::Serialize for NdJsonFormat {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rsindex 632b16929f..1876102ea9 100644--- a/datafusion/proto-common/src/generated/prost.rs+++ b/datafusion/proto-common/src/generated/prost.rs@@ -1014,6 +1014,18 @@ pub struct ColumnStats {#[prost(message, optional, tag = "6")]pub byte_size: ::core::option::Option<Precision>,}+/// Wire encoding for `datafusion_common::format::ExplainAnalyzeCategories`.+///+/// If `all` is true, every category is shown (the `only` list is ignored).+/// If `all` is false, only the categories listed in `only` are shown — an+/// empty `only` means "plan only", i.e. suppress all metrics.+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]+pub struct ExplainAnalyzeCategoriesNode {+ #[prost(bool, tag = "1")]+ pub all: bool,+ #[prost(enumeration = "MetricCategory", repeated, tag = "2")]+ pub only: ::prost::alloc::vec::Vec<i32>,+}#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]#[repr(i32)]pub enum JoinType {@@ -1360,3 +1372,65 @@ impl ExplainFormat {}}}+/// Verbosity level for `EXPLAIN ANALYZE`. Mirrors+/// `datafusion_common::format::MetricType`.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]+#[repr(i32)]+pub enum MetricType {+ Summary = 0,+ Dev = 1,+}+impl MetricType {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::Summary => "METRIC_TYPE_SUMMARY",+ Self::Dev => "METRIC_TYPE_DEV",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "METRIC_TYPE_SUMMARY" => Some(Self::Summary),+ "METRIC_TYPE_DEV" => Some(Self::Dev),+ _ => None,+ }+ }+}+/// Category of an `EXPLAIN ANALYZE` metric. Mirrors+/// `datafusion_common::format::MetricCategory`.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]+#[repr(i32)]+pub enum MetricCategory {+ Rows = 0,+ Bytes = 1,+ Timing = 2,+ Uncategorized = 3,+}+impl MetricCategory {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::Rows => "METRIC_CATEGORY_ROWS",+ Self::Bytes => "METRIC_CATEGORY_BYTES",+ Self::Timing => "METRIC_CATEGORY_TIMING",+ Self::Uncategorized => "METRIC_CATEGORY_UNCATEGORIZED",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "METRIC_CATEGORY_ROWS" => Some(Self::Rows),+ "METRIC_CATEGORY_BYTES" => Some(Self::Bytes),+ "METRIC_CATEGORY_TIMING" => Some(Self::Timing),+ "METRIC_CATEGORY_UNCATEGORIZED" => Some(Self::Uncategorized),+ _ => None,+ }+ }+}diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex 2185748c70..ebae6c1abb 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -224,12 +224,21 @@ message ValuesNode {message AnalyzeNode {LogicalPlanNode input = 1;bool verbose = 2;+ // Statement-level override for `datafusion.explain.analyze_level`.+ // Absent means "fall back to session config".+ optional datafusion_common.MetricType analyze_level = 3;+ // Statement-level override for `datafusion.explain.analyze_categories`.+ // Absent means "fall back to session config".+ optional datafusion_common.ExplainAnalyzeCategoriesNode analyze_categories = 4;}message ExplainNode {LogicalPlanNode input = 1;bool verbose = 2;datafusion_common.ExplainFormat format = 3;+ // Statement-level override for `datafusion.explain.show_statistics`.+ // Absent means "fall back to session config".+ optional bool show_statistics = 4;}message AggregateNode {diff --git a/datafusion/proto-models/src/generated/datafusion_proto_common.rs b/datafusion/proto-models/src/generated/datafusion_proto_common.rsindex 632b16929f..1876102ea9 100644--- a/datafusion/proto-models/src/generated/datafusion_proto_common.rs+++ b/datafusion/proto-models/src/generated/datafusion_proto_common.rs@@ -1014,6 +1014,18 @@ pub struct ColumnStats {#[prost(message, optional, tag = "6")]pub byte_size: ::core::option::Option<Precision>,}+/// Wire encoding for `datafusion_common::format::ExplainAnalyzeCategories`.+///+/// If `all` is true, every category is shown (the `only` list is ignored).+/// If `all` is false, only the categories listed in `only` are shown — an+/// empty `only` means "plan only", i.e. suppress all metrics.+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]+pub struct ExplainAnalyzeCategoriesNode {+ #[prost(bool, tag = "1")]+ pub all: bool,+ #[prost(enumeration = "MetricCategory", repeated, tag = "2")]+ pub only: ::prost::alloc::vec::Vec<i32>,+}#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]#[repr(i32)]pub enum JoinType {@@ -1360,3 +1372,65 @@ impl ExplainFormat {}}}+/// Verbosity level for `EXPLAIN ANALYZE`. Mirrors+/// `datafusion_common::format::MetricType`.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]+#[repr(i32)]+pub enum MetricType {+ Summary = 0,+ Dev = 1,+}+impl MetricType {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::Summary => "METRIC_TYPE_SUMMARY",+ Self::Dev => "METRIC_TYPE_DEV",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "METRIC_TYPE_SUMMARY" => Some(Self::Summary),+ "METRIC_TYPE_DEV" => Some(Self::Dev),+ _ => None,+ }+ }+}+/// Category of an `EXPLAIN ANALYZE` metric. Mirrors+/// `datafusion_common::format::MetricCategory`.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]+#[repr(i32)]+pub enum MetricCategory {+ Rows = 0,+ Bytes = 1,+ Timing = 2,+ Uncategorized = 3,+}+impl MetricCategory {+ /// String value of the enum field names used in the ProtoBuf definition.+ ///+ /// The values are not transformed in any way and thus are considered stable+ /// (if the ProtoBuf definition does not change) and safe for programmatic use.+ pub fn as_str_name(&self) -> &'static str {+ match self {+ Self::Rows => "METRIC_CATEGORY_ROWS",+ Self::Bytes => "METRIC_CATEGORY_BYTES",+ Self::Timing => "METRIC_CATEGORY_TIMING",+ Self::Uncategorized => "METRIC_CATEGORY_UNCATEGORIZED",+ }+ }+ /// Creates an enum from field names used in the ProtoBuf definition.+ pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {+ match value {+ "METRIC_CATEGORY_ROWS" => Some(Self::Rows),+ "METRIC_CATEGORY_BYTES" => Some(Self::Bytes),+ "METRIC_CATEGORY_TIMING" => Some(Self::Timing),+ "METRIC_CATEGORY_UNCATEGORIZED" => Some(Self::Uncategorized),+ _ => None,+ }+ }+}diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex 4136cd2785..6e1901b1e4 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -1166,6 +1166,12 @@ impl serde::Serialize for AnalyzeNode {if self.verbose {len += 1;}+ if self.analyze_level.is_some() {+ len += 1;+ }+ if self.analyze_categories.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.AnalyzeNode", len)?;if let Some(v) = self.input.as_ref() {struct_ser.serialize_field("input", v)?;@@ -1173,6 +1179,14 @@ impl serde::Serialize for AnalyzeNode {if self.verbose {struct_ser.serialize_field("verbose", &self.verbose)?;}+ if let Some(v) = self.analyze_level.as_ref() {+ let v = super::datafusion_common::MetricType::try_from(*v)+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", *v)))?;+ struct_ser.serialize_field("analyzeLevel", &v)?;+ }+ if let Some(v) = self.analyze_categories.as_ref() {+ struct_ser.serialize_field("analyzeCategories", v)?;+ }struct_ser.end()}}@@ -1185,12 +1199,18 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {const FIELDS: &[&str] = &["input","verbose",+ "analyze_level",+ "analyzeLevel",+ "analyze_categories",+ "analyzeCategories",];#[allow(clippy::enum_variant_names)]enum GeneratedField {Input,Verbose,+ AnalyzeLevel,+ AnalyzeCategories,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -1214,6 +1234,8 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {match value {"input" => Ok(GeneratedField::Input),"verbose" => Ok(GeneratedField::Verbose),+ "analyzeLevel" | "analyze_level" => Ok(GeneratedField::AnalyzeLevel),+ "analyzeCategories" | "analyze_categories" => Ok(GeneratedField::AnalyzeCategories),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -1235,6 +1257,8 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {{let mut input__ = None;let mut verbose__ = None;+ let mut analyze_level__ = None;+ let mut analyze_categories__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Input => {@@ -1249,11 +1273,25 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {}verbose__ = Some(map_.next_value()?);}+ GeneratedField::AnalyzeLevel => {+ if analyze_level__.is_some() {+ return Err(serde::de::Error::duplicate_field("analyzeLevel"));+ }+ analyze_level__ = map_.next_value::<::std::option::Option<super::datafusion_common::MetricType>>()?.map(|x| x as i32);+ }+ GeneratedField::AnalyzeCategories => {+ if analyze_categories__.is_some() {+ return Err(serde::de::Error::duplicate_field("analyzeCategories"));+ }+ analyze_categories__ = map_.next_value()?;+ }}}Ok(AnalyzeNode {input: input__,verbose: verbose__.unwrap_or_default(),+ analyze_level: analyze_level__,+ analyze_categories: analyze_categories__,})}}@@ -6218,6 +6256,9 @@ impl serde::Serialize for ExplainNode {if self.format != 0 {len += 1;}+ if self.show_statistics.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.ExplainNode", len)?;if let Some(v) = self.input.as_ref() {struct_ser.serialize_field("input", v)?;@@ -6230,6 +6271,9 @@ impl serde::Serialize for ExplainNode {.map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.format)))?;struct_ser.serialize_field("format", &v)?;}+ if let Some(v) = self.show_statistics.as_ref() {+ struct_ser.serialize_field("showStatistics", v)?;+ }struct_ser.end()}}@@ -6243,6 +6287,8 @@ impl<'de> serde::Deserialize<'de> for ExplainNode {"input","verbose","format",+ "show_statistics",+ "showStatistics",];#[allow(clippy::enum_variant_names)]@@ -6250,6 +6296,7 @@ impl<'de> serde::Deserialize<'de> for ExplainNode {Input,Verbose,Format,+ ShowStatistics,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -6274,6 +6321,7 @@ impl<'de> serde::Deserialize<'de> for ExplainNode {"input" => Ok(GeneratedField::Input),"verbose" => Ok(GeneratedField::Verbose),"format" => Ok(GeneratedField::Format),+ "showStatistics" | "show_statistics" => Ok(GeneratedField::ShowStatistics),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -6296,6 +6344,7 @@ impl<'de> serde::Deserialize<'de> for ExplainNode {let mut input__ = None;let mut verbose__ = None;let mut format__ = None;+ let mut show_statistics__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Input => {@@ -6316,12 +6365,19 @@ impl<'de> serde::Deserialize<'de> for ExplainNode {}format__ = Some(map_.next_value::<super::datafusion_common::ExplainFormat>()? as i32);}+ GeneratedField::ShowStatistics => {+ if show_statistics__.is_some() {+ return Err(serde::de::Error::duplicate_field("showStatistics"));+ }+ show_statistics__ = map_.next_value()?;+ }}}Ok(ExplainNode {input: input__,verbose: verbose__.unwrap_or_default(),format: format__.unwrap_or_default(),+ show_statistics: show_statistics__,})}}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 4e473668e8..d2b25695cb 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -344,6 +344,16 @@ pub struct AnalyzeNode {pub input: ::core::option::Option<::prost::alloc::boxed::Box<LogicalPlanNode>>,#[prost(bool, tag = "2")]pub verbose: bool,+ /// Statement-level override for `datafusion.explain.analyze_level`.+ /// Absent means "fall back to session config".+ #[prost(enumeration = "super::datafusion_common::MetricType", optional, tag = "3")]+ pub analyze_level: ::core::option::Option<i32>,+ /// Statement-level override for `datafusion.explain.analyze_categories`.+ /// Absent means "fall back to session config".+ #[prost(message, optional, tag = "4")]+ pub analyze_categories: ::core::option::Option<+ super::datafusion_common::ExplainAnalyzeCategoriesNode,+ >,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct ExplainNode {@@ -353,6 +363,10 @@ pub struct ExplainNode {pub verbose: bool,#[prost(enumeration = "super::datafusion_common::ExplainFormat", tag = "3")]pub format: i32,+ /// Statement-level override for `datafusion.explain.show_statistics`.+ /// Absent means "fall back to session config".+ #[prost(bool, optional, tag = "4")]+ pub show_statistics: ::core::option::Option<bool>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct AggregateNode {diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex e378532667..12016387d4 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -38,7 +38,9 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaBuilder, SchemaRef};use datafusion_catalog::cte_worktable::CteWorkTable;use datafusion_catalog::empty::EmptyTable;use datafusion_common::file_options::file_type::FileType;-use datafusion_common::format::ExplainFormat;+use datafusion_common::format::{+ ExplainAnalyzeCategories, ExplainFormat, MetricCategory, MetricType,+};use datafusion_common::{NullEquality, Result, TableReference, assert_or_internal_err, context,internal_datafusion_err, internal_err, not_impl_err, plan_err,@@ -70,6 +72,7 @@ use datafusion_expr::{builder::project,},};+use datafusion_proto_common::protobuf_common;use self::to_proto::{serialize_expr, serialize_exprs};use crate::logical_plan::to_proto::serialize_sorts;@@ -377,6 +380,80 @@ fn from_table_source(LogicalPlanNode::try_from_logical_plan(&r, extension_codec)}+fn metric_type_from_proto(value: i32) -> Result<MetricType> {+ let pb = protobuf_common::MetricType::try_from(value)+ .map_err(|_| proto_error(format!("Unknown MetricType discriminant: {value}")))?;+ Ok(match pb {+ protobuf_common::MetricType::Summary => MetricType::Summary,+ protobuf_common::MetricType::Dev => MetricType::Dev,+ })+}++fn metric_type_to_proto(value: MetricType) -> protobuf_common::MetricType {+ match value {+ MetricType::Summary => protobuf_common::MetricType::Summary,+ MetricType::Dev => protobuf_common::MetricType::Dev,+ }+}++fn metric_category_from_proto(value: i32) -> Result<MetricCategory> {+ let pb = protobuf_common::MetricCategory::try_from(value).map_err(|_| {+ proto_error(format!("Unknown MetricCategory discriminant: {value}"))+ })?;+ Ok(match pb {+ protobuf_common::MetricCategory::Rows => MetricCategory::Rows,+ protobuf_common::MetricCategory::Bytes => MetricCategory::Bytes,+ protobuf_common::MetricCategory::Timing => MetricCategory::Timing,+ protobuf_common::MetricCategory::Uncategorized => MetricCategory::Uncategorized,+ })+}++fn metric_category_to_proto(value: MetricCategory) -> protobuf_common::MetricCategory {+ match value {+ MetricCategory::Rows => protobuf_common::MetricCategory::Rows,+ MetricCategory::Bytes => protobuf_common::MetricCategory::Bytes,+ MetricCategory::Timing => protobuf_common::MetricCategory::Timing,+ MetricCategory::Uncategorized => protobuf_common::MetricCategory::Uncategorized,+ }+}++fn explain_analyze_categories_from_proto(+ node: &protobuf_common::ExplainAnalyzeCategoriesNode,+) -> Result<ExplainAnalyzeCategories> {+ if node.all {+ Ok(ExplainAnalyzeCategories::All)+ } else {+ let cats = node+ .only+ .iter()+ .copied()+ .map(metric_category_from_proto)+ .collect::<Result<Vec<_>>>()?;+ Ok(ExplainAnalyzeCategories::Only(cats))+ }+}++fn explain_analyze_categories_to_proto(+ value: &ExplainAnalyzeCategories,+) -> protobuf_common::ExplainAnalyzeCategoriesNode {+ match value {+ ExplainAnalyzeCategories::All => protobuf_common::ExplainAnalyzeCategoriesNode {+ all: true,+ only: vec![],+ },+ ExplainAnalyzeCategories::Only(cats) => {+ protobuf_common::ExplainAnalyzeCategoriesNode {+ all: false,+ only: cats+ .iter()+ .copied()+ .map(|c| metric_category_to_proto(c) as i32)+ .collect(),+ }+ }+ }+}+impl AsLogicalPlan for LogicalPlanNode {fn try_decode(buf: &[u8]) -> Result<Self>where@@ -788,8 +865,23 @@ impl AsLogicalPlan for LogicalPlanNode {LogicalPlanType::Analyze(analyze) => {let input: LogicalPlan =into_logical_plan!(analyze.input, ctx, extension_codec)?;+ let analyze_level = analyze+ .analyze_level+ .map(metric_type_from_proto)+ .transpose()?;+ let analyze_categories = analyze+ .analyze_categories+ .as_ref()+ .map(explain_analyze_categories_from_proto)+ .transpose()?;+ let explain_option =+ datafusion_expr::logical_plan::ExplainOption::default()+ .with_verbose(analyze.verbose)+ .with_analyze(true)+ .with_analyze_level(analyze_level)+ .with_analyze_categories(analyze_categories);LogicalPlanBuilder::from(input)- .explain(analyze.verbose, true)?+ .explain_option_format(explain_option)?.build()}LogicalPlanType::Explain(explain) => {@@ -811,7 +903,8 @@ impl AsLogicalPlan for LogicalPlanNode {let explain_option =datafusion_expr::logical_plan::ExplainOption::default().with_verbose(explain.verbose)- .with_format(explain_format);+ .with_format(explain_format)+ .with_show_statistics(explain.show_statistics);LogicalPlanBuilder::from(input).explain_option_format(explain_option)?.build()@@ -1778,6 +1871,13 @@ impl AsLogicalPlan for LogicalPlanNode {protobuf::AnalyzeNode {input: Some(Box::new(input)),verbose: a.verbose,+ analyze_level: a+ .analyze_level+ .map(|m| metric_type_to_proto(m) as i32),+ analyze_categories: a+ .analyze_categories+ .as_ref()+ .map(explain_analyze_categories_to_proto),},))),})@@ -1803,6 +1903,7 @@ impl AsLogicalPlan for LogicalPlanNode {}}.into(),+ show_statistics: a.show_statistics,},))),})diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rsindex 1bcc6eeb67..7f1d0a666f 100644--- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs@@ -68,7 +68,9 @@ use datafusion::physical_expr::PhysicalExpr;use datafusion::prelude::*;use datafusion::test_util::{TestTableFactory, TestTableProvider};use datafusion_common::config::TableOptions;-use datafusion_common::format::ExplainFormat;+use datafusion_common::format::{+ ExplainAnalyzeCategories, ExplainFormat, MetricCategory, MetricType,+};use datafusion_common::scalar::ScalarStructBuilder;use datafusion_common::{DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, TableReference,@@ -80,7 +82,9 @@ use datafusion_expr::expr::{self, Between, BinaryExpr, Case, Cast, GroupingSet, InList, Like, NullTreatment,ScalarFunction, Unnest, WildcardOptions,};-use datafusion_expr::logical_plan::{Extension, UserDefinedLogicalNodeCore};+use datafusion_expr::logical_plan::{+ ExplainOption, Extension, UserDefinedLogicalNodeCore,+};use datafusion_expr::{Accumulator, AggregateUDF, ColumnarValue, ExprFunctionExt, ExprSchemable,LimitEffect, Literal, LogicalPlan, LogicalPlanBuilder, Operator, PartitionEvaluator,@@ -299,6 +303,74 @@ async fn roundtrip_explain_format_tree() -> Result<()> {Ok(())}+/// Build an `EXPLAIN`/`EXPLAIN ANALYZE` plan with statement-level overrides+/// set directly via the builder, then assert the proto round-trip preserves+/// every field. Going through the builder avoids depending on parser support+/// for the parenthesized option syntax in this test crate.+async fn assert_explain_roundtrip(option: ExplainOption) -> Result<()> {+ let ctx = SessionContext::new();+ let input = ctx.sql("SELECT 1 AS x").await?.into_optimized_plan()?;+ let plan = LogicalPlanBuilder::from(input)+ .explain_option_format(option)?+ .build()?;++ let bytes = logical_plan_to_bytes(&plan)?;+ let round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;+ assert_eq!(plan, round_trip);+ Ok(())+}++#[tokio::test]+async fn roundtrip_explain_show_statistics_override() -> Result<()> {+ for show_statistics in [None, Some(true), Some(false)] {+ assert_explain_roundtrip(+ ExplainOption::default()+ .with_format(ExplainFormat::Indent)+ .with_show_statistics(show_statistics),+ )+ .await?;+ }+ Ok(())+}++#[tokio::test]+async fn roundtrip_analyze_level_override() -> Result<()> {+ for analyze_level in [None, Some(MetricType::Summary), Some(MetricType::Dev)] {+ assert_explain_roundtrip(+ ExplainOption::default()+ .with_analyze(true)+ .with_analyze_level(analyze_level),+ )+ .await?;+ }+ Ok(())+}++#[tokio::test]+async fn roundtrip_analyze_categories_override() -> Result<()> {+ let cases = [+ None,+ Some(ExplainAnalyzeCategories::All),+ Some(ExplainAnalyzeCategories::Only(vec![])),+ Some(ExplainAnalyzeCategories::Only(vec![MetricCategory::Rows])),+ Some(ExplainAnalyzeCategories::Only(vec![+ MetricCategory::Rows,+ MetricCategory::Bytes,+ MetricCategory::Timing,+ MetricCategory::Uncategorized,+ ])),+ ];+ for analyze_categories in cases {+ assert_explain_roundtrip(+ ExplainOption::default()+ .with_analyze(true)+ .with_analyze_categories(analyze_categories),+ )+ .await?;+ }+ Ok(())+}+#[tokio::test]async fn roundtrip_custom_listing_tables() -> Result<()> {let ctx = SessionContext::new();diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rsindex ba37a2d702..67453f8f28 100644--- a/datafusion/sql/src/parser.rs+++ b/datafusion/sql/src/parser.rs@@ -22,6 +22,7 @@use datafusion_common::DataFusionError;use datafusion_common::config::SqlParserOptions;+use datafusion_common::format::{ExplainFormat, ExplainStatementOptions};use datafusion_common::{Diagnostic, Span, sql_err};use sqlparser::ast::{ExprWithAlias, Ident, OrderByOptions};use sqlparser::tokenizer::TokenWithSpan;@@ -36,6 +37,7 @@ use sqlparser::{};use std::collections::VecDeque;use std::fmt;+use std::str::FromStr;// Use `Parser::expected` instead, if possiblemacro_rules! parser_err {@@ -55,18 +57,25 @@ fn parse_file_type(s: &str) -> Result<String, DataFusionError> {/// DataFusion specific `EXPLAIN`///-/// Syntax:+/// Supports both the legacy keyword form and, on dialects whose+/// [`Dialect::supports_explain_with_utility_options`] returns `true`+/// (PostgreSQL, DuckDB, etc.), the Postgres-style parenthesized option list:+////// ```sql+/// -- Legacy keyword form (any dialect)/// EXPLAIN <ANALYZE> <VERBOSE> [FORMAT format] statement+///+/// -- Postgres-style option form (dialect-gated)+/// EXPLAIN (option [arg] [, ...]) statement/// ```+///+/// See [`ExplainStatementOptions`] for the list of supported options in the+/// parenthesized form.#[derive(Debug, Clone, PartialEq, Eq)]pub struct ExplainStatement {- /// `EXPLAIN ANALYZE ..`- pub analyze: bool,- /// `EXPLAIN .. VERBOSE ..`- pub verbose: bool,- /// `EXPLAIN .. FORMAT `- pub format: Option<String>,+ /// Normalized options parsed from either the legacy keyword form or the+ /// parenthesized option list.+ pub options: ExplainStatementOptions,/// The statement to analyze. Note this is a DataFusion [`Statement`] (not a/// [`sqlparser::ast::Statement`] so that we can use `EXPLAIN`, `COPY`, and other/// DataFusion specific statements@@ -75,22 +84,47 @@ pub struct ExplainStatement {impl fmt::Display for ExplainStatement {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- let Self {- analyze,- verbose,- format,- statement,- } = self;+ let Self { options, statement } = self;++ // If only the legacy-era fields are set, print the legacy keyword+ // form so existing round-trip tests continue to pass.+ let uses_parenthesized = options.analyze_level.is_some()+ || options.analyze_categories.is_some()+ || options.show_statistics.is_some();write!(f, "EXPLAIN ")?;- if *analyze {- write!(f, "ANALYZE ")?;- }- if *verbose {- write!(f, "VERBOSE ")?;- }- if let Some(format) = format.as_ref() {- write!(f, "FORMAT {format} ")?;+ if uses_parenthesized {+ // Emit a parenthesized option list.+ let mut parts: Vec<String> = Vec::new();+ if options.analyze {+ parts.push("ANALYZE".to_string());+ }+ if options.verbose {+ parts.push("VERBOSE".to_string());+ }+ if let Some(format) = &options.format {+ parts.push(format!("FORMAT {format}"));+ }+ if let Some(level) = options.analyze_level {+ parts.push(format!("LEVEL {level}"));+ }+ if let Some(cats) = &options.analyze_categories {+ parts.push(format!("METRICS '{cats}'"));+ }+ if let Some(stats) = options.show_statistics {+ parts.push(format!("COSTS {}", if stats { "ON" } else { "OFF" }));+ }+ write!(f, "({}) ", parts.join(", "))?;+ } else {+ if options.analyze {+ write!(f, "ANALYZE ")?;+ }+ if options.verbose {+ write!(f, "VERBOSE ")?;+ }+ if let Some(format) = &options.format {+ write!(f, "FORMAT {format} ")?;+ }}write!(f, "{statement}")@@ -325,6 +359,10 @@ fn ensure_not_set<T>(field: &Option<T>, name: &str) -> Result<(), DataFusionErropub struct DFParser<'a> {pub parser: Parser<'a>,options: SqlParserOptions,+ /// Whether the configured dialect supports Postgres-style+ /// `EXPLAIN (option, ...)` utility-option syntax. Cached here because+ /// sqlparser's [`Parser::dialect`] field is private.+ supports_explain_with_utility_options: bool,}/// Same as `sqlparser`@@ -437,10 +475,34 @@ impl<'a, 'b> DFParserBuilder<'a, 'b> {recursion_limit: self.recursion_limit,..Default::default()},+ supports_explain_with_utility_options: self+ .dialect+ .supports_explain_with_utility_options(),})}}+/// Returns true when `tok` is the start of a query / parenthesized query+/// group. Used to disambiguate `EXPLAIN (SELECT ...)` (a parenthesized query)+/// from `EXPLAIN (ANALYZE) SELECT ...` (a Postgres-style option list).+fn token_starts_query(tok: &Token) -> bool {+ match tok {+ Token::LParen => true,+ Token::Word(Word { keyword, .. }) => matches!(+ keyword,+ Keyword::SELECT+ | Keyword::WITH+ | Keyword::VALUES+ | Keyword::TABLE+ | Keyword::INSERT+ | Keyword::UPDATE+ | Keyword::DELETE+ | Keyword::MERGE+ ),+ _ => false,+ }+}+impl<'a> DFParser<'a> {#[deprecated(since = "46.0.0", note = "DFParserBuilder")]pub fn new(sql: &'a str) -> Result<Self, DataFusionError> {@@ -758,18 +820,46 @@ impl<'a> DFParser<'a> {}/// Parse a SQL `EXPLAIN`+ ///+ /// After the `EXPLAIN` keyword, if the dialect supports the Postgres-style+ /// option list and the next non-whitespace token is `(`, we must+ /// disambiguate between an option list (`EXPLAIN (ANALYZE) SELECT ...`)+ /// and a parenthesized query (`EXPLAIN (SELECT ...)` or+ /// `EXPLAIN (q1 EXCEPT q2) UNION ALL ...`).pub fn parse_explain(&mut self) -> Result<Statement, DataFusionError> {+ if self.supports_explain_with_utility_options+ && self.parser.peek_token().token == Token::LParen+ && !token_starts_query(&self.parser.peek_nth_token(1).token)+ {+ let raw = self.parser.parse_utility_options()?;+ let options = ExplainStatementOptions::from_utility_options(&raw)?;+ let statement = self.parse_statement()?;+ return Ok(Statement::Explain(ExplainStatement {+ statement: Box::new(statement),+ options,+ }));+ }++ // Legacy keyword form.let analyze = self.parser.parse_keyword(Keyword::ANALYZE);let verbose = self.parser.parse_keyword(Keyword::VERBOSE);- let format = self.parse_explain_format()?;+ let format = self+ .parse_explain_format()?+ .map(|s| ExplainFormat::from_str(&s))+ .transpose()?;let statement = self.parse_statement()?;- Ok(Statement::Explain(ExplainStatement {- statement: Box::new(statement),+ let options = ExplainStatementOptions {analyze,verbose,format,+ ..Default::default()+ };++ Ok(Statement::Explain(ExplainStatement {+ statement: Box::new(statement),+ options,}))}@@ -1873,9 +1963,14 @@ mod tests {options: vec![],});let expected = Statement::Explain(ExplainStatement {- analyze,- verbose,- format: None,+ options: ExplainStatementOptions {+ analyze,+ verbose,+ format: None,+ analyze_level: None,+ analyze_categories: None,+ show_statistics: None,+ },statement: Box::new(expected_copy),});assert_eq!(verified_stmt(sql), expected);@@ -2203,4 +2298,164 @@ mod tests {"Expected: end of expression, found: bar",)}++ // ------------------------------------------------------------------+ // Postgres-style `EXPLAIN (option, ...)` tests+ // ------------------------------------------------------------------++ fn parse_with_pg(sql: &str) -> Result<Statement, DataFusionError> {+ let dialect = sqlparser::dialect::PostgreSqlDialect {};+ let mut statements = DFParser::parse_sql_with_dialect(sql, &dialect)?;+ assert_eq!(statements.len(), 1, "Expected exactly one statement");+ Ok(statements.pop_front().unwrap())+ }++ fn parse_with_generic(sql: &str) -> Result<Statement, DataFusionError> {+ let mut statements = DFParser::parse_sql(sql)?;+ assert_eq!(statements.len(), 1, "Expected exactly one statement");+ Ok(statements.pop_front().unwrap())+ }++ #[test]+ fn explain_legacy_keyword_form_postgres_dialect() {+ // The legacy keyword form still works under PostgreSQL dialect.+ let stmt = parse_with_pg("EXPLAIN ANALYZE VERBOSE SELECT 1").unwrap();+ let Statement::Explain(ExplainStatement { options, .. }) = stmt else {+ panic!("Expected Statement::Explain");+ };+ assert!(options.analyze);+ assert!(options.verbose);+ assert!(options.format.is_none());+ assert!(options.analyze_level.is_none());+ }++ #[test]+ fn explain_paren_form_on_generic_supports_utility_options() {+ // sqlparser's GenericDialect also declares+ // `supports_explain_with_utility_options = true`, so DataFusion's+ // default parser accepts the parenthesized form too.+ let stmt = parse_with_generic("EXPLAIN (FORMAT TREE) SELECT 1").unwrap();+ let Statement::Explain(ExplainStatement { options, .. }) = stmt else {+ panic!("Expected Statement::Explain");+ };+ assert_eq!(options.format, Some(ExplainFormat::Tree));+ }++ #[test]+ fn explain_paren_form_on_non_supporting_dialect_is_parse_error() {+ // Dialects that do NOT declare support for utility options (e.g.+ // Snowflake) must still error on the parenthesized form — proving+ // the dialect gate itself works.+ use sqlparser::dialect::SnowflakeDialect;+ let dialect = SnowflakeDialect {};+ let res =+ DFParser::parse_sql_with_dialect("EXPLAIN (FORMAT TREE) SELECT 1", &dialect);+ assert!(+ res.is_err(),+ "expected parse error under non-supporting dialect"+ );+ }++ #[test]+ fn explain_paren_grouping_query_is_not_mistaken_for_options() {+ // Historic DataFusion behavior allows parentheses around the+ // query after EXPLAIN (e.g. `EXPLAIN (SELECT ...)` or+ // `EXPLAIN (q1 EXCEPT q2) UNION ALL (q3 EXCEPT q4)`). The dialect+ // gate for Postgres-style options must not swallow these.+ for sql in [+ "EXPLAIN (SELECT 1)",+ "EXPLAIN (WITH t AS (SELECT 1) SELECT * FROM t)",+ "EXPLAIN (VALUES (1), (2))",+ "EXPLAIN ((SELECT 1))",+ ] {+ let stmt = parse_with_pg(sql).unwrap_or_else(|e| {+ panic!("{sql} failed under PG dialect: {e}");+ });+ let Statement::Explain(ExplainStatement { options, .. }) = stmt else {+ panic!("Expected Statement::Explain for {sql}");+ };+ assert!(!options.analyze, "{sql} should not be ANALYZE");+ assert!(!options.verbose, "{sql} should not be VERBOSE");+ assert!(options.format.is_none(), "{sql} should have no FORMAT");+ }+ }++ #[test]+ fn explain_paren_form_analyze_verbose() {+ let stmt = parse_with_pg("EXPLAIN (ANALYZE, VERBOSE) SELECT 1").unwrap();+ let Statement::Explain(ExplainStatement { options, .. }) = stmt else {+ panic!("Expected Statement::Explain");+ };+ assert!(options.analyze);+ assert!(options.verbose);+ }++ #[test]+ fn explain_paren_form_format_tree() {+ let stmt = parse_with_pg("EXPLAIN (FORMAT tree) SELECT 1").unwrap();+ let Statement::Explain(ExplainStatement { options, .. }) = stmt else {+ panic!("Expected Statement::Explain");+ };+ assert!(!options.analyze);+ assert_eq!(options.format, Some(ExplainFormat::Tree));+ }++ #[test]+ fn explain_paren_form_metrics_level() {+ use datafusion_common::format::{+ ExplainAnalyzeCategories, MetricCategory, MetricType,+ };+ let stmt =+ parse_with_pg("EXPLAIN (ANALYZE, METRICS 'rows,bytes', LEVEL dev) SELECT 1")+ .unwrap();+ let Statement::Explain(ExplainStatement { options, .. }) = stmt else {+ panic!("Expected Statement::Explain");+ };+ assert!(options.analyze);+ assert_eq!(options.analyze_level, Some(MetricType::Dev));+ assert_eq!(+ options.analyze_categories,+ Some(ExplainAnalyzeCategories::Only(vec![+ MetricCategory::Rows,+ MetricCategory::Bytes,+ ]))+ );+ }++ #[test]+ fn explain_paren_form_bool_spellings() {+ let stmt =+ parse_with_pg("EXPLAIN (ANALYZE ON, VERBOSE OFF, COSTS TRUE) SELECT 1")+ .unwrap();+ let Statement::Explain(ExplainStatement { options, .. }) = stmt else {+ panic!("Expected Statement::Explain");+ };+ assert!(options.analyze);+ assert!(!options.verbose);+ assert_eq!(options.show_statistics, Some(true));+ }++ #[test]+ fn explain_paren_form_buffers_rejected() {+ let err = parse_with_pg("EXPLAIN (BUFFERS) SELECT 1").unwrap_err();+ let msg = err.to_string();+ assert!(+ msg.contains("BUFFERS"),+ "error should mention BUFFERS: {msg}"+ );+ assert!(+ msg.contains("not supported"),+ "error should say not supported: {msg}"+ );+ }++ #[test]+ fn explain_paren_form_unknown_option_rejected() {+ let err = parse_with_pg("EXPLAIN (ASDF) SELECT 1").unwrap_err();+ let msg = err.to_string();+ assert!(+ msg.contains("unknown EXPLAIN option"),+ "error should describe unknown option: {msg}"+ );+ }}diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rsindex 8c94610f77..00fd3ddf9e 100644--- a/datafusion/sql/src/statement.rs+++ b/datafusion/sql/src/statement.rs@@ -31,6 +31,7 @@ use crate::utils::normalize_ident;use arrow::datatypes::{Field, FieldRef, Fields};use datafusion_common::error::_plan_err;+use datafusion_common::format::ExplainStatementOptions;use datafusion_common::parsers::CompressionTypeVariant;use datafusion_common::{Column, Constraint, Constraints, DFSchema, DFSchemaRef, DataFusionError, Result,@@ -227,12 +228,9 @@ impl<S: ContextProvider> SqlToRel<'_, S> {DFStatement::CreateExternalTable(s) => self.external_table_to_plan(s),DFStatement::Statement(s) => self.sql_statement_to_plan(*s),DFStatement::CopyTo(s) => self.copy_to_plan(s),- DFStatement::Explain(ExplainStatement {- verbose,- analyze,- format,- statement,- }) => self.explain_to_plan(verbose, analyze, format, *statement),+ DFStatement::Explain(ExplainStatement { options, statement }) => {+ self.explain_to_plan(options, *statement)+ }DFStatement::Reset(statement) => self.reset_statement_to_plan(statement),}}@@ -283,9 +281,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {describe_alias: _,..} => {- let format = format.map(|format| format.to_string());+ let format = format+ .map(|format| ExplainFormat::from_str(&format.to_string()))+ .transpose()?;let statement = DFStatement::Statement(statement);- self.explain_to_plan(verbose, analyze, format, statement)+ let options = ExplainStatementOptions {+ analyze,+ verbose,+ format,+ analyze_level: None,+ analyze_categories: None,+ show_statistics: None,+ };+ self.explain_to_plan(options, statement)}Statement::Query(query) => self.query_to_plan(*query, planner_context),Statement::ShowVariable { variable } => self.show_variable_to_plan(&variable),@@ -2004,9 +2012,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {/// datafusion `EXPLAIN` statement.fn explain_to_plan(&self,- verbose: bool,- analyze: bool,- format: Option<String>,+ opts: ExplainStatementOptions,statement: DFStatement,) -> Result<LogicalPlan> {let plan = self.statement_to_plan(statement)?;@@ -2018,9 +2024,30 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let schema = LogicalPlan::explain_schema();let schema = schema.to_dfschema_ref()?;+ let ExplainStatementOptions {+ analyze,+ verbose,+ format,+ analyze_level,+ analyze_categories,+ show_statistics,+ } = opts;++ // Mutual exclusivity checksif verbose && format.is_some() {return plan_err!("EXPLAIN VERBOSE with FORMAT is not supported");}+ if !analyze {+ if analyze_level.is_some() {+ return plan_err!("EXPLAIN option LEVEL requires ANALYZE");+ }+ if analyze_categories.is_some() {+ return plan_err!("EXPLAIN option METRICS requires ANALYZE");+ }+ }+ if analyze && show_statistics.is_some() {+ return plan_err!("EXPLAIN option COSTS cannot be combined with ANALYZE");+ }if analyze {if format.is_some() {@@ -2030,6 +2057,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {verbose,input: plan,schema,+ analyze_level,+ analyze_categories,}))} else {let stringified_plans =@@ -2041,7 +2070,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {let format = if verbose {ExplainFormat::Indent} else if let Some(format) = format {- ExplainFormat::from_str(&format)?+ format} else {options.explain.format.clone()};@@ -2053,6 +2082,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {stringified_plans,schema,logical_optimization_succeeded: false,+ show_statistics,}))}}diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.sltindex 0df26c4274..9d250ebea1 100644--- a/datafusion/sqllogictest/test_files/explain.slt+++ b/datafusion/sqllogictest/test_files/explain.slt@@ -687,3 +687,149 @@ logical_planstatement okdrop table foo;++# ------------------------------------------------------------------+# Postgres-style `EXPLAIN (option, ...)` tests (dialect-gated).+#+# These require a dialect whose `supports_explain_with_utility_options()`+# returns true. DataFusion's default Generic dialect also declares this+# (mirroring sqlparser-rs 0.61.0), so the parenthesized form works there+# too. We set PostgreSQL explicitly for clarity.+# ------------------------------------------------------------------++statement ok+set datafusion.sql_parser.dialect = 'PostgreSQL';++# `EXPLAIN (FORMAT tree)` matches the legacy `EXPLAIN FORMAT tree` form.+query TT+EXPLAIN (FORMAT tree) SELECT 1;+----+physical_plan+01)┌───────────────────────────┐+02)│ ProjectionExec │+03)│ -------------------- │+04)│ Int64(1): 1 │+05)└─────────────┬─────────────┘+06)┌─────────────┴─────────────┐+07)│ PlaceholderRowExec │+08)└───────────────────────────┘++# Unknown options are rejected with a clear error.+statement error DataFusion error: Error during planning: unknown EXPLAIN option: FOO+EXPLAIN (FOO) SELECT 1;++# Postgres-only options return a "not supported" message pointing at METRICS.+statement error DataFusion error: This feature is not implemented: EXPLAIN option BUFFERS is not supported by DataFusion+EXPLAIN (BUFFERS) SELECT 1;++statement error DataFusion error: This feature is not implemented: EXPLAIN option WAL is not supported by DataFusion+EXPLAIN (WAL) SELECT 1;++# LEVEL / METRICS / TIMING / SUMMARY all require ANALYZE.+statement error DataFusion error: Error during planning: EXPLAIN option LEVEL requires ANALYZE+EXPLAIN (LEVEL dev) SELECT 1;++statement error DataFusion error: Error during planning: EXPLAIN option METRICS requires ANALYZE+EXPLAIN (METRICS 'rows') SELECT 1;++# COSTS and ANALYZE are mutually exclusive (COSTS only applies to plan-only+# EXPLAIN).+statement error DataFusion error: Error during planning: EXPLAIN option COSTS cannot be combined with ANALYZE+EXPLAIN (ANALYZE, COSTS ON) SELECT 1;++# TIMING and SUMMARY are sugar for METRICS/LEVEL and likewise need ANALYZE.+statement error DataFusion error: Error during planning: EXPLAIN option METRICS requires ANALYZE+EXPLAIN (TIMING ON) SELECT 1;++statement error DataFusion error: Error during planning: EXPLAIN option LEVEL requires ANALYZE+EXPLAIN (SUMMARY ON) SELECT 1;++# FORMAT is incompatible with both ANALYZE and VERBOSE (same as the legacy+# keyword form — these mappings come from the planner, not the parser).+statement error DataFusion error: Error during planning: EXPLAIN ANALYZE with FORMAT is not supported+EXPLAIN (ANALYZE, FORMAT tree) SELECT 1;++statement error DataFusion error: Error during planning: EXPLAIN VERBOSE with FORMAT is not supported+EXPLAIN (VERBOSE, FORMAT tree) SELECT 1;++# FORMAT argument can be a bare identifier (already tested) or a quoted+# string and produces the same plan either way.+query TT+EXPLAIN (FORMAT 'tree') SELECT 1;+----+physical_plan+01)┌───────────────────────────┐+02)│ ProjectionExec │+03)│ -------------------- │+04)│ Int64(1): 1 │+05)└─────────────┬─────────────┘+06)┌─────────────┴─────────────┐+07)│ PlaceholderRowExec │+08)└───────────────────────────┘++# Bool option arguments accept bare/ON|OFF/TRUE|FALSE/1|0/=value forms.+# `ANALYZE OFF` is the same as a plain `EXPLAIN`.+query TT+EXPLAIN (ANALYZE OFF, FORMAT tree) SELECT 1;+----+physical_plan+01)┌───────────────────────────┐+02)│ ProjectionExec │+03)│ -------------------- │+04)│ Int64(1): 1 │+05)└─────────────┬─────────────┘+06)┌─────────────┴─────────────┐+07)│ PlaceholderRowExec │+08)└───────────────────────────┘++# `COSTS OFF` overrides `datafusion.explain.show_statistics` per-statement+# (ANALYZE+COSTS is rejected above).+query TT+EXPLAIN (COSTS OFF) SELECT 1;+----+logical_plan+01)Projection: Int64(1)+02)--EmptyRelation: rows=1+physical_plan+01)ProjectionExec: expr=[1 as Int64(1)]+02)--PlaceholderRowExec++# Bool argument forms: ON / TRUE / 1 all enable the option. The parenthesized+# form does not support `= value` for booleans (sqlparser's utility option+# grammar). Quoted-string booleans are accepted by the option parser.+statement ok+EXPLAIN (COSTS ON) SELECT 1;++statement ok+EXPLAIN (COSTS TRUE) SELECT 1;++statement ok+EXPLAIN (COSTS 1) SELECT 1;++statement ok+EXPLAIN (COSTS 'true') SELECT 1;++# Unrecognized argument for a boolean option.+statement error DataFusion error: Error during planning: expected boolean for EXPLAIN option costs, got 'maybe'+EXPLAIN (COSTS maybe) SELECT 1;++# Unrecognized argument for a string/ident option.+statement error DataFusion error: Invalid or Unsupported Configuration: Invalid explain format\. Expected 'indent', 'tree', 'pgjson' or 'graphviz'\. Got 'bogus'+EXPLAIN (FORMAT bogus) SELECT 1;++# Legacy keyword form still works on PostgreSQL dialect.+query TT+EXPLAIN FORMAT tree SELECT 1;+----+physical_plan+01)┌───────────────────────────┐+02)│ ProjectionExec │+03)│ -------------------- │+04)│ Int64(1): 1 │+05)└─────────────┬─────────────┘+06)┌─────────────┴─────────────┐+07)│ PlaceholderRowExec │+08)└───────────────────────────┘++statement ok+reset datafusion.sql_parser.dialect;diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.sltindex f84994d97c..e3bd83d569 100644--- a/datafusion/sqllogictest/test_files/explain_analyze.slt+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt@@ -300,6 +300,114 @@ reset datafusion.explain.analyze_categories;statement okreset datafusion.explain.analyze_level;+# ------------------------------------------------------------------+# Same category/level filtering, but via the Postgres-style+# `EXPLAIN (ANALYZE, METRICS ..., LEVEL ...)` statement option list.+#+# Each block below mirrors one of the `set datafusion.explain.*`+# tests above so the equivalence between session config and+# per-statement overrides is exercised side-by-side.+# ------------------------------------------------------------------++statement ok+set datafusion.sql_parser.dialect = 'PostgreSQL';++# ---- (METRICS 'none', LEVEL summary) — plan only, no metrics ----++query TT+EXPLAIN (ANALYZE, METRICS 'none', LEVEL summary) select * from cat_tracking where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[]++# ---- (METRICS 'rows', LEVEL summary) — row-count metrics only ----++query TT+EXPLAIN (ANALYZE, METRICS 'rows', LEVEL summary) select * from cat_tracking where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=21.75% (485/2.23 K)]++# ---- Quoted-string METRICS with multiple categories ----++query TT+EXPLAIN (ANALYZE, METRICS 'rows,bytes', LEVEL summary) select * from cat_tracking where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]++# ---- (METRICS 'timing', LEVEL summary) — timing metrics only ----++query TT+EXPLAIN (ANALYZE, METRICS 'timing', LEVEL summary) select * from cat_tracking where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[elapsed_compute=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[elapsed_compute=<slt:ignore>, metadata_load_time=<slt:ignore>]++# ---- TIMING sugar: `METRICS 'rows,bytes', TIMING off` ↔ rows+bytes only ----+# Equivalent to METRICS 'rows,bytes' since the sugar removes the timing+# category from the explicit METRICS selection.++query TT+EXPLAIN (ANALYZE, METRICS 'rows,bytes', TIMING off, LEVEL summary) select * from cat_tracking where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]++# ---- TIMING sugar: `METRICS 'rows', TIMING on` ↔ rows + timing ----++query TT+EXPLAIN (ANALYZE, METRICS 'rows', TIMING on, LEVEL summary) select * from cat_tracking where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, elapsed_compute=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=21.75% (485/2.23 K)]++# ---- SUMMARY sugar: `SUMMARY on` ↔ `LEVEL summary` ----+# Equivalent to METRICS 'rows', LEVEL summary above.++query TT+EXPLAIN (ANALYZE, METRICS 'rows', SUMMARY on) select * from cat_tracking where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=21.75% (485/2.23 K)]++# ---- Statement option overrides session config ----+# Session says 'timing' but statement-level `METRICS 'rows'` wins.++statement ok+set datafusion.explain.analyze_categories = 'timing';++query TT+EXPLAIN (ANALYZE, METRICS 'rows', LEVEL summary) select * from cat_tracking where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=21.75% (485/2.23 K)]++statement ok+reset datafusion.explain.analyze_categories;++# ---- Argument syntax variants for METRICS ----+# Bare identifier, `= value`, and quoted string forms should all parse+# to the same selection.++query TT+EXPLAIN (ANALYZE, METRICS rows, LEVEL summary) select * from cat_tracking where species > 'M' AND s >= 50 order by species limit 3;+----+Plan with Metrics+01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=21.75% (485/2.23 K)]++statement ok+reset datafusion.sql_parser.dialect;+# --- Teardown ---statement okdiff --git a/docs/source/user-guide/explain-usage.md b/docs/source/user-guide/explain-usage.mdindex 9e06acbce4..0f3f008c52 100644--- a/docs/source/user-guide/explain-usage.md+++ b/docs/source/user-guide/explain-usage.md@@ -241,6 +241,46 @@ When predicate pushdown is enabled, `DataSourceExec` with `ParquetSource` gains- `row_pushdown_eval_time`: time spent evaluating row-level filters- `page_index_eval_time`: time required to evaluate the page index filters+## Postgres-style `EXPLAIN (...)` options++In addition to the legacy keyword form (`EXPLAIN ANALYZE VERBOSE FORMAT tree SELECT ...`),+DataFusion accepts a Postgres-style option list on dialects whose+[`supports_explain_with_utility_options`](https://docs.rs/sqlparser/latest/sqlparser/dialect/trait.Dialect.html#method.supports_explain_with_utility_options)+returns `true`. This includes the default `GenericDialect`, `PostgreSqlDialect`, and+`DuckDbDialect`, among others.++```sql+EXPLAIN (ANALYZE, VERBOSE, METRICS 'rows,bytes', LEVEL dev)+SELECT ... ;+```++The recognized options are:++| Option | Argument | Effect |+| --------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------ |+| `ANALYZE` | boolean, optional | Execute the plan and collect metrics. Defaults to `TRUE` when bare. Equivalent to the `ANALYZE` keyword. |+| `VERBOSE` | boolean, optional | Show per-partition metrics and additional detail. Equivalent to the `VERBOSE` keyword. |+| `FORMAT` | identifier/string | One of `indent`, `tree`, `pgjson`, `graphviz`. Equivalent to the `FORMAT <format>` clause. |+| `METRICS` | string | Filter `ANALYZE` metrics by category. Accepts `'all'`, `'none'`, or any comma-separated subset of `rows,bytes,timing,uncategorized`. |+| `LEVEL` | identifier/string | `summary` or `dev`. Controls metric verbosity for `ANALYZE`. |+| `TIMING` | boolean | Sugar over `METRICS`: toggles inclusion of the `timing` category. |+| `SUMMARY` | boolean | Sugar over `LEVEL`: `TRUE` → `summary`, `FALSE` → `dev`. |+| `COSTS` | boolean | Include statistics in plain `EXPLAIN` output (equivalent to `SET datafusion.explain.show_statistics`). Not valid with `ANALYZE`. |++Boolean arguments can be written bare (`ANALYZE` → `true`), as `TRUE`/`FALSE`,+`ON`/`OFF`, or `0`/`1`.++The statement-level options take precedence over session config, so you can leave+the session defaults alone and override just for the current query:++```sql+EXPLAIN (ANALYZE, LEVEL dev, METRICS 'rows,bytes') SELECT ...;+```++Postgres options that DataFusion does not model (`BUFFERS`, `WAL`, `SETTINGS`,+`GENERIC_PLAN`, `MEMORY`) return a clear error rather than being silently+accepted — use `METRICS` to filter what appears in the output.+## Partitions and ExecutionDataFusion determines the optimal number of cores to use as part of query