Skip to content

Apache DataFusion SQL Query Engine

feat: add pgjson format support for EXPLAIN ANALYZE (#21767)

Adrian Garcia Badaracco496f2c2065c0
diff --git a/Cargo.lock b/Cargo.lock
index c08be6f29f..0611af1227 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2464,6 +2464,7 @@ dependencies = [
"rand 0.9.4",
"rstest",
"rstest_reuse",
+ "serde_json",
"tokio",
]
diff --git a/datafusion/core/src/datasource/physical_plan/parquet.rs b/datafusion/core/src/datasource/physical_plan/parquet.rs
index 87e7fb1af4..d562bbe849 100644
--- a/datafusion/core/src/datasource/physical_plan/parquet.rs
+++ b/datafusion/core/src/datasource/physical_plan/parquet.rs
@@ -62,10 +62,10 @@ mod tests {
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_expr::{Expr, col, lit, when};
use datafusion_physical_expr::planner::logical2physical;
- use datafusion_physical_plan::analyze::AnalyzeExec;
+ use datafusion_physical_plan::analyze::AnalyzeExecBuilder;
use datafusion_physical_plan::collect;
use datafusion_physical_plan::metrics::{
- ExecutionPlanMetricsSet, MetricType, MetricValue, MetricsSet,
+ ExecutionPlanMetricsSet, MetricValue, MetricsSet,
};
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};
@@ -231,21 +231,22 @@ mod tests {
let parquet_exec =
self.build_parquet_exec(file_group.clone(), Arc::clone(&parquet_source));
- let analyze_exec = Arc::new(AnalyzeExec::new(
- false,
- false,
- vec![MetricType::Summary, MetricType::Dev],
- None,
- // use a new ParquetSource to avoid sharing execution metrics
- self.build_parquet_exec(
- file_group.clone(),
- self.build_file_source(Arc::clone(table_schema)),
- ),
- Arc::new(Schema::new(vec![
- Field::new("plan_type", DataType::Utf8, true),
- Field::new("plan", DataType::Utf8, true),
- ])),
- ));
+ let analyze_exec = Arc::new(
+ AnalyzeExecBuilder::new(
+ false,
+ false,
+ // use a new ParquetSource to avoid sharing execution metrics
+ self.build_parquet_exec(
+ file_group.clone(),
+ self.build_file_source(Arc::clone(table_schema)),
+ ),
+ Arc::new(Schema::new(vec![
+ Field::new("plan_type", DataType::Utf8, true),
+ Field::new("plan", DataType::Utf8, true),
+ ])),
+ )
+ .build(),
+ );
let session_ctx = SessionContext::new();
let task_ctx = session_ctx.task_ctx();
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index e5e1b34642..a6c98179cf 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -2801,14 +2801,13 @@ impl DefaultPhysicalPlanner {
ExplainAnalyzeCategories::All => None,
ExplainAnalyzeCategories::Only(cats) => Some(cats),
};
- Ok(Arc::new(AnalyzeExec::new(
- a.verbose,
- show_statistics,
- metric_types,
- metric_categories,
- input,
- schema,
- )))
+ Ok(Arc::new(
+ AnalyzeExec::builder(a.verbose, show_statistics, input, schema)
+ .with_metric_types(metric_types)
+ .with_metric_categories(metric_categories)
+ .with_format(a.format.clone())
+ .build(),
+ ))
}
/// Optimize a physical plan by applying each physical optimizer,
diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs
index e107d233b6..29bc448c8f 100644
--- a/datafusion/expr/src/logical_plan/builder.rs
+++ b/datafusion/expr/src/logical_plan/builder.rs
@@ -1335,6 +1335,7 @@ impl LogicalPlanBuilder {
if explain_option.analyze {
Ok(Self::new(LogicalPlan::Analyze(Analyze {
verbose: explain_option.verbose,
+ format: explain_option.format,
input: self.plan,
schema,
analyze_level: explain_option.analyze_level,
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index 0a953e759c..cef20dcd5a 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -1093,6 +1093,7 @@ impl LogicalPlan {
let input = self.only_input(inputs)?;
Ok(LogicalPlan::Analyze(Analyze {
verbose: a.verbose,
+ format: a.format.clone(),
schema: Arc::clone(&a.schema),
input: Arc::new(input),
analyze_level: a.analyze_level,
@@ -3469,6 +3470,8 @@ impl PartialOrd for Explain {
pub struct Analyze {
/// Should extra detail be included?
pub verbose: bool,
+ /// Output syntax/format for the rendered physical plan + metrics.
+ pub format: ExplainFormat,
/// The logical plan that is being EXPLAIN ANALYZE'd
pub input: Arc<LogicalPlan>,
/// The output schema of the explain (2 columns of text)
diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rs
index 98ac27aa2b..2c6be54705 100644
--- a/datafusion/expr/src/logical_plan/tree_node.rs
+++ b/datafusion/expr/src/logical_plan/tree_node.rs
@@ -217,6 +217,7 @@ impl TreeNode for LogicalPlan {
}),
LogicalPlan::Analyze(Analyze {
verbose,
+ format,
input,
schema,
analyze_level,
@@ -224,6 +225,7 @@ impl TreeNode for LogicalPlan {
}) => input.map_elements(f)?.update_data(|input| {
LogicalPlan::Analyze(Analyze {
verbose,
+ format,
input,
schema,
analyze_level,
diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml
index 515b65ac1b..0fc75043bf 100644
--- a/datafusion/physical-plan/Cargo.toml
+++ b/datafusion/physical-plan/Cargo.toml
@@ -85,6 +85,7 @@ log = { workspace = true }
num-traits = { workspace = true }
parking_lot = { workspace = true }
pin-project-lite = "^0.2.7"
+serde_json = { workspace = true, features = ["preserve_order"] }
tokio = { workspace = true }
[dev-dependencies]
diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs
index 582af8f1e3..580bf31231 100644
--- a/datafusion/physical-plan/src/analyze.rs
+++ b/datafusion/physical-plan/src/analyze.rs
@@ -29,8 +29,11 @@ use crate::metrics::{MetricCategory, MetricType};
use crate::{DisplayFormatType, ExecutionPlan, Partitioning};
use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};
+use datafusion_common::format::ExplainFormat;
use datafusion_common::instant::Instant;
-use datafusion_common::{DataFusionError, Result, assert_eq_or_internal_err};
+use datafusion_common::{
+ DataFusionError, Result, assert_eq_or_internal_err, internal_err,
+};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
@@ -48,6 +51,8 @@ pub struct AnalyzeExec {
metric_types: Vec<MetricType>,
/// Optional filter by semantic category (rows / bytes / timing).
metric_categories: Option<Vec<MetricCategory>>,
+ /// Output format for the rendered plan + metrics.
+ format: ExplainFormat,
/// The input plan (the plan being analyzed)
pub(crate) input: Arc<dyn ExecutionPlan>,
/// The output schema for RecordBatches of this exec node
@@ -55,27 +60,81 @@ pub struct AnalyzeExec {
cache: Arc<PlanProperties>,
}
-impl AnalyzeExec {
- /// Create a new AnalyzeExec
+/// Builder for [`AnalyzeExec`].
+///
+/// Builder for [AnalyzeExec].
+pub struct AnalyzeExecBuilder {
+ verbose: bool,
+ show_statistics: bool,
+ input: Arc<dyn ExecutionPlan>,
+ schema: SchemaRef,
+ metric_types: Vec<MetricType>,
+ metric_categories: Option<Vec<MetricCategory>>,
+ format: ExplainFormat,
+}
+
+impl AnalyzeExecBuilder {
pub fn new(
verbose: bool,
show_statistics: bool,
- metric_types: Vec<MetricType>,
- metric_categories: Option<Vec<MetricCategory>>,
input: Arc<dyn ExecutionPlan>,
schema: SchemaRef,
) -> Self {
- let cache = Self::compute_properties(&input, Arc::clone(&schema));
- AnalyzeExec {
+ Self {
verbose,
show_statistics,
- metric_types,
- metric_categories,
input,
schema,
+ metric_types: vec![MetricType::Summary, MetricType::Dev],
+ metric_categories: None,
+ format: ExplainFormat::Indent,
+ }
+ }
+
+ pub fn with_metric_types(mut self, metric_types: Vec<MetricType>) -> Self {
+ self.metric_types = metric_types;
+ self
+ }
+
+ pub fn with_metric_categories(
+ mut self,
+ metric_categories: Option<Vec<MetricCategory>>,
+ ) -> Self {
+ self.metric_categories = metric_categories;
+ self
+ }
+
+ pub fn with_format(mut self, format: ExplainFormat) -> Self {
+ self.format = format;
+ self
+ }
+
+ pub fn build(self) -> AnalyzeExec {
+ let cache =
+ AnalyzeExec::compute_properties(&self.input, Arc::clone(&self.schema));
+ AnalyzeExec {
+ verbose: self.verbose,
+ show_statistics: self.show_statistics,
+ metric_types: self.metric_types,
+ metric_categories: self.metric_categories,
+ format: self.format,
+ input: self.input,
+ schema: self.schema,
cache: Arc::new(cache),
}
}
+}
+
+impl AnalyzeExec {
+ /// Returns a builder for constructing an [`AnalyzeExec`].
+ pub fn builder(
+ verbose: bool,
+ show_statistics: bool,
+ input: Arc<dyn ExecutionPlan>,
+ schema: SchemaRef,
+ ) -> AnalyzeExecBuilder {
+ AnalyzeExecBuilder::new(verbose, show_statistics, input, schema)
+ }
/// Access to verbose
pub fn verbose(&self) -> bool {
@@ -92,6 +151,11 @@ impl AnalyzeExec {
self.metric_categories.as_deref()
}
+ /// Access to format
+ pub fn format(&self) -> &ExplainFormat {
+ &self.format
+ }
+
/// The input plan
pub fn input(&self) -> &Arc<dyn ExecutionPlan> {
&self.input
@@ -151,14 +215,18 @@ impl ExecutionPlan for AnalyzeExec {
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
) -> Result<Arc<dyn ExecutionPlan>> {
- Ok(Arc::new(Self::new(
- self.verbose,
- self.show_statistics,
- self.metric_types.clone(),
- self.metric_categories.clone(),
- children.pop().unwrap(),
- Arc::clone(&self.schema),
- )))
+ Ok(Arc::new(
+ AnalyzeExec::builder(
+ self.verbose,
+ self.show_statistics,
+ children.pop().unwrap(),
+ Arc::clone(&self.schema),
+ )
+ .with_metric_types(self.metric_types.clone())
+ .with_metric_categories(self.metric_categories.clone())
+ .with_format(self.format.clone())
+ .build(),
+ ))
}
fn execute(
@@ -195,6 +263,7 @@ impl ExecutionPlan for AnalyzeExec {
let show_statistics = self.show_statistics;
let metric_types = self.metric_types.clone();
let metric_categories = self.metric_categories.clone();
+ let format = self.format.clone();
// future that gathers the results from all the tasks in the
// JoinSet that computes the overall row count and final
@@ -217,6 +286,7 @@ impl ExecutionPlan for AnalyzeExec {
&captured_schema,
&metric_types,
metric_categories.as_deref(),
+ &format,
)
};
@@ -238,39 +308,61 @@ fn create_output_batch(
schema: &SchemaRef,
metric_types: &[MetricType],
metric_categories: Option<&[MetricCategory]>,
+ format: &ExplainFormat,
) -> Result<RecordBatch> {
let mut type_builder = StringBuilder::with_capacity(1, 1024);
let mut plan_builder = StringBuilder::with_capacity(1, 1024);
- // TODO use some sort of enum rather than strings?
- type_builder.append_value("Plan with Metrics");
-
- let annotated_plan = DisplayableExecutionPlan::with_metrics(input.as_ref())
- .set_metric_types(metric_types.to_vec())
- .set_metric_categories(metric_categories.map(|c| c.to_vec()))
- .set_show_statistics(show_statistics)
- .indent(verbose)
- .to_string();
- plan_builder.append_value(annotated_plan);
-
- // Verbose output
- // TODO make this more sophisticated
- if verbose {
- type_builder.append_value("Plan with Full Metrics");
-
- let annotated_plan = DisplayableExecutionPlan::with_full_metrics(input.as_ref())
- .set_metric_types(metric_types.to_vec())
- .set_metric_categories(metric_categories.map(|c| c.to_vec()))
- .set_show_statistics(show_statistics)
- .indent(verbose)
- .to_string();
- plan_builder.append_value(annotated_plan);
-
- type_builder.append_value("Output Rows");
- plan_builder.append_value(total_rows.to_string());
-
- type_builder.append_value("Duration");
- plan_builder.append_value(format!("{duration:?}"));
+ match format {
+ ExplainFormat::Indent => {
+ // TODO use some sort of enum rather than strings?
+ type_builder.append_value("Plan with Metrics");
+ let annotated_plan = DisplayableExecutionPlan::with_metrics(input.as_ref())
+ .set_metric_types(metric_types.to_vec())
+ .set_metric_categories(metric_categories.map(|c| c.to_vec()))
+ .set_show_statistics(show_statistics)
+ .indent(verbose)
+ .to_string();
+ plan_builder.append_value(annotated_plan);
+ // Verbose output
+ // TODO make this more sophisticated
+ if verbose {
+ type_builder.append_value("Plan with Full Metrics");
+ let annotated_plan =
+ DisplayableExecutionPlan::with_full_metrics(input.as_ref())
+ .set_metric_types(metric_types.to_vec())
+ .set_metric_categories(metric_categories.map(|c| c.to_vec()))
+ .set_show_statistics(show_statistics)
+ .indent(verbose)
+ .to_string();
+ plan_builder.append_value(annotated_plan);
+ type_builder.append_value("Output Rows");
+ plan_builder.append_value(total_rows.to_string());
+ type_builder.append_value("Duration");
+ plan_builder.append_value(format!("{duration:?}"));
+ }
+ }
+ ExplainFormat::PostgresJSON => {
+ // `show_statistics` is intentionally not forwarded here: the pgjson
+ // renderer does not emit statistics, and the planner rejects the
+ // `show_statistics` + pgjson combination up front.
+ type_builder.append_value("Plan with Metrics");
+ let mut displayable = if verbose {
+ DisplayableExecutionPlan::with_full_metrics(input.as_ref())
+ } else {
+ DisplayableExecutionPlan::with_metrics(input.as_ref())
+ };
+ displayable = displayable
+ .set_metric_types(metric_types.to_vec())
+ .set_metric_categories(metric_categories.map(|c| c.to_vec()));
+ if verbose {
+ displayable = displayable.set_summary(Some(total_rows), Some(duration));
+ }
+ plan_builder.append_value(displayable.pgjson(verbose).to_string());
+ }
+ ExplainFormat::Tree | ExplainFormat::Graphviz => {
+ return internal_err!("AnalyzeExec does not support {format} output format");
+ }
}
RecordBatch::try_new(
@@ -305,14 +397,8 @@ mod tests {
let blocking_exec = Arc::new(BlockingExec::new(Arc::clone(&schema), 1));
let refs = blocking_exec.refs();
- let analyze_exec = Arc::new(AnalyzeExec::new(
- true,
- false,
- vec![MetricType::Summary, MetricType::Dev],
- None,
- blocking_exec,
- schema,
- ));
+ let analyze_exec =
+ Arc::new(AnalyzeExec::builder(true, false, blocking_exec, schema).build());
let fut = collect(analyze_exec, task_ctx);
let mut fut = fut.boxed();
diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rs
index 8ad1f60651..4642a9a4b1 100644
--- a/datafusion/physical-plan/src/display.rs
+++ b/datafusion/physical-plan/src/display.rs
@@ -21,6 +21,7 @@
use std::collections::{BTreeMap, HashMap};
use std::fmt;
use std::fmt::Formatter;
+use std::time::Duration;
use arrow::datatypes::SchemaRef;
@@ -28,7 +29,7 @@ use datafusion_common::display::{GraphvizBuilder, PlanType, StringifiedPlan};
use datafusion_expr::display_schema;
use datafusion_physical_expr::LexOrdering;
-use crate::metrics::{MetricCategory, MetricType};
+use crate::metrics::{MetricCategory, MetricType, MetricValue};
use crate::render_tree::RenderTree;
use super::{ExecutionPlan, ExecutionPlanVisitor, accept};
@@ -128,6 +129,17 @@ pub struct DisplayableExecutionPlan<'a> {
metric_categories: Option<Vec<MetricCategory>>,
// (TreeRender) Maximum total width of the rendered tree
tree_maximum_render_width: usize,
+ /// Optional summary totals (currently only used by `pgjson`) — the total
+ /// row count and wall-clock duration of the `AnalyzeExec` execution.
+ summary: Option<AnalyzeSummary>,
+}
+
+/// Summary information attached to the root of an `EXPLAIN ANALYZE`
+/// pgjson render.
+#[derive(Debug, Clone, Copy)]
+struct AnalyzeSummary {
+ total_rows: Option<usize>,
+ duration: Option<Duration>,
}
impl<'a> DisplayableExecutionPlan<'a> {
@@ -146,6 +158,7 @@ impl<'a> DisplayableExecutionPlan<'a> {
metric_types: Self::default_metric_types(),
metric_categories: None,
tree_maximum_render_width: 240,
+ summary: None,
}
}
@@ -161,6 +174,7 @@ impl<'a> DisplayableExecutionPlan<'a> {
metric_types: Self::default_metric_types(),
metric_categories: None,
tree_maximum_render_width: 240,
+ summary: None,
}
}
@@ -176,6 +190,7 @@ impl<'a> DisplayableExecutionPlan<'a> {
metric_types: Self::default_metric_types(),
metric_categories: None,
tree_maximum_render_width: 240,
+ summary: None,
}
}
@@ -223,6 +238,21 @@ impl<'a> DisplayableExecutionPlan<'a> {
self
}
+ /// Attach an `EXPLAIN ANALYZE` summary (total output rows and duration)
+ /// to the rendered output. Currently only used by [`Self::pgjson`], which
+ /// serializes the summary alongside the root plan object.
+ pub fn set_summary(
+ mut self,
+ total_rows: Option<usize>,
+ duration: Option<Duration>,
+ ) -> Self {
+ self.summary = Some(AnalyzeSummary {
+ total_rows,
+ duration,
+ });
+ self
+ }
+
/// Return a `format`able structure that produces a single line
/// per node.
///
@@ -349,6 +379,75 @@ impl<'a> DisplayableExecutionPlan<'a> {
}
}
+ /// Returns a `format`able structure that produces PostgreSQL-style JSON
+ /// output, mirroring the logical-plan pgjson format.
+ ///
+ /// Each node is rendered as a JSON object with:
+ /// - `"Node Type"` — `ExecutionPlan::name()`
+ /// - `"Details"` — the one-line `DisplayAs::Default` rendering
+ /// - `"Output"` — schema column names (when `set_show_schema(true)`)
+ /// - `"Actual Rows"` / `"Actual Total Time"` — PG-canonical metric keys
+ /// populated from `output_rows` / `elapsed_compute` when available
+ /// - `"Extras"` — remaining metrics keyed by DataFusion metric name
+ /// - `"Plans"` — array of child nodes
+ ///
+ /// When a summary has been set via [`Self::set_summary`], `"Total Rows"`
+ /// and `"Duration"` fields are attached at the root.
+ pub fn pgjson(&self, verbose: bool) -> impl fmt::Display + 'a {
+ struct Wrapper<'a> {
+ plan: &'a dyn ExecutionPlan,
+ verbose: bool,
+ show_metrics: ShowMetrics,
+ show_schema: bool,
+ metric_types: Vec<MetricType>,
+ metric_categories: Option<Vec<MetricCategory>>,
+ summary: Option<AnalyzeSummary>,
+ }
+ impl fmt::Display for Wrapper<'_> {
+ fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+ let mut visitor = PgJsonExecutionPlanVisitor {
+ verbose: self.verbose,
+ show_metrics: self.show_metrics,
+ show_schema: self.show_schema,
+ metric_types: &self.metric_types,
+ metric_categories: self.metric_categories.as_deref(),
+ objects: HashMap::new(),
+ parent_ids: Vec::new(),
+ next_id: 0,
+ root: None,
+ };
+ accept(self.plan, &mut visitor).map_err(|_| fmt::Error)?;
+ let root = visitor.root.ok_or(fmt::Error)?;
+ let mut root_entry = serde_json::json!({ "Plan": root });
+ if let Some(summary) = self.summary {
+ if let Some(total_rows) = summary.total_rows {
+ root_entry["Total Rows"] = serde_json::Value::from(total_rows);
+ }
+ if let Some(duration) = summary.duration {
+ root_entry["Duration"] =
+ serde_json::Value::from(format!("{duration:?}"));
+ }
+ }
+ let doc = serde_json::Value::Array(vec![root_entry]);
+ write!(
+ f,
+ "{}",
+ serde_json::to_string_pretty(&doc).map_err(|_| fmt::Error)?
+ )
+ }
+ }
+
+ Wrapper {
+ plan: self.inner,
+ verbose,
+ show_metrics: self.show_metrics,
+ show_schema: self.show_schema,
+ metric_types: self.metric_types.clone(),
+ metric_categories: self.metric_categories.clone(),
+ summary: self.summary,
+ }
+ }
+
/// Return a single-line summary of the root of the plan
/// Example: `ProjectionExec: expr=[a@0 as a]`.
pub fn one_line(&self) -> impl fmt::Display + 'a {
@@ -611,6 +710,182 @@ impl ExecutionPlanVisitor for GraphvizVisitor<'_, '_> {
}
}
+/// Formats physical plans into PostgreSQL-style JSON output with live
+/// per-operator metrics.
+///
+/// This visitor mirrors the logical-plan `PgJsonVisitor` in
+/// `datafusion-expr`: during `pre_visit` it assembles a JSON object for the
+/// current node; during `post_visit` it attaches that object into its
+/// parent's `"Plans"` array (or stores it as the root).
+struct PgJsonExecutionPlanVisitor<'a> {
+ verbose: bool,
+ show_metrics: ShowMetrics,
+ show_schema: bool,
+ metric_types: &'a [MetricType],
+ metric_categories: Option<&'a [MetricCategory]>,
+ objects: HashMap<u32, serde_json::Value>,
+ parent_ids: Vec<u32>,
+ next_id: u32,
+ root: Option<serde_json::Value>,
+}
+
+impl PgJsonExecutionPlanVisitor<'_> {
+ /// Produce the one-line `DisplayAs::Default` rendering of a node.
+ fn one_line_details(plan: &dyn ExecutionPlan) -> String {
+ struct One<'b>(&'b dyn ExecutionPlan);
+ impl fmt::Display for One<'_> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ self.0.fmt_as(DisplayFormatType::Default, f)
+ }
+ }
+ // Some operators include internal newlines; collapse them so the
+ // rendered JSON value stays on a single line.
+ format!("{}", One(plan))
+ .replace('\n', " ")
+ .trim()
+ .to_string()
+ }
+
+ /// Render the given `MetricValue` into the most natural `serde_json::Value`
+ /// we can produce: a number for simple counts/gauges/times, a float-ms for
+ /// `ElapsedCompute`, and a string fallback for anything else.
+ fn metric_value_to_json(value: &MetricValue) -> serde_json::Value {
+ match value {
+ MetricValue::OutputRows(c) => serde_json::Value::from(c.value()),
+ MetricValue::SpillCount(c)
+ | MetricValue::OutputBatches(c)
+ | MetricValue::SpilledRows(c) => serde_json::Value::from(c.value()),
+ MetricValue::SpilledBytes(c) | MetricValue::OutputBytes(c) => {
+ serde_json::Value::from(c.value())
+ }
+ MetricValue::CurrentMemoryUsage(g) => serde_json::Value::from(g.value()),
+ MetricValue::ElapsedCompute(t) => {
+ // Emit as float milliseconds to align with PG's
+ // `"Actual Total Time"` convention. DataFusion tracks compute
+ // time (summed across partitions), not wall time — visualizers
+ // should be read with that caveat in mind.
+ let ms = (t.value() as f64) / 1_000_000.0;
+ serde_json::Value::from(ms)
+ }
+ MetricValue::Count { count, .. } => serde_json::Value::from(count.value()),
+ MetricValue::Gauge { gauge, .. } => serde_json::Value::from(gauge.value()),
+ MetricValue::Time { time, .. } => {
+ let ms = (time.value() as f64) / 1_000_000.0;
+ serde_json::Value::from(ms)
+ }
+ // Timestamps, PruningMetrics, Ratio, Custom: fall back to Display.
+ other => serde_json::Value::String(format!("{other}")),
+ }
+ }
+
+ /// Populate `"Actual Rows"`, `"Actual Total Time"`, and `"Extras"` for
+ /// the given node from its aggregated `MetricsSet`, honoring the same
+ /// filtering pipeline used by `IndentVisitor`.
+ fn attach_metrics(&self, plan: &dyn ExecutionPlan, object: &mut serde_json::Value) {
+ if matches!(self.show_metrics, ShowMetrics::None) {
+ return;
+ }
+ let Some(metrics) = plan.metrics() else {
+ return;
+ };
+
+ let metrics = match self.show_metrics {
+ ShowMetrics::None => return,
+ ShowMetrics::Aggregated => metrics
+ .filter_by_metric_types(self.metric_types)
+ .aggregate_by_name()
+ .sorted_for_display()
+ .timestamps_removed(),
+ ShowMetrics::Full => metrics.filter_by_metric_types(self.metric_types),
+ };
+ let metrics = if let Some(cats) = self.metric_categories {
+ metrics.filter_by_categories(cats)
+ } else {
+ metrics
+ };
+
+ // Build the Extras bucket, while extracting PG-canonical keys to the
+ // top level.
+ let mut extras = serde_json::Map::new();
+ for metric in metrics.iter() {
+ let value = metric.value();
+ match value {
+ MetricValue::OutputRows(c) => {
+ object["Actual Rows"] = serde_json::Value::from(c.value());
+ }
+ MetricValue::ElapsedCompute(t) => {
+ let ms = (t.value() as f64) / 1_000_000.0;
+ object["Actual Total Time"] = serde_json::Value::from(ms);
+ }
+ _ => {
+ extras.insert(
+ value.name().to_string(),
+ Self::metric_value_to_json(value),
+ );
+ }
+ }
+ }
+ if !extras.is_empty() {
+ object["Extras"] = serde_json::Value::Object(extras);
+ }
+ }
+}
+
+impl ExecutionPlanVisitor for PgJsonExecutionPlanVisitor<'_> {
+ type Error = fmt::Error;
+
+ fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {
+ let id = self.next_id;
+ self.next_id += 1;
+
+ // Build fields in reading order: Node Type, Details, (schema),
+ // (metrics), Plans last — so the JSON output reads top-down like a
+ // PostgreSQL plan.
+ let mut object = serde_json::json!({
+ "Node Type": plan.name(),
+ "Details": Self::one_line_details(plan),
+ });
+
+ if self.show_schema || self.verbose {
+ // Always include output columns when a caller asked for schema;
+ // also include them in verbose mode so the pgjson output mirrors
+ // the extra context shown by indent's verbose flag.
+ let columns: Vec<serde_json::Value> = plan
+ .schema()
+ .fields()
+ .iter()
+ .map(|f| serde_json::Value::String(f.name().to_string()))
+ .collect();
+ object["Output"] = serde_json::Value::Array(columns);
+ }
+
+ self.attach_metrics(plan, &mut object);
+
+ object["Plans"] = serde_json::Value::Array(vec![]);
+
+ self.objects.insert(id, object);
+ self.parent_ids.push(id);
+ Ok(true)
+ }
+
+ fn post_visit(&mut self, _plan: &dyn ExecutionPlan) -> Result<bool, Self::Error> {
+ let id = self.parent_ids.pop().ok_or(fmt::Error)?;
+ let current = self.objects.remove(&id).ok_or(fmt::Error)?;
+
+ if let Some(parent_id) = self.parent_ids.last() {
+ let parent = self.objects.get_mut(parent_id).ok_or(fmt::Error)?;
+ let plans = parent
+ .get_mut("Plans")
+ .and_then(|p| p.as_array_mut())
+ .ok_or(fmt::Error)?;
+ plans.push(current);
+ } else {
+ self.root = Some(current);
+ }
+ Ok(true)
+ }
+}
+
/// This module implements a tree-like art renderer for execution plans,
/// based on DuckDB's implementation:
/// <https://github.com/duckdb/duckdb/blob/main/src/include/duckdb/common/tree_renderer/text_tree_renderer.hpp>
@@ -1275,4 +1550,192 @@ mod tests {
fn test_display_when_stats_ok_with_show_stats() {
test_stats_display(TestStatsExecPlan::Ok, false);
}
+
+ mod pgjson {
+ use std::sync::Arc;
+ use std::time::Duration;
+
+ use arrow::datatypes::{DataType, Field, Schema};
+ use insta::assert_snapshot;
+
+ use super::super::DisplayableExecutionPlan;
+ use crate::empty::EmptyExec;
+ use crate::filter::FilterExec;
+ use crate::projection::ProjectionExec;
+ use datafusion_physical_expr::expressions::{binary, col, lit};
+ use datafusion_physical_expr::{Partitioning, PhysicalExpr};
+
+ fn sample_plan() -> Arc<dyn crate::ExecutionPlan> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ ]));
+ let empty = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let predicate = binary(
+ col("a", &schema).unwrap(),
+ datafusion_expr::Operator::Gt,
+ lit(5i32),
+ &schema,
+ )
+ .unwrap();
+ let filter = Arc::new(FilterExec::try_new(predicate, empty).unwrap());
+ let proj_expr: Vec<(Arc<dyn PhysicalExpr>, String)> =
+ vec![(col("a", &schema).unwrap(), "a".to_string())];
+ let _ = Partitioning::UnknownPartitioning(1);
+ Arc::new(ProjectionExec::try_new(proj_expr, filter).unwrap())
+ }
+
+ #[test]
+ fn pgjson_renders_plan_without_metrics() {
+ let plan = sample_plan();
+ let out = DisplayableExecutionPlan::new(plan.as_ref())
+ .pgjson(false)
+ .to_string();
+ let value: serde_json::Value = serde_json::from_str(&out).unwrap();
+ // Root is an array with one {"Plan": ...} entry.
+ let root = value
+ .as_array()
+ .expect("root array")
+ .first()
+ .expect("root entry")
+ .get("Plan")
+ .expect("plan object");
+ assert_eq!(root["Node Type"].as_str(), Some("ProjectionExec"));
+ assert!(root.get("Actual Rows").is_none());
+ assert!(root.get("Extras").is_none());
+ let plans = root["Plans"].as_array().expect("Plans array");
+ assert_eq!(plans.len(), 1);
+ assert_eq!(plans[0]["Node Type"].as_str(), Some("FilterExec"));
+ }
+
+ #[test]
+ fn pgjson_emits_pg_canonical_metric_keys() {
+ use crate::metrics::{Count, Metric, MetricValue, MetricsSet, Time};
+ use crate::{DisplayFormatType, ExecutionPlan, PlanProperties};
+ use datafusion_common::Result;
+ use datafusion_execution::{SendableRecordBatchStream, TaskContext};
+
+ // Wrap `sample_plan()` with an adapter node that exposes a
+ // hand-crafted `MetricsSet` so we can assert the PG key mapping
+ // without running anything.
+ #[derive(Debug)]
+ struct WithMetrics {
+ inner: Arc<dyn ExecutionPlan>,
+ metrics: MetricsSet,
+ }
+ impl crate::DisplayAs for WithMetrics {
+ fn fmt_as(
+ &self,
+ _t: DisplayFormatType,
+ f: &mut std::fmt::Formatter,
+ ) -> std::fmt::Result {
+ write!(f, "WithMetrics")
+ }
+ }
+ impl ExecutionPlan for WithMetrics {
+ fn name(&self) -> &'static str {
+ "WithMetrics"
+ }
+ fn properties(&self) -> &Arc<PlanProperties> {
+ self.inner.properties()
+ }
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ vec![&self.inner]
+ }
+ fn with_new_children(
+ self: Arc<Self>,
+ _: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ unimplemented!()
+ }
+ fn execute(
+ &self,
+ _: usize,
+ _: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ unimplemented!()
+ }
+ fn metrics(&self) -> Option<MetricsSet> {
+ Some(self.metrics.clone())
+ }
+ }
+
+ let mut metrics = MetricsSet::new();
+ let rows = Count::new();
+ rows.add(42);
+ metrics.push(Arc::new(Metric::new(MetricValue::OutputRows(rows), None)));
+ let elapsed = Time::new();
+ elapsed.add_duration(Duration::from_millis(5));
+ metrics.push(Arc::new(Metric::new(
+ MetricValue::ElapsedCompute(elapsed),
+ None,
+ )));
+ let batches = Count::new();
+ batches.add(7);
+ metrics.push(Arc::new(Metric::new(
+ MetricValue::OutputBatches(batches),
+ None,
+ )));
+
+ let plan: Arc<dyn ExecutionPlan> = Arc::new(WithMetrics {
+ inner: sample_plan(),
+ metrics,
+ });
+
+ let out = DisplayableExecutionPlan::with_metrics(plan.as_ref())
+ .pgjson(false)
+ .to_string();
+ let value: serde_json::Value = serde_json::from_str(&out).unwrap();
+ let root = value[0].get("Plan").expect("plan");
+ assert_eq!(root["Actual Rows"].as_u64(), Some(42));
+ assert_eq!(root["Actual Total Time"].as_f64(), Some(5.0));
+ assert_eq!(root["Extras"]["output_batches"].as_u64(), Some(7));
+ }
+
+ #[test]
+ fn pgjson_includes_summary_when_set() {
+ let plan = sample_plan();
+ let out = DisplayableExecutionPlan::with_metrics(plan.as_ref())
+ .set_summary(Some(42), Some(Duration::from_millis(7)))
+ .pgjson(false)
+ .to_string();
+ let value: serde_json::Value = serde_json::from_str(&out).unwrap();
+ let entry = &value.as_array().unwrap()[0];
+ assert_eq!(entry["Total Rows"].as_u64(), Some(42));
+ assert!(entry["Duration"].is_string());
+ }
+
+ #[test]
+ fn pgjson_snapshot_of_sample_plan() {
+ let plan = sample_plan();
+ let out = DisplayableExecutionPlan::new(plan.as_ref())
+ .pgjson(false)
+ .to_string();
+ // This snapshot assumes `serde_json` is built with the
+ // `preserve_order` feature (enabled via this crate's dev-deps).
+ assert_snapshot!(out, @r#"
+ [
+ {
+ "Plan": {
+ "Node Type": "ProjectionExec",
+ "Details": "ProjectionExec: expr=[a@0 as a]",
+ "Plans": [
+ {
+ "Node Type": "FilterExec",
+ "Details": "FilterExec: a@0 > 5",
+ "Plans": [
+ {
+ "Node Type": "EmptyExec",
+ "Details": "EmptyExec",
+ "Plans": []
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ "#);
+ }
+ }
}
diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto
index ebae6c1abb..376d94fa16 100644
--- a/datafusion/proto-models/proto/datafusion.proto
+++ b/datafusion/proto-models/proto/datafusion.proto
@@ -230,6 +230,7 @@ message AnalyzeNode {
// Statement-level override for `datafusion.explain.analyze_categories`.
// Absent means "fall back to session config".
optional datafusion_common.ExplainAnalyzeCategoriesNode analyze_categories = 4;
+ datafusion_common.ExplainFormat format = 5;
}
message ExplainNode {
@@ -1243,6 +1244,7 @@ message AnalyzeExecNode {
// Empty means "plan only". Absent (has_metric_categories=false) means "all".
bool has_metric_categories = 5;
repeated string metric_categories = 6;
+ datafusion_common.ExplainFormat format = 7;
}
message CrossJoinExecNode {
diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs
index 6e1901b1e4..59860a57ac 100644
--- a/datafusion/proto-models/src/generated/pbjson.rs
+++ b/datafusion/proto-models/src/generated/pbjson.rs
@@ -999,6 +999,9 @@ impl serde::Serialize for AnalyzeExecNode {
if !self.metric_categories.is_empty() {
len += 1;
}
+ if self.format != 0 {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.AnalyzeExecNode", len)?;
if self.verbose {
struct_ser.serialize_field("verbose", &self.verbose)?;
@@ -1018,6 +1021,11 @@ impl serde::Serialize for AnalyzeExecNode {
if !self.metric_categories.is_empty() {
struct_ser.serialize_field("metricCategories", &self.metric_categories)?;
}
+ if self.format != 0 {
+ let v = super::datafusion_common::ExplainFormat::try_from(self.format)
+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.format)))?;
+ struct_ser.serialize_field("format", &v)?;
+ }
struct_ser.end()
}
}
@@ -1037,6 +1045,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeExecNode {
"hasMetricCategories",
"metric_categories",
"metricCategories",
+ "format",
];
#[allow(clippy::enum_variant_names)]
@@ -1047,6 +1056,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeExecNode {
Schema,
HasMetricCategories,
MetricCategories,
+ Format,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -1074,6 +1084,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeExecNode {
"schema" => Ok(GeneratedField::Schema),
"hasMetricCategories" | "has_metric_categories" => Ok(GeneratedField::HasMetricCategories),
"metricCategories" | "metric_categories" => Ok(GeneratedField::MetricCategories),
+ "format" => Ok(GeneratedField::Format),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -1099,6 +1110,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeExecNode {
let mut schema__ = None;
let mut has_metric_categories__ = None;
let mut metric_categories__ = None;
+ let mut format__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::Verbose => {
@@ -1137,6 +1149,12 @@ impl<'de> serde::Deserialize<'de> for AnalyzeExecNode {
}
metric_categories__ = Some(map_.next_value()?);
}
+ GeneratedField::Format => {
+ if format__.is_some() {
+ return Err(serde::de::Error::duplicate_field("format"));
+ }
+ format__ = Some(map_.next_value::<super::datafusion_common::ExplainFormat>()? as i32);
+ }
}
}
Ok(AnalyzeExecNode {
@@ -1146,6 +1164,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeExecNode {
schema: schema__,
has_metric_categories: has_metric_categories__.unwrap_or_default(),
metric_categories: metric_categories__.unwrap_or_default(),
+ format: format__.unwrap_or_default(),
})
}
}
@@ -1172,6 +1191,9 @@ impl serde::Serialize for AnalyzeNode {
if self.analyze_categories.is_some() {
len += 1;
}
+ if self.format != 0 {
+ len += 1;
+ }
let mut struct_ser = serializer.serialize_struct("datafusion.AnalyzeNode", len)?;
if let Some(v) = self.input.as_ref() {
struct_ser.serialize_field("input", v)?;
@@ -1187,6 +1209,11 @@ impl serde::Serialize for AnalyzeNode {
if let Some(v) = self.analyze_categories.as_ref() {
struct_ser.serialize_field("analyzeCategories", v)?;
}
+ if self.format != 0 {
+ let v = super::datafusion_common::ExplainFormat::try_from(self.format)
+ .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.format)))?;
+ struct_ser.serialize_field("format", &v)?;
+ }
struct_ser.end()
}
}
@@ -1203,6 +1230,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {
"analyzeLevel",
"analyze_categories",
"analyzeCategories",
+ "format",
];
#[allow(clippy::enum_variant_names)]
@@ -1211,6 +1239,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {
Verbose,
AnalyzeLevel,
AnalyzeCategories,
+ Format,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -1236,6 +1265,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {
"verbose" => Ok(GeneratedField::Verbose),
"analyzeLevel" | "analyze_level" => Ok(GeneratedField::AnalyzeLevel),
"analyzeCategories" | "analyze_categories" => Ok(GeneratedField::AnalyzeCategories),
+ "format" => Ok(GeneratedField::Format),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -1259,6 +1289,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {
let mut verbose__ = None;
let mut analyze_level__ = None;
let mut analyze_categories__ = None;
+ let mut format__ = None;
while let Some(k) = map_.next_key()? {
match k {
GeneratedField::Input => {
@@ -1285,6 +1316,12 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {
}
analyze_categories__ = map_.next_value()?;
}
+ GeneratedField::Format => {
+ if format__.is_some() {
+ return Err(serde::de::Error::duplicate_field("format"));
+ }
+ format__ = Some(map_.next_value::<super::datafusion_common::ExplainFormat>()? as i32);
+ }
}
}
Ok(AnalyzeNode {
@@ -1292,6 +1329,7 @@ impl<'de> serde::Deserialize<'de> for AnalyzeNode {
verbose: verbose__.unwrap_or_default(),
analyze_level: analyze_level__,
analyze_categories: analyze_categories__,
+ format: format__.unwrap_or_default(),
})
}
}
diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs
index d2b25695cb..022acdfda7 100644
--- a/datafusion/proto-models/src/generated/prost.rs
+++ b/datafusion/proto-models/src/generated/prost.rs
@@ -354,6 +354,8 @@ pub struct AnalyzeNode {
pub analyze_categories: ::core::option::Option<
super::datafusion_common::ExplainAnalyzeCategoriesNode,
>,
+ #[prost(enumeration = "super::datafusion_common::ExplainFormat", tag = "5")]
+ pub format: i32,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ExplainNode {
@@ -1852,6 +1854,8 @@ pub struct AnalyzeExecNode {
pub has_metric_categories: bool,
#[prost(string, repeated, tag = "6")]
pub metric_categories: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+ #[prost(enumeration = "super::datafusion_common::ExplainFormat", tag = "7")]
+ pub format: i32,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CrossJoinExecNode {
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index d60ce130ca..9bb6e74329 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -874,12 +874,26 @@ impl AsLogicalPlan for LogicalPlanNode {
.as_ref()
.map(explain_analyze_categories_from_proto)
.transpose()?;
+ let pb_format = protobuf::ExplainFormat::try_from(analyze.format)
+ .map_err(|_| {
+ proto_error(format!(
+ "Received an AnalyzeNode message with unknown ExplainFormat {}",
+ analyze.format
+ ))
+ })?;
+ let analyze_format = match pb_format {
+ protobuf::ExplainFormat::Indent => ExplainFormat::Indent,
+ protobuf::ExplainFormat::Tree => ExplainFormat::Tree,
+ protobuf::ExplainFormat::Pgjson => ExplainFormat::PostgresJSON,
+ protobuf::ExplainFormat::Graphviz => ExplainFormat::Graphviz,
+ };
let explain_option =
datafusion_expr::logical_plan::ExplainOption::default()
.with_verbose(analyze.verbose)
.with_analyze(true)
.with_analyze_level(analyze_level)
- .with_analyze_categories(analyze_categories);
+ .with_analyze_categories(analyze_categories)
+ .with_format(analyze_format);
LogicalPlanBuilder::from(input)
.explain_option_format(explain_option)?
.build()
@@ -1878,6 +1892,16 @@ impl AsLogicalPlan for LogicalPlanNode {
.analyze_categories
.as_ref()
.map(explain_analyze_categories_to_proto),
+ format: match &a.format {
+ ExplainFormat::Indent => protobuf::ExplainFormat::Indent,
+ ExplainFormat::Tree => protobuf::ExplainFormat::Tree,
+ ExplainFormat::PostgresJSON => {
+ protobuf::ExplainFormat::Pgjson
+ }
+ ExplainFormat::Graphviz => {
+ protobuf::ExplainFormat::Graphviz
+ }
+ } as i32,
},
))),
})
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 89871318ee..9efcd25fcb 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -26,6 +26,7 @@ use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef};
use datafusion_catalog::memory::MemorySourceConfig;
use datafusion_common::config::CsvOptions;
use datafusion_common::display::StringifiedPlan;
+use datafusion_common::format::ExplainFormat;
use datafusion_common::{
DataFusionError, JoinType, NullEquality, Result, internal_datafusion_err,
internal_err, not_impl_err,
@@ -82,7 +83,7 @@ use datafusion_physical_plan::joins::{
};
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::memory::LazyMemoryExec;
-use datafusion_physical_plan::metrics::{MetricCategory, MetricType};
+use datafusion_physical_plan::metrics::MetricCategory;
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion_physical_plan::repartition::RepartitionExec;
@@ -1962,14 +1963,30 @@ pub trait PhysicalPlanNodeExt: Sized {
} else {
None
};
- Ok(Arc::new(AnalyzeExec::new(
- analyze.verbose,
- analyze.show_statistics,
- vec![MetricType::Summary, MetricType::Dev],
- metric_categories,
- input,
- Arc::new(convert_required!(analyze.schema)?),
- )))
+ let pb_format =
+ protobuf::ExplainFormat::try_from(analyze.format).map_err(|_| {
+ DataFusionError::Internal(format!(
+ "Received an AnalyzeExecNode message with unknown ExplainFormat {}",
+ analyze.format
+ ))
+ })?;
+ let format = match pb_format {
+ protobuf::ExplainFormat::Indent => ExplainFormat::Indent,
+ protobuf::ExplainFormat::Tree => ExplainFormat::Tree,
+ protobuf::ExplainFormat::Pgjson => ExplainFormat::PostgresJSON,
+ protobuf::ExplainFormat::Graphviz => ExplainFormat::Graphviz,
+ };
+ Ok(Arc::new(
+ AnalyzeExec::builder(
+ analyze.verbose,
+ analyze.show_statistics,
+ input,
+ Arc::new(convert_required!(analyze.schema)?),
+ )
+ .with_metric_categories(metric_categories)
+ .with_format(format)
+ .build(),
+ ))
}
fn try_into_json_sink_physical_plan(
@@ -2463,6 +2480,12 @@ pub trait PhysicalPlanNodeExt: Sized {
Some(cats) => (true, cats.iter().map(|c| c.to_string()).collect()),
None => (false, vec![]),
};
+ let format = match exec.format() {
+ ExplainFormat::Indent => protobuf::ExplainFormat::Indent,
+ ExplainFormat::Tree => protobuf::ExplainFormat::Tree,
+ ExplainFormat::PostgresJSON => protobuf::ExplainFormat::Pgjson,
+ ExplainFormat::Graphviz => protobuf::ExplainFormat::Graphviz,
+ } as i32;
Ok(protobuf::PhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::Analyze(Box::new(
protobuf::AnalyzeExecNode {
@@ -2472,6 +2495,7 @@ pub trait PhysicalPlanNodeExt: Sized {
schema: Some(exec.schema().as_ref().try_into()?),
has_metric_categories,
metric_categories,
+ format,
},
))),
})
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
index bd996eb692..8e80467788 100644
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
@@ -74,7 +74,6 @@ use datafusion::physical_plan::joins::{
StreamJoinPartitionMode, SymmetricHashJoinExec,
};
use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
-use datafusion::physical_plan::metrics::MetricType;
use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion::physical_plan::repartition::RepartitionExec;
@@ -1558,14 +1557,9 @@ fn roundtrip_analyze() -> Result<()> {
let schema = Schema::new(vec![field_a, field_b]);
let input = Arc::new(PlaceholderRowExec::new(Arc::new(schema.clone())));
- roundtrip_test(Arc::new(AnalyzeExec::new(
- false,
- false,
- vec![MetricType::Summary, MetricType::Dev],
- None,
- input,
- Arc::new(schema),
- )))
+ roundtrip_test(Arc::new(
+ AnalyzeExec::builder(false, false, input, Arc::new(schema)).build(),
+ ))
}
#[tokio::test]
diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
index 00fd3ddf9e..389dce0875 100644
--- a/datafusion/sql/src/statement.rs
+++ b/datafusion/sql/src/statement.rs
@@ -2049,12 +2049,43 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
return plan_err!("EXPLAIN option COSTS cannot be combined with ANALYZE");
}
+ // Resolve the requested output format.
+ //
+ // Verbose mode only supports indent format, and for EXPLAIN ANALYZE
+ // only `Indent` and `PostgresJSON` are supported today — `Tree` and
+ // `Graphviz` require additional work to render with live metrics.
+ let options = self.context_provider.options();
+ let format = if verbose {
+ ExplainFormat::Indent
+ } else if let Some(format) = format {
+ format
+ } else if analyze {
+ ExplainFormat::Indent
+ } else {
+ options.explain.format.clone()
+ };
+
if analyze {
- if format.is_some() {
- return plan_err!("EXPLAIN ANALYZE with FORMAT is not supported");
+ match &format {
+ ExplainFormat::Indent => {}
+ ExplainFormat::PostgresJSON => {
+ // The pgjson renderer does not emit statistics yet, so
+ // reject the combination rather than silently ignoring it.
+ if options.explain.show_statistics {
+ return plan_err!(
+ "EXPLAIN ANALYZE with FORMAT pgjson does not support show_statistics"
+ );
+ }
+ }
+ ExplainFormat::Tree | ExplainFormat::Graphviz => {
+ return plan_err!(
+ "EXPLAIN ANALYZE with FORMAT {format} is not supported"
+ );
+ }
}
Ok(LogicalPlan::Analyze(Analyze {
verbose,
+ format,
input: plan,
schema,
analyze_level,
@@ -2064,17 +2095,6 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
let stringified_plans =
vec![plan.to_stringified(PlanType::InitialLogicalPlan)];
- // default to configuration value
- // verbose mode only supports indent format
- let options = self.context_provider.options();
- let format = if verbose {
- ExplainFormat::Indent
- } else if let Some(format) = format {
- format
- } else {
- options.explain.format.clone()
- };
-
Ok(LogicalPlan::Explain(Explain {
verbose,
explain_format: format,
diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.slt
index 9d250ebea1..24b1262e02 100644
--- a/datafusion/sqllogictest/test_files/explain.slt
+++ b/datafusion/sqllogictest/test_files/explain.slt
@@ -744,9 +744,10 @@ EXPLAIN (TIMING ON) SELECT 1;
statement error DataFusion error: Error during planning: EXPLAIN option LEVEL requires ANALYZE
EXPLAIN (SUMMARY ON) SELECT 1;
-# FORMAT is incompatible with both ANALYZE and VERBOSE (same as the legacy
-# keyword form — these mappings come from the planner, not the parser).
-statement error DataFusion error: Error during planning: EXPLAIN ANALYZE with FORMAT is not supported
+# VERBOSE is incompatible with any FORMAT, and ANALYZE only supports the
+# `indent` and `pgjson` formats — `tree` and `graphviz` are rejected (these
+# mappings come from the planner, not the parser).
+statement error DataFusion error: Error during planning: EXPLAIN ANALYZE with FORMAT tree is not supported
EXPLAIN (ANALYZE, FORMAT tree) SELECT 1;
statement error DataFusion error: Error during planning: EXPLAIN VERBOSE with FORMAT is not supported
diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.slt
index e3bd83d569..b1856e0add 100644
--- a/datafusion/sqllogictest/test_files/explain_analyze.slt
+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt
@@ -391,9 +391,57 @@ Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3]
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=21.75% (485/2.23 K)]
+# ---- pgjson format: structural golden with no metrics ----
+
+query TT
+EXPLAIN (ANALYZE, FORMAT PGJSON, METRICS 'none') SELECT * FROM (VALUES (1), (2), (3)) t(x);
+----
+Plan with Metrics
+01)[
+02)--{
+03)----"Plan": {
+04)------"Node Type": "ProjectionExec",
+05)------"Details": "ProjectionExec: expr=[column1@0 as x]",
+06)------"Plans": [
+07)--------{
+08)----------"Node Type": "DataSourceExec",
+09)----------"Details": "DataSourceExec: partitions=1, partition_sizes=[1]",
+10)----------"Plans": []
+11)--------}
+12)------]
+13)----}
+14)--}
+15)]
+
statement ok
reset datafusion.explain.analyze_categories;
+# ---- pgjson with METRICS 'rows': row-count surfaces as Actual Rows ----
+
+query TT
+EXPLAIN (ANALYZE, FORMAT PGJSON, METRICS 'rows') SELECT * FROM (VALUES (1), (2), (3)) t(x);
+----
+Plan with Metrics
+01)[
+02)--{
+03)----"Plan": {
+04)------"Node Type": "ProjectionExec",
+05)------"Details": "ProjectionExec: expr=[column1@0 as x]",
+06)------"Actual Rows": 3,
+07)------"Extras": {
+08)--------"output_batches": 1
+09)------},
+10)------"Plans": [
+11)--------{
+12)----------"Node Type": "DataSourceExec",
+13)----------"Details": "DataSourceExec: partitions=1, partition_sizes=[1]",
+14)----------"Plans": []
+15)--------}
+16)------]
+17)----}
+18)--}
+19)]
+
# ---- Argument syntax variants for METRICS ----
# Bare identifier, `= value`, and quoted string forms should all parse
# to the same selection.
@@ -408,6 +456,25 @@ Plan with Metrics
statement ok
reset datafusion.sql_parser.dialect;
+# ---- Reject formats that AnalyzeExec cannot render with live metrics ----
+
+query error DataFusion error: Error during planning: EXPLAIN ANALYZE with FORMAT tree is not supported
+explain analyze format tree select 1;
+
+query error DataFusion error: Error during planning: EXPLAIN ANALYZE with FORMAT graphviz is not supported
+explain analyze format graphviz select 1;
+
+# ---- pgjson does not render statistics yet, so reject show_statistics ----
+
+statement ok
+set datafusion.explain.show_statistics = true;
+
+query error DataFusion error: Error during planning: EXPLAIN ANALYZE with FORMAT pgjson does not support show_statistics
+explain analyze format pgjson select 1;
+
+statement ok
+reset datafusion.explain.show_statistics;
+
# --- Teardown ---
statement ok
diff --git a/docs/source/user-guide/explain-usage.md b/docs/source/user-guide/explain-usage.md
index 0f3f008c52..40ff369b58 100644
--- a/docs/source/user-guide/explain-usage.md
+++ b/docs/source/user-guide/explain-usage.md
@@ -270,6 +270,15 @@ The recognized options are:
Boolean arguments can be written bare (`ANALYZE` → `true`), as `TRUE`/`FALSE`,
`ON`/`OFF`, or `0`/`1`.
+When combined with `ANALYZE`, `FORMAT` supports `indent` (the default) and
+`pgjson`; `tree` and `graphviz` are rejected. The `pgjson` form emits the
+physical plan with live metrics, which is handy for plan visualizers — see
+[`pgjson` format with `ANALYZE`](sql/explain.md#pgjson-format-with-analyze):
+
+```sql
+EXPLAIN (ANALYZE, FORMAT pgjson) SELECT ...;
+```
+
The statement-level options take precedence over session config, so you can leave
the session defaults alone and override just for the current query:
diff --git a/docs/source/user-guide/sql/explain.md b/docs/source/user-guide/sql/explain.md
index 2310163262..e7be47b350 100644
--- a/docs/source/user-guide/sql/explain.md
+++ b/docs/source/user-guide/sql/explain.md
@@ -227,8 +227,9 @@ Elapsed 0.010 seconds.
## `EXPLAIN ANALYZE`
-Shows the execution plan and metrics of a statement. Note that `EXPLAIN ANALYZE`
-only supports the `indent` format.
+Shows the execution plan and metrics of a statement. `EXPLAIN ANALYZE` supports
+the `indent` format (the default) and the [`pgjson`](#pgjson-format-with-analyze)
+format; the `tree` and `graphviz` formats are not supported with `ANALYZE`.
```sql
EXPLAIN ANALYZE SELECT SUM(x) FROM table GROUP BY b;
@@ -251,4 +252,46 @@ By default `EXPLAIN ANALYZE` shows the aggregated metrics from all partitions fo
You can also set `datafusion.explain.analyze_level` from the [configuration value] to control the detail level for the metrics displayed.
+### `pgjson` format with `ANALYZE`
+
+`EXPLAIN ANALYZE` can also emit the physical plan and its live execution
+metrics in the [`pgjson`](#pgjson-format) format, so the analyzed plan can be
+loaded into PostgreSQL plan visualizers such as [dalibo]. Each node reports its
+`Actual Rows` and `Actual Total Time` (compute time, in milliseconds) using the
+PostgreSQL key names, with any remaining DataFusion metrics under `Extras`.
+
+The format can be requested with either the keyword form
+(`EXPLAIN ANALYZE FORMAT pgjson ...`) or, more idiomatically, the PostgreSQL
+[option-list form](../explain-usage.md), which also lets you combine it with the
+`METRICS` and `LEVEL` knobs in a single statement:
+
+```sql
+> CREATE TABLE t(x int, b int) AS VALUES (1, 2), (2, 3);
+> EXPLAIN (ANALYZE, FORMAT pgjson, METRICS 'rows') SELECT x FROM t WHERE b > 2;
++-------------------+---------------------------------------------------------------------------+
+| plan_type | plan |
++-------------------+---------------------------------------------------------------------------+
+| Plan with Metrics | [ |
+| | { |
+| | "Plan": { |
+| | "Node Type": "FilterExec", |
+| | "Details": "FilterExec: b@1 > 2, projection=[x@0]", |
+| | "Actual Rows": 1, |
+| | "Extras": { |
+| | "output_batches": 1, |
+| | "selectivity": "50% (1/2)" |
+| | }, |
+| | "Plans": [ |
+| | { |
+| | "Node Type": "DataSourceExec", |
+| | "Details": "DataSourceExec: partitions=1, partition_sizes=[1]", |
+| | "Plans": [] |
+| | } |
+| | ] |
+| | } |
+| | } |
+| | ] |
++-------------------+---------------------------------------------------------------------------+
+```
+
[configuration value]: ../configs.md