Apache DataFusion SQL Query Engine
fix: Remove `serde_json/preserve_order` feature from library crates (#25884)
Adrian Garcia Badaracco284f2114948e
diff --git a/Cargo.lock b/Cargo.lockindex 0ea11794b1..28eeb6f6dd 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2214,6 +2214,7 @@ dependencies = ["insta","itertools 0.15.0","recursive",+ "serde","serde_json","sqlparser",]@@ -2559,6 +2560,7 @@ dependencies = ["rand 0.9.4","rstest","rstest_reuse",+ "serde","serde_json","tokio",]diff --git a/Cargo.toml b/Cargo.tomlindex 622c947d7d..8da1274aa9 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -197,6 +197,7 @@ rand = "0.9"recursive = "0.1.1"regex = "1.12"rstest = "0.27.0"+serde = { version = "1", features = ["derive"] }serde_json = "1"sha2 = "^0.11.0"sqlparser = { version = "0.63.0", default-features = false, features = ["std", "visitor"] }diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.tomlindex 74b85501dd..45b66ee898 100644--- a/datafusion/expr/Cargo.toml+++ b/datafusion/expr/Cargo.toml@@ -65,6 +65,7 @@ datafusion-proto-models = { workspace = true, optional = true }indexmap = { workspace = true }itertools = { workspace = true }recursive = { workspace = true, optional = true }+serde = { workspace = true }serde_json = { workspace = true }sqlparser = { workspace = true, optional = true }@@ -73,9 +74,6 @@ criterion = { workspace = true }ctor = { workspace = true }env_logger = { workspace = true }insta = { workspace = true }-# Makes sure `test_display_pg_json` behaves in a consistent way regardless of-# feature unification with dependencies-serde_json = { workspace = true, features = ["preserve_order"] }[[bench]]name = "normalize_columns"diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rsindex 1d95bf894b..53c027d0f7 100644--- a/datafusion/expr/src/logical_plan/display.rs+++ b/datafusion/expr/src/logical_plan/display.rs@@ -32,7 +32,8 @@ use arrow::datatypes::Schema;use datafusion_common::display::GraphvizBuilder;use datafusion_common::tree_node::{TreeNodeRecursion, TreeNodeVisitor};use datafusion_common::{Column, DataFusionError, internal_datafusion_err};-use serde_json::json;+use serde::Serialize;+use serde::ser::SerializeMap;/// Formats plans with a single line per node. For example:///@@ -220,6 +221,53 @@ impl<'n> TreeNodeVisitor<'n> for GraphvizVisitor<'_, '_> {}}+/// Builds a [`PgJsonFields`] from `"key": value` pairs, keeping their order.+macro_rules! pg_fields {+ ($($key:literal : $value:expr),* $(,)?) => {+ PgJsonFields(vec![$(($key, serde_json::json!($value))),*])+ };+}++/// Node-specific fields of a `pgjson` node, serialized as a JSON object whose+/// keys keep their insertion order (independent of `serde_json`'s+/// `preserve_order` feature).+struct PgJsonFields(Vec<(&'static str, serde_json::Value)>);++impl PgJsonFields {+ fn push(&mut self, key: &'static str, value: impl Into<serde_json::Value>) {+ self.0.push((key, value.into()));+ }+}++/// One node of the `pgjson` output: the node-specific fields, then+/// `"Plans"`, then `"Output"` (if the schema is shown).+struct PgJsonNode {+ fields: PgJsonFields,+ plans: Vec<PgJsonNode>,+ output: Option<Vec<String>>,+}++impl Serialize for PgJsonNode {+ fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ let mut map = serializer.serialize_map(None)?;+ for (key, value) in &self.fields.0 {+ map.serialize_entry(key, value)?;+ }+ map.serialize_entry("Plans", &self.plans)?;+ if let Some(output) = &self.output {+ map.serialize_entry("Output", output)?;+ }+ map.end()+ }+}++/// Top-level entry of the `pgjson` output.+#[derive(Serialize)]+struct PgJsonRoot {+ #[serde(rename = "Plan")]+ plan: PgJsonNode,+}+/// Formats plans to display as postgresql plan json format.////// There are already many existing visualizer for this format, for example [dalibo](https://explain.dalibo.com/).@@ -271,7 +319,7 @@ pub struct PgJsonVisitor<'a, 'b> {f: &'a mut fmt::Formatter<'b>,/// A mapping from plan node id to the plan node json representation.- objects: HashMap<u32, serde_json::Value>,+ objects: HashMap<u32, PgJsonNode>,next_id: u32,@@ -300,18 +348,18 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {}/// Converts a logical plan node to a json object.- fn to_json_value(node: &LogicalPlan) -> serde_json::Value {+ fn to_json_value(node: &LogicalPlan) -> PgJsonFields {match node {LogicalPlan::EmptyRelation(_) => {- json!({+ pg_fields!("Node Type": "EmptyRelation",- })+ )}LogicalPlan::RecursiveQuery(RecursiveQuery { is_distinct, .. }) => {- json!({+ pg_fields!("Node Type": "RecursiveQuery","Is Distinct": is_distinct,- })+ )}LogicalPlan::Values(Values { values, .. }) => {let str_values = values@@ -332,10 +380,10 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {let eclipse = if values.len() > 5 { "..." } else { "" };let values_str = format!("{str_values}{eclipse}");- json!({+ pg_fields!("Node Type": "Values","Values": values_str- })+ )}LogicalPlan::TableScan(TableScan {source,@@ -344,17 +392,17 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {fetch,..}) => {- let mut object = json!({+ let mut object = pg_fields!("Node Type": "TableScan","Relation Name": table_name.table(),- });+ );if let Some(s) = table_name.schema() {- object["Schema"] = serde_json::Value::String(s.to_string());+ object.push("Schema", serde_json::Value::String(s.to_string()));}if let Some(c) = table_name.catalog() {- object["Catalog"] = serde_json::Value::String(c.to_string());+ object.push("Catalog", serde_json::Value::String(c.to_string()));}if !filters.is_empty() {@@ -378,37 +426,43 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {}if !full_filter.is_empty() {- object["Full Filters"] =- serde_json::Value::String(expr_vec_fmt!(full_filter));+ object.push(+ "Full Filters",+ serde_json::Value::String(expr_vec_fmt!(full_filter)),+ );}if !partial_filter.is_empty() {- object["Partial Filters"] =- serde_json::Value::String(expr_vec_fmt!(partial_filter));+ object.push(+ "Partial Filters",+ serde_json::Value::String(expr_vec_fmt!(partial_filter)),+ );}if !unsupported_filters.is_empty() {- object["Unsupported Filters"] =- serde_json::Value::String(expr_vec_fmt!(unsupported_filters));+ object.push(+ "Unsupported Filters",+ serde_json::Value::String(expr_vec_fmt!(unsupported_filters)),+ );}}if let Some(f) = fetch {- object["Fetch"] = serde_json::Value::Number((*f).into());+ object.push("Fetch", serde_json::Value::Number((*f).into()));}object}LogicalPlan::Projection(Projection { expr, .. }) => {- json!({+ pg_fields!("Node Type": "Projection","Expressions": expr.iter().map(|e| e.to_string()).collect::<Vec<_>>()- })+ )}LogicalPlan::Dml(DmlStatement { table_name, op, .. }) => {- json!({+ pg_fields!("Node Type": "Projection","Operation": op.name(),"Table Name": table_name.table()- })+ )}LogicalPlan::Copy(CopyTo {input: _,@@ -423,52 +477,52 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {.map(|(k, v)| format!("{k}={v}")).collect::<Vec<_>>().join(", ");- json!({+ pg_fields!("Node Type": "CopyTo","Output URL": output_url,"File Type": file_type.get_ext(),"Options": op_str- })+ )}LogicalPlan::Ddl(ddl) => {- json!({+ pg_fields!("Node Type": "Ddl","Operation": format!("{}", ddl.display())- })+ )}LogicalPlan::Filter(Filter {predicate: expr, ..}) => {- json!({+ pg_fields!("Node Type": "Filter","Condition": format!("{}", expr)- })+ )}LogicalPlan::Window(Window { window_expr, .. }) => {- json!({+ pg_fields!("Node Type": "WindowAggr","Expressions": expr_vec_fmt!(window_expr)- })+ )}LogicalPlan::Aggregate(Aggregate {group_expr,aggr_expr,..}) => {- json!({+ pg_fields!("Node Type": "Aggregate","Group By": expr_vec_fmt!(group_expr),"Aggregates": expr_vec_fmt!(aggr_expr)- })+ )}LogicalPlan::Sort(Sort { expr, fetch, .. }) => {- let mut object = json!({+ let mut object = pg_fields!("Node Type": "Sort","Sort Key": expr_vec_fmt!(expr),- });+ );if let Some(fetch) = fetch {- object["Fetch"] = serde_json::Value::Number((*fetch).into());+ object.push("Fetch", serde_json::Value::Number((*fetch).into()));}object@@ -486,12 +540,12 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {.as_ref().map(|expr| format!(" Filter: {expr}")).unwrap_or_else(|| "".to_string());- json!({+ pg_fields!("Node Type": format!("{} Join", join_type),"Join Constraint": format!("{:?}", join_constraint),"Join Keys": join_expr.join(", "),"Filter": filter_expr.to_string()- })+ )}LogicalPlan::AsOfJoin(AsOfJoin {on,@@ -501,34 +555,34 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {}) => {let join_expr: Vec<String> =on.iter().map(|(l, r)| format!("{l} = {r}")).collect();- json!({+ pg_fields!("Node Type": "AsOf Join","Join Constraint": format!("{join_constraint:?}"),"Join Keys": join_expr.join(", "),- "Match Condition": match_condition.to_string(),- })+ "Match Condition": match_condition.to_string()+ )}LogicalPlan::Repartition(Repartition {partitioning_scheme,..}) => match partitioning_scheme {Partitioning::RoundRobinBatch(n) => {- json!({+ pg_fields!("Node Type": "Repartition","Partitioning Scheme": "RoundRobinBatch","Partition Count": n- })+ )}Partitioning::Hash(expr, n) => {let hash_expr: Vec<String> =expr.iter().map(|e| format!("{e}")).collect();- json!({+ pg_fields!("Node Type": "Repartition","Partitioning Scheme": "Hash","Partition Count": n,"Partitioning Key": hash_expr- })+ )}Partitioning::Range(range) => {let range_expr: Vec<String> =@@ -539,60 +593,56 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {.map(|e| format!("{e}")).collect();- json!({+ pg_fields!("Node Type": "Repartition","Partitioning Scheme": "Range","Partition Count": range.partition_count(),"Partitioning Key": range_expr,"Split Points": split_points- })+ )}Partitioning::DistributeBy(expr) => {let dist_by_expr: Vec<String> =expr.iter().map(|e| format!("{e}")).collect();- json!({+ pg_fields!("Node Type": "Repartition","Partitioning Scheme": "DistributeBy","Partitioning Key": dist_by_expr- })+ )}},LogicalPlan::Limit(Limit { skip, fetch, .. }) => {- let mut object = serde_json::json!(- {- "Node Type": "Limit",- }- );+ let mut object = pg_fields!("Node Type": "Limit");if let Some(s) = skip {- object["Skip"] = s.to_string().into()+ object.push("Skip", s.to_string());}if let Some(f) = fetch {- object["Fetch"] = f.to_string().into()+ object.push("Fetch", f.to_string());}object}LogicalPlan::Subquery(Subquery { .. }) => {- json!({+ pg_fields!("Node Type": "Subquery"- })+ )}LogicalPlan::SubqueryAlias(SubqueryAlias { alias, .. }) => {- json!({+ pg_fields!("Node Type": "Subquery","Alias": alias.table(),- })+ )}LogicalPlan::Statement(statement) => {- json!({+ pg_fields!("Node Type": "Statement","Statement": format!("{}", statement.display())- })+ )}LogicalPlan::Distinct(distinct) => match distinct {Distinct::All(_) => {- json!({+ pg_fields!("Node Type": "DistinctAll"- })+ )}Distinct::On(DistinctOn {on_expr,@@ -600,44 +650,46 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {sort_expr,..}) => {- let mut object = json!({+ let mut object = pg_fields!("Node Type": "DistinctOn","On": expr_vec_fmt!(on_expr),"Select": expr_vec_fmt!(select_expr),- });+ );if let Some(sort_expr) = sort_expr {- object["Sort"] =- serde_json::Value::String(expr_vec_fmt!(sort_expr));+ object.push(+ "Sort",+ serde_json::Value::String(expr_vec_fmt!(sort_expr)),+ );}object}},LogicalPlan::Explain { .. } => {- json!({+ pg_fields!("Node Type": "Explain"- })+ )}LogicalPlan::Analyze { .. } => {- json!({+ pg_fields!("Node Type": "Analyze"- })+ )}LogicalPlan::Union(_) => {- json!({+ pg_fields!("Node Type": "Union"- })+ )}LogicalPlan::Extension(e) => {- json!({+ pg_fields!("Node Type": e.node.name(),"Detail": format!("{:?}", e.node)- })+ )}LogicalPlan::DescribeTable(DescribeTable { .. }) => {- json!({+ pg_fields!("Node Type": "DescribeTable"- })+ )}LogicalPlan::Unnest(Unnest {input: plan,@@ -656,11 +708,11 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {.iter().map(|i| &input_columns[*i]).collect::<Vec<&Column>>();- json!({+ pg_fields!("Node Type": "Unnest","ListColumn": expr_vec_fmt!(list_type_columns),"StructColumn": expr_vec_fmt!(struct_type_columns),- })+ )}}}@@ -675,20 +727,17 @@ impl<'n> TreeNodeVisitor<'n> for PgJsonVisitor<'_, '_> {) -> datafusion_common::Result<TreeNodeRecursion> {let id = self.next_id;self.next_id += 1;- let mut object = Self::to_json_value(node);-- object["Plans"] = serde_json::Value::Array(vec![]);-- if self.with_schema {- object["Output"] = serde_json::Value::Array(+ let object = PgJsonNode {+ fields: Self::to_json_value(node),+ plans: vec![],+ output: self.with_schema.then(|| {node.schema().fields().iter().map(|f| f.name().to_string())- .map(serde_json::Value::String)- .collect(),- );- }+ .collect()+ }),+ };self.objects.insert(id, object);self.parent_ids.push(id);@@ -711,15 +760,10 @@ impl<'n> TreeNodeVisitor<'n> for PgJsonVisitor<'_, '_> {.objects.get_mut(parent_id).expect("Missing parent node!");- let plans = parent_node- .get_mut("Plans")- .and_then(|p| p.as_array_mut())- .expect("Plans should be an array");-- plans.push(current_node);+ parent_node.plans.push(current_node);} else {// This is the root node- let plan = serde_json::json!([{"Plan": current_node}]);+ let plan = [PgJsonRoot { plan: current_node }];write!(self.f,"{}",diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.tomlindex 17e98a4764..73b20f422b 100644--- a/datafusion/physical-plan/Cargo.toml+++ b/datafusion/physical-plan/Cargo.toml@@ -87,7 +87,8 @@ log = { workspace = true }num-traits = { workspace = true }parking_lot = { workspace = true }pin-project-lite = { workspace = true }-serde_json = { workspace = true, features = ["preserve_order"] }+serde = { workspace = true }+serde_json = { workspace = true }tokio = { workspace = true, features = ["time"] }[dev-dependencies]diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rsindex 44522e76af..37f21ad4d6 100644--- a/datafusion/physical-plan/src/display.rs+++ b/datafusion/physical-plan/src/display.rs@@ -23,6 +23,8 @@ use std::fmt;use std::fmt::Formatter;use std::time::Duration;+use serde::Serialize;+use arrow::datatypes::SchemaRef;use datafusion_common::display::GraphvizBuilder;@@ -448,17 +450,14 @@ impl<'a> DisplayableExecutionPlan<'a> {};accept(self.plan, &mut visitor).map_err(|_| fmt::Error)?;let root = visitor.root.ok_or(fmt::Error)?;- let mut root_entry = serde_json::json!({ "Plan": root });- if let Some(summary) = self.summary {- if let Some(total_rows) = summary.total_rows {- root_entry["Total Rows"] = serde_json::Value::from(total_rows);- }- if let Some(duration) = summary.duration {- root_entry["Duration"] =- serde_json::Value::from(format!("{duration:?}"));- }- }- let doc = serde_json::Value::Array(vec![root_entry]);+ let doc = [PgJsonRoot {+ plan: root,+ total_rows: self.summary.and_then(|s| s.total_rows),+ duration: self+ .summary+ .and_then(|s| s.duration)+ .map(|d| format!("{d:?}")),+ }];write!(f,"{}",@@ -768,10 +767,67 @@ struct PgJsonExecutionPlanVisitor<'a> {metric_types: &'a [MetricType],metric_categories: Option<&'a [MetricCategory]>,metric_names: Option<&'a [String]>,- objects: HashMap<u32, serde_json::Value>,+ objects: HashMap<u32, PgJsonNode>,parent_ids: Vec<u32>,next_id: u32,- root: Option<serde_json::Value>,+ root: Option<PgJsonNode>,+}++/// Top-level entry of the `pgjson` output. Field order is the output order.+#[derive(Serialize)]+struct PgJsonRoot {+ #[serde(rename = "Plan")]+ plan: PgJsonNode,+ #[serde(rename = "Total Rows", skip_serializing_if = "Option::is_none")]+ total_rows: Option<usize>,+ #[serde(rename = "Duration", skip_serializing_if = "Option::is_none")]+ duration: Option<String>,+}++/// One node of the `pgjson` output. Field order is the output order, so the+/// JSON reads top-down like a PostgreSQL plan, with `"Plans"` last.+#[derive(Serialize)]+struct PgJsonNode {+ #[serde(rename = "Node Type")]+ node_type: String,+ #[serde(rename = "Details")]+ details: String,+ #[serde(rename = "Output", skip_serializing_if = "Option::is_none")]+ output: Option<Vec<String>>,+ #[serde(rename = "Actual Rows", skip_serializing_if = "Option::is_none")]+ actual_rows: Option<usize>,+ #[serde(rename = "Actual Total Time", skip_serializing_if = "Option::is_none")]+ actual_total_time: Option<f64>,+ #[serde(rename = "Extras", skip_serializing_if = "PgJsonExtras::is_empty")]+ extras: PgJsonExtras,+ #[serde(rename = "Plans")]+ plans: Vec<PgJsonNode>,+}++/// Non-canonical metrics of a node, serialized as a JSON object whose keys+/// keep their insertion order (independent of `serde_json`'s+/// `preserve_order` feature).+#[derive(Default)]+struct PgJsonExtras(Vec<(String, serde_json::Value)>);++impl PgJsonExtras {+ fn is_empty(&self) -> bool {+ self.0.is_empty()+ }++ /// Insert or replace `key`. A replaced key keeps its original position.+ fn insert(&mut self, key: &str, value: serde_json::Value) {+ match self.0.iter_mut().find(|(k, _)| k == key) {+ Some((_, v)) => *v = value,+ None => self.0.push((key.to_string(), value)),+ }+ }+}++impl Serialize for PgJsonExtras {+ fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ serializer.collect_map(self.0.iter().map(|(k, v)| (k, v)))+ }}impl PgJsonExecutionPlanVisitor<'_> {@@ -829,7 +885,7 @@ impl PgJsonExecutionPlanVisitor<'_> {/// Populate `"Actual Rows"`, `"Actual Total Time"`, and `"Extras"` for/// the given node from its aggregated `MetricsSet`, honoring the same/// filtering pipeline used by `IndentVisitor`.- fn attach_metrics(&self, plan: &dyn ExecutionPlan, object: &mut serde_json::Value) {+ fn attach_metrics(&self, plan: &dyn ExecutionPlan, object: &mut PgJsonNode) {if matches!(self.show_metrics, ShowMetrics::None) {return;}@@ -860,28 +916,23 @@ impl PgJsonExecutionPlanVisitor<'_> {// Build the Extras bucket, while extracting PG-canonical keys to the// top level.- let mut extras = serde_json::Map::new();for metric in metrics.iter() {let value = metric.value();match value {MetricValue::OutputRows(c) => {- object["Actual Rows"] = serde_json::Value::from(c.value());+ object.actual_rows = Some(c.value());}MetricValue::ElapsedCompute(t) => {let ms = (t.value() as f64) / 1_000_000.0;- object["Actual Total Time"] = serde_json::Value::from(ms);+ object.actual_total_time = Some(ms);}_ => {- extras.insert(- value.name().to_string(),- Self::metric_value_to_json(value),- );+ object+ .extras+ .insert(value.name(), Self::metric_value_to_json(value));}}}- if !extras.is_empty() {- object["Extras"] = serde_json::Value::Object(extras);- }}}@@ -895,28 +946,31 @@ impl ExecutionPlanVisitor for PgJsonExecutionPlanVisitor<'_> {// Build fields in reading order: Node Type, Details, (schema),// (metrics), Plans last — so the JSON output reads top-down like a// PostgreSQL plan.- let mut object = serde_json::json!({- "Node Type": plan.name(),- "Details": Self::one_line_details(plan),- });+ let mut object = PgJsonNode {+ node_type: plan.name().to_string(),+ details: Self::one_line_details(plan),+ output: None,+ actual_rows: None,+ actual_total_time: None,+ extras: PgJsonExtras::default(),+ plans: vec![],+ };if self.show_schema || self.verbose {// Always include output columns when a caller asked for schema;// also include them in verbose mode so the pgjson output mirrors// the extra context shown by indent's verbose flag.- let columns: Vec<serde_json::Value> = plan- .schema()- .fields()- .iter()- .map(|f| serde_json::Value::String(f.name().to_string()))- .collect();- object["Output"] = serde_json::Value::Array(columns);+ object.output = Some(+ plan.schema()+ .fields()+ .iter()+ .map(|f| f.name().to_string())+ .collect(),+ );}self.attach_metrics(plan, &mut object);- object["Plans"] = serde_json::Value::Array(vec![]);-self.objects.insert(id, object);self.parent_ids.push(id);Ok(true)@@ -928,11 +982,7 @@ impl ExecutionPlanVisitor for PgJsonExecutionPlanVisitor<'_> {if let Some(parent_id) = self.parent_ids.last() {let parent = self.objects.get_mut(parent_id).ok_or(fmt::Error)?;- let plans = parent- .get_mut("Plans")- .and_then(|p| p.as_array_mut())- .ok_or(fmt::Error)?;- plans.push(current);+ parent.plans.push(current);} else {self.root = Some(current);}@@ -1874,8 +1924,6 @@ mod tests {let out = DisplayableExecutionPlan::new(plan.as_ref()).pgjson(false).to_string();- // This snapshot assumes `serde_json` is built with the- // `preserve_order` feature (enabled via this crate's dev-deps).assert_snapshot!(out, @r#"[{