:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): stop the rust parser's json emitter copying subtrees (#125458)
George Munyorocbebe23ca82b
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex bcd207c538d8..7b6790256d6c 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -5758,7 +5758,7 @@ dependencies = [[[package]]name = "hogql_parser_rs"-version = "1.3.109"+version = "1.3.110"dependencies = ["pyo3","serde_json",diff --git a/rust/hogql/parser/Cargo.toml b/rust/hogql/parser/Cargo.tomlindex efb855114f0c..e569282d978b 100644--- a/rust/hogql/parser/Cargo.toml+++ b/rust/hogql/parser/Cargo.toml@@ -1,7 +1,7 @@[package]name = "hogql_parser_rs"# Versioned independently from the cpp parser in `common/hogql_parser/setup.py`. Both started from the same HogQL grammar revision, but the Rust port is bumped on its own whenever the hand-rolled parser changes; each ships as its own PyPI wheel.-version = "1.3.109"+version = "1.3.110"edition = "2021"publish = falsediff --git a/rust/hogql/parser/pyproject.toml b/rust/hogql/parser/pyproject.tomlindex 049ef3b6bacf..7af556915a14 100644--- a/rust/hogql/parser/pyproject.toml+++ b/rust/hogql/parser/pyproject.toml@@ -9,7 +9,7 @@ build-backend = "maturin"[project]name = "hogql_parser_rs"# Mirrors `rust/hogql/parser/Cargo.toml` (always identical). Tracks but is NOT locked to `common/hogql_parser/setup.py`: the Rust port is versioned independently and bumped on its own whenever the hand-rolled parser changes.-version = "1.3.109"+version = "1.3.110"description = "Hand-rolled Rust HogQL parser with C++-parity AST output"readme = "README.md"requires-python = ">=3.12"diff --git a/rust/hogql/parser/src/emit.rs b/rust/hogql/parser/src/emit.rsindex 69d39306797d..f4837215e886 100644--- a/rust/hogql/parser/src/emit.rs+++ b/rust/hogql/parser/src/emit.rs@@ -2,7 +2,7 @@//!//! Field names and shapes mirror the dataclasses at [`posthog/hogql/ast.py`]. The Python deserialiser at [`posthog/hogql/json_ast.py`] also builds these dataclasses from the JSON shape, so the two backends produce `__eq__`-equivalent objects.-use serde_json::{json, Value};+use serde_json::Value;use std::borrow::Cow;/// AST node + value constructor surface. Every construction site in the parser routes through this trait so the same parse logic can produce `serde_json::Value` (kept for WASM and tests) or Python dataclass instances directly via PyO3.@@ -283,6 +283,13 @@ pub trait Emitter {// ===== Mutation =====/// Remove a named field from a node, returning the previous value (or None when absent). Used to drop sentinel keys before the node leaves the parser, e.g. `__rust_offset_liftable`.fn remove_field(&self, v: &mut Self::Value, name: &str) -> Option<Self::Value>;+ /// Return a field that the caller does not read from `v` again. `JsonEmitter` moves the field out of `v` instead of deep-copying it, so `v` is left without that field. The default copies the field and leaves `v` unchanged, because a cloned `PyEmitter` value shares its Python object and removing the field would change every other reference to it.+ fn take_field(&self, v: &mut Self::Value, name: &str) -> Option<Self::Value> {+ self.get_field(v, name)+ }+ fn into_list(&self, v: Self::Value) -> Option<Vec<Self::Value>> {+ self.as_list(&v)+ }/// Set / overwrite a named field on a node. Used by AST-surgery sites that copy a node and tweak one field — primarily BETWEEN-split recursion, merge-select-decorators, and CTE injection. JSON inserts into the underlying `Map`; Py calls `obj.setattr(name, value)`.fn set_field(&self, v: &mut Self::Value, name: &str, value: Self::Value);}@@ -291,6 +298,15 @@ pub trait Emitter {// JsonEmitter — `serde_json::Value` impl, mirrors the previous free `emit::xxx` functions. Used by `parse_*_json` entry points and the future WASM build.// ============================================================================+/// Build a JSON object from `"key": value` pairs. `json!` serializes interpolated variables by reference, so every child subtree passed to it is deep-copied and the original is dropped. This macro moves each value into the map instead, so the cost of building a node does not grow with the size of its children.+macro_rules! json_object {+ ({ $($key:literal : $value:expr),* $(,)? }) => {{+ let mut map = serde_json::Map::new();+ $( map.insert($key.to_string(), Value::from($value)); )*+ Value::Object(map)+ }};+}+#[derive(Default, Clone, Copy)]pub struct JsonEmitter;@@ -329,72 +345,72 @@ impl Emitter for JsonEmitter {}fn constant(&self, value: Value) -> Value {- json!({"node": "Constant", "value": value})+ json_object!({"node": "Constant", "value": value})}fn constant_special_number(&self, name: &'static str) -> Value {- json!({"node": "Constant", "value": name, "value_type": "number"})+ json_object!({"node": "Constant", "value": name, "value_type": "number"})}fn constant_number_string(&self, text: String) -> Value {- json!({"node": "Constant", "value": text, "value_type": "number"})+ json_object!({"node": "Constant", "value": text, "value_type": "number"})}fn field(&self, chain: Vec<Value>) -> Value {- json!({"node": "Field", "chain": chain})+ json_object!({"node": "Field", "chain": chain})}fn arith(&self, left: Value, op: &str, right: Value) -> Value {- json!({"node": "ArithmeticOperation", "left": left, "right": right, "op": op})+ json_object!({"node": "ArithmeticOperation", "left": left, "right": right, "op": op})}fn compare(&self, left: Value, op: &str, right: Value) -> Value {- json!({"node": "CompareOperation", "left": left, "right": right, "op": op})+ json_object!({"node": "CompareOperation", "left": left, "right": right, "op": op})}fn compare_is_null(&self, left: Value, negated: bool) -> Value {- json!({+ json_object!({"node": "CompareOperation","left": left,- "right": json!({"node": "Constant", "value": Value::Null}),+ "right": json_object!({"node": "Constant", "value": Value::Null}),"op": if negated { "!=" } else { "==" },"is_null_comparison_style": true,})}fn is_distinct_from(&self, left: Value, right: Value, negated: bool) -> Value {- json!({"node": "IsDistinctFrom", "left": left, "right": right, "negated": negated})+ json_object!({"node": "IsDistinctFrom", "left": left, "right": right, "negated": negated})}fn between(&self, expr: Value, low: Value, high: Value, negated: bool) -> Value {- json!({"node": "BetweenExpr", "expr": expr, "low": low, "high": high, "negated": negated})+ json_object!({"node": "BetweenExpr", "expr": expr, "low": low, "high": high, "negated": negated})}fn not_(&self, expr: Value) -> Value {- json!({"node": "Not", "expr": expr})+ json_object!({"node": "Not", "expr": expr})}fn and_(&self, exprs: Vec<Value>) -> Value {- json!({"node": "And", "exprs": exprs})+ json_object!({"node": "And", "exprs": exprs})}fn or_(&self, exprs: Vec<Value>) -> Value {- json!({"node": "Or", "exprs": exprs})+ json_object!({"node": "Or", "exprs": exprs})}fn tuple_(&self, exprs: Vec<Value>) -> Value {- json!({"node": "Tuple", "exprs": exprs})+ json_object!({"node": "Tuple", "exprs": exprs})}fn array_(&self, exprs: Vec<Value>) -> Value {- json!({"node": "Array", "exprs": exprs})+ json_object!({"node": "Array", "exprs": exprs})}fn array_access(&self, array: Value, property: Value, nullish: bool) -> Value {if nullish {- json!({"node": "ArrayAccess", "array": array, "property": property, "nullish": true})+ json_object!({"node": "ArrayAccess", "array": array, "property": property, "nullish": true})} else {- json!({"node": "ArrayAccess", "array": array, "property": property})+ json_object!({"node": "ArrayAccess", "array": array, "property": property})}}fn tuple_access(&self, tuple_: Value, index: i64, nullish: bool) -> Value {if nullish {- json!({"node": "TupleAccess", "tuple": tuple_, "index": index, "nullish": true})+ json_object!({"node": "TupleAccess", "tuple": tuple_, "index": index, "nullish": true})} else {- json!({"node": "TupleAccess", "tuple": tuple_, "index": index})+ json_object!({"node": "TupleAccess", "tuple": tuple_, "index": index})}}fn alias(&self, expr: Value, name: &str) -> Value {- json!({"node": "Alias", "expr": expr, "alias": name})+ json_object!({"node": "Alias", "expr": expr, "alias": name})}fn call(&self, name: &str, args: Vec<Value>) -> Value {- json!({"node": "Call", "name": name, "args": args})+ json_object!({"node": "Call", "name": name, "args": args})}fn call_full(&self,@@ -429,16 +445,16 @@ impl Emitter for JsonEmitter {}fn lambda(&self, args: Vec<String>, expr: Value) -> Value {let args_json: Vec<Value> = args.into_iter().map(Value::String).collect();- json!({"node": "Lambda", "args": args_json, "expr": expr})+ json_object!({"node": "Lambda", "args": args_json, "expr": expr})}fn expr_call(&self, expr: Value, args: Vec<Value>) -> Value {- json!({"node": "ExprCall", "expr": expr, "args": args})+ json_object!({"node": "ExprCall", "expr": expr, "args": args})}fn type_cast(&self, expr: Value, type_name: &str) -> Value {- json!({"node": "TypeCast", "expr": expr, "type_name": type_name})+ json_object!({"node": "TypeCast", "expr": expr, "type_name": type_name})}fn try_cast(&self, expr: Value, type_name: &str) -> Value {- json!({"node": "TryCast", "expr": expr, "type_name": type_name})+ json_object!({"node": "TryCast", "expr": expr, "type_name": type_name})}fn array_slice(&self, array: Value, start: Option<Value>, end: Option<Value>) -> Value {let mut obj = serde_json::Map::new();@@ -457,13 +473,13 @@ impl Emitter for JsonEmitter {.into_iter().map(|(k, v)| Value::Array(vec![k, v])).collect();- json!({"node": "Dict", "items": items_json})+ json_object!({"node": "Dict", "items": items_json})}fn placeholder(&self, expr: Value) -> Value {- json!({"node": "Placeholder", "expr": expr})+ json_object!({"node": "Placeholder", "expr": expr})}fn named_argument(&self, name: &str, value: Value) -> Value {- self.no_pos(json!({"node": "NamedArgument", "name": name, "value": value}))+ self.no_pos(json_object!({"node": "NamedArgument", "name": name, "value": value}))}fn order_expr(&self, expr: Value, order: &str, with_fill: Option<Value>) -> Value {let mut obj = serde_json::Map::new();@@ -510,26 +526,26 @@ impl Emitter for JsonEmitter {Value::Object(obj)}fn spread_expr(&self, expr: Value) -> Value {- json!({"node": "SpreadExpr", "expr": expr})+ json_object!({"node": "SpreadExpr", "expr": expr})}fn positional_ref(&self, index: i64) -> Value {- json!({"node": "PositionalRef", "index": index})+ json_object!({"node": "PositionalRef", "index": index})}fn program(&self, declarations: Vec<Value>) -> Value {- json!({"node": "Program", "declarations": declarations})+ json_object!({"node": "Program", "declarations": declarations})}fn block(&self, declarations: Vec<Value>) -> Value {- json!({"node": "Block", "declarations": declarations})+ json_object!({"node": "Block", "declarations": declarations})}fn expr_statement(&self, expr: Value) -> Value {- json!({"node": "ExprStatement", "expr": expr})+ json_object!({"node": "ExprStatement", "expr": expr})}fn if_statement(&self, cond: Value, then: Value, else_: Value) -> Value {- json!({"node": "IfStatement", "expr": cond, "then": then, "else_": else_})+ json_object!({"node": "IfStatement", "expr": cond, "then": then, "else_": else_})}fn while_statement(&self, cond: Value, body: Value) -> Value {- json!({"node": "WhileStatement", "expr": cond, "body": body})+ json_object!({"node": "WhileStatement", "expr": cond, "body": body})}fn for_in_statement(&self,@@ -538,7 +554,7 @@ impl Emitter for JsonEmitter {expr: Value,body: Value,) -> Value {- json!({+ json_object!({"node": "ForInStatement","keyVar": key_var,"valueVar": value_var,@@ -553,7 +569,7 @@ impl Emitter for JsonEmitter {increment: Value,body: Value,) -> Value {- json!({+ json_object!({"node": "ForStatement","initializer": initializer,"condition": condition,@@ -562,10 +578,10 @@ impl Emitter for JsonEmitter {})}fn function_(&self, name: &str, params: Vec<Value>, body: Value) -> Value {- json!({"node": "Function", "name": name, "params": params, "body": body})+ json_object!({"node": "Function", "name": name, "params": params, "body": body})}fn variable_assignment(&self, left: Value, right: Value) -> Value {- json!({"node": "VariableAssignment", "left": left, "right": right})+ json_object!({"node": "VariableAssignment", "left": left, "right": right})}fn return_statement(&self, expr: Value) -> Value {let mut obj = serde_json::Map::new();@@ -579,7 +595,7 @@ impl Emitter for JsonEmitter {catches: Vec<Value>,finally_stmt: Value,) -> Value {- json!({+ json_object!({"node": "TryCatchStatement","try_stmt": try_stmt,"catches": catches,@@ -587,16 +603,16 @@ impl Emitter for JsonEmitter {})}fn throw_statement(&self, expr: Value) -> Value {- json!({"node": "ThrowStatement", "expr": expr})+ json_object!({"node": "ThrowStatement", "expr": expr})}fn variable_declaration(&self, name: &str, expr: Value) -> Value {- json!({"node": "VariableDeclaration", "name": name, "expr": expr})+ json_object!({"node": "VariableDeclaration", "name": name, "expr": expr})}fn catch_clause(&self, var: Value, ty: Value, body: Value) -> Value {- json!([var, ty, body])+ Value::Array(vec![var, ty, body])}fn cte(&self, name: &str, expr: Value, cte_type: &str) -> Value {- json!({"node": "CTE", "name": name, "expr": expr, "cte_type": cte_type})+ json_object!({"node": "CTE", "name": name, "expr": expr, "cte_type": cte_type})}fn cte_subquery(&self,@@ -629,13 +645,13 @@ impl Emitter for JsonEmitter {Value::Object(obj)}fn join_constraint(&self, expr: Value, constraint_type: &str) -> Value {- json!({"node": "JoinConstraint", "expr": expr, "constraint_type": constraint_type})+ json_object!({"node": "JoinConstraint", "expr": expr, "constraint_type": constraint_type})}fn values_query(&self, rows: Vec<Value>) -> Value {- json!({"node": "ValuesQuery", "rows": rows})+ json_object!({"node": "ValuesQuery", "rows": rows})}fn pivot_column(&self, column: Value, values: Vec<Value>) -> Value {- json!({"node": "PivotColumn", "column": column, "values": values})+ json_object!({"node": "PivotColumn", "column": column, "values": values})}fn unpivot_column(&self,@@ -643,7 +659,7 @@ impl Emitter for JsonEmitter {name_columns: Value,unpivot_values: Vec<Value>,) -> Value {- json!({+ json_object!({"node": "UnpivotColumn","value_columns": value_columns,"name_columns": name_columns,@@ -651,13 +667,13 @@ impl Emitter for JsonEmitter {})}fn grouping_set(&self, exprs: Vec<Value>) -> Value {- json!({"node": "GroupingSet", "exprs": exprs})+ json_object!({"node": "GroupingSet", "exprs": exprs})}fn hogqlx_tag(&self, kind: &str, attributes: Vec<Value>) -> Value {- json!({"node": "HogQLXTag", "kind": kind, "attributes": attributes})+ json_object!({"node": "HogQLXTag", "kind": kind, "attributes": attributes})}fn hogqlx_attribute(&self, name: &str, value: Value) -> Value {- json!({"node": "HogQLXAttribute", "name": name, "value": value})+ json_object!({"node": "HogQLXAttribute", "name": name, "value": value})}fn window_function(&self,@@ -775,17 +791,17 @@ impl Emitter for JsonEmitter {Value::Object(obj)}fn select_set_query(&self, initial: Value, subsequent: Vec<Value>) -> Value {- json!({+ json_object!({"node": "SelectSetQuery","initial_select_query": initial,"subsequent_select_queries": subsequent,})}fn window_expr_empty(&self) -> Value {- json!({"node": "WindowExpr"})+ json_object!({"node": "WindowExpr"})}fn window_frame_bound(&self, frame_type: &str, frame_value: Value) -> Value {- json!({"node": "WindowFrameExpr", "frame_type": frame_type, "frame_value": frame_value})+ json_object!({"node": "WindowFrameExpr", "frame_type": frame_type, "frame_value": frame_value})}fn interpolate_expr(&self, expr: Value, value: Option<Value>) -> Value {let mut obj = serde_json::Map::new();@@ -807,7 +823,7 @@ impl Emitter for JsonEmitter {Value::Object(obj)}fn select_query_empty(&self) -> Value {- json!({"node": "SelectQuery"})+ json_object!({"node": "SelectQuery"})}fn select_set_node(&self, select_query: Value, set_operator: Option<&str>) -> Value {let mut obj = serde_json::Map::new();@@ -820,7 +836,7 @@ impl Emitter for JsonEmitter {}fn position(&self, line: u32, column: u32, offset: usize) -> Value {- json!({"line": line, "column": column, "offset": offset})+ json_object!({"line": line, "column": column, "offset": offset})}fn with_pos(&self, mut value: Value, start: Value, end: Value) -> Value {if let Some(obj) = value.as_object_mut() {@@ -873,6 +889,15 @@ impl Emitter for JsonEmitter {fn remove_field(&self, v: &mut Value, name: &str) -> Option<Value> {v.as_object_mut().and_then(|obj| obj.remove(name))}+ fn take_field(&self, v: &mut Value, name: &str) -> Option<Value> {+ self.remove_field(v, name)+ }+ fn into_list(&self, v: Value) -> Option<Vec<Value>> {+ match v {+ Value::Array(items) => Some(items),+ _ => None,+ }+ }fn set_field(&self, v: &mut Value, name: &str, value: Value) {if let Some(obj) = v.as_object_mut() {obj.insert(name.into(), value);diff --git a/rust/hogql/parser/src/parse.rs b/rust/hogql/parser/src/parse.rsindex 6727c60ee861..b204f82e30f5 100644--- a/rust/hogql/parser/src/parse.rs+++ b/rust/hogql/parser/src/parse.rs@@ -1350,18 +1350,13 @@ pub(crate) fn chain_join<E: Emitter>(// so unwind via owned recursion: pop the existing next_join, recurse// to append, then put it back.fn append_at_tail<E: Emitter>(emit: &E, mut node: E::Value, new_tail: E::Value) -> E::Value {- let has_next = emit- .get_field(&node, "next_join")- .map(|v| !emit.is_null(&v))- .unwrap_or(false);- if !has_next {- emit.set_field(&mut node, "next_join", new_tail);- return node;+ match emit.remove_field(&mut node, "next_join") {+ Some(existing) if !emit.is_null(&existing) => {+ let updated = append_at_tail(emit, existing, new_tail);+ emit.set_field(&mut node, "next_join", updated);+ }+ _ => emit.set_field(&mut node, "next_join", new_tail),}- // Move the existing next_join out, recurse with it, set it back.- let existing = emit.get_field(&node, "next_join").expect("just checked");- let updated = append_at_tail(emit, existing, new_tail);- emit.set_field(&mut node, "next_join", updated);node}append_at_tail(emit, left, right)diff --git a/rust/hogql/parser/src/parse/bp.rs b/rust/hogql/parser/src/parse/bp.rsindex 7af47a164227..77a7eeee5330 100644--- a/rust/hogql/parser/src/parse/bp.rs+++ b/rust/hogql/parser/src/parse/bp.rs@@ -175,10 +175,13 @@ pub(super) fn build_infix<E: Emitter>(/// for the CONCAT alt: `a || b || c` becomes `concat(a, b, c)` rather/// than `concat(concat(a, b), c)`. Either side that's already a/// `Call(name="concat", ...)` contributes its existing args.-pub(super) fn merge_concat<E: Emitter>(emit: &E, lhs: E::Value, rhs: E::Value) -> E::Value {+pub(super) fn merge_concat<E: Emitter>(emit: &E, mut lhs: E::Value, mut rhs: E::Value) -> E::Value {let mut args: Vec<E::Value> = Vec::new();if is_concat_call(emit, &lhs) {- if let Some(left_args) = emit.get_field(&lhs, "args").and_then(|v| emit.as_list(&v)) {+ if let Some(left_args) = emit+ .take_field(&mut lhs, "args")+ .and_then(|v| emit.into_list(v))+ {args.extend(left_args);} else {args.push(lhs);@@ -187,7 +190,10 @@ pub(super) fn merge_concat<E: Emitter>(emit: &E, lhs: E::Value, rhs: E::Value) -args.push(lhs);}if is_concat_call(emit, &rhs) {- if let Some(right_args) = emit.get_field(&rhs, "args").and_then(|v| emit.as_list(&v)) {+ if let Some(right_args) = emit+ .take_field(&mut rhs, "args")+ .and_then(|v| emit.into_list(v))+ {args.extend(right_args);} else {args.push(rhs);@@ -229,11 +235,14 @@ pub(super) fn merge_and_or<E: Emitter>(fn extend_with_node_children<E: Emitter>(emit: &E,out: &mut Vec<E::Value>,- v: E::Value,+ mut v: E::Value,node: &str,) {if emit.node_kind(&v).as_deref() == Some(node) {- if let Some(children) = emit.get_field(&v, "exprs").and_then(|f| emit.as_list(&f)) {+ if let Some(children) = emit+ .take_field(&mut v, "exprs")+ .and_then(|f| emit.into_list(f))+ {out.extend(children);return;}diff --git a/rust/hogql/parser/src/parse/join.rs b/rust/hogql/parser/src/parse/join.rsindex 7f267de31085..e734cceef652 100644--- a/rust/hogql/parser/src/parse/join.rs+++ b/rust/hogql/parser/src/parse/join.rs@@ -1608,7 +1608,7 @@ fn attach_constraint_to_outermost_unconstrained_join<E: Emitter>(return (node, false);}// Try recursing into next_join first.- if let Some(nj) = emit.get_field(&node, "next_join") {+ if let Some(nj) = emit.remove_field(&mut node, "next_join") {if emit.node_kind(&nj).is_some() {let (updated, attached) = attach_constraint_to_outermost_unconstrained_join(emit,@@ -1621,6 +1621,8 @@ fn attach_constraint_to_outermost_unconstrained_join<E: Emitter>(if attached {return (node, true);}+ } else {+ emit.set_field(&mut node, "next_join", nj);}}if current_depth < min_attachable_depth {