Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

76.2 KB
Raw Blame
use std::cell::RefCell;
use std::cmp::Ordering;
use std::rc::Rc;
use std::sync::Arc;

use serde_json::{json, Value as JsonValue};

use crate::{
    context::{ExecutionContext, Symbol},
    error::VmError,
    memory::{HeapReference, VmHeap},
    ops::Operation,
    program::Token,
    state::{
        chunk_str, chunk_to_symbol, close_over_cells, closure_from_json, closure_to_json,
        collect_cells, rebuild_cells, root_closure_json, value_from_json, value_to_json,
        CallFrameJson, CellRegistry, Resumable, ThrowFrameJson, UpvalueJson, VmSnapshot,
    },
    util::{get_json_nested, like, regex_match},
    values::{
        boolean_text_equals, compare_values, temporal_seconds_pair, Callable, Closure,
        FromHogLiteral, HogLiteral, HogMap, HogStr, HogValue, LocalCallable, Num, NumOp, Upvalue,
        UpvalueCell,
    },
};

// Recursion guard for walking nested JSON/Hog containers (input deserialization, output
// serialization, deep clones, json stringify). The reference VMs impose no explicit JSON-nesting
// limit, so a low cap here shows up as a shadow `status_mismatch` (Rust errors where Node succeeds)
// on legitimately deep event properties. 256 clears any realistic event nesting by a wide margin
// while still bounding native recursion well within a worker thread's stack.
pub const MAX_JSON_SERDE_DEPTH: usize = 256;

/// The outcome of a virtual machine step.
#[derive(Debug, Clone)]
pub enum StepOutcome {
    /// The program has completed, returning a value
    Finished(JsonValue),
    /// The program has requested a global function call. The driver dispatches it: a registered
    /// native (STL/ext) runs inline; a registered *async* function (see [`ExecutionContext::is_async`])
    /// suspends the resumable driver instead — at this point the args are already popped and `ip` is
    /// past the call, so a snapshot taken now resumes correctly once the result is pushed.
    NativeCall(String, Vec<HogValue>),
    /// The program has requested another step
    Continue,
}

/// Some debug information about a virtual machine failure, returned by some
/// helpers when a VM step returns an error
#[derive(Debug, Clone)]
pub struct VmFailure {
    pub error: VmError,
    pub ip: usize,
    pub stack: Vec<HogValue>, // TODO - only used for debugging, should remove
    pub step: usize,
}

/// The Hog Virtual Machine. Represents the state of a running program.
pub struct HogVM<'a> {
    /// The heap of the virtual machine. Generally used for `HogValue::deref`, to allow you to access
    /// `HogLiteral` values for native function implementation.
    pub heap: VmHeap, // Needs to be pub to allow users to write their own extension native functions
    stack: Vec<HogValue>,

    stack_frames: Vec<CallFrame>,
    throw_frames: Vec<ThrowFrame>,
    // Upvalues still open (viewing a live stack slot). Closures created in this scope share these
    // cells; when a slot leaves scope (stack truncation) the matching upvalue is closed (snapshotted)
    // and removed from here. See `capture_upvalue` / `close_upvalues`.
    open_upvalues: Vec<UpvalueCell>,
    ip: usize,
    // How many times this run has suspended on an async function call. Carried into snapshots and
    // restored on resume, so the per-run async budget survives the suspend/resume round-trip.
    async_steps: usize,
    // Count of opcodes dispatched this run (the reference's `ops`). Carried across suspend/resume.
    ops: usize,
    // Per-opcode execution trace, collected only when the context opts into telemetry. Each entry is
    // the reference's `[time, chunk, ip, "op/NAME", debug]` tuple. Carried across suspend/resume.
    telemetry: Vec<JsonValue>,

    context: &'a ExecutionContext,
    // The base program is None, but calling into e.g. hog standard library functions involves changing the "module"
    // the pointer is currently pointing into to e.g. "arrayExists", as part of the function call that branches into
    // that function.
    current_symbol: Option<Symbol>,
    // The decoded token slice of the chunk `current_symbol` names, kept in lockstep with it (via
    // `set_chunk`) so the per-step fetch is a bounds-checked index instead of a symbol match plus
    // table lookup. Borrows from the context, like every other program view.
    current_tokens: &'a [Token],
}

struct CallFrame {
    ret_ptr: usize,             // Where to jump back to when we're done
    ret_symbol: Option<Symbol>, // The module to return to when we're done
    stack_start: usize,         // Point in the stack the frame values start
    // The closure this frame is running (its callable + captured upvalues). Captures are read by
    // `frame_capture`; the callable identity is what lets a snapshot reconstruct the reference VM's
    // per-frame `closure`. Cross-module (STL) frames synthesize a callable for their symbol.
    closure: Closure,
}

struct ThrowFrame {
    catch_ptr: usize,             // The ptr to jump to if we throw
    catch_symbol: Option<Symbol>, // The module to return to if we throw
    stack_start: usize,           // The stack size when we entered the try
    call_depth: usize,            // The depth of the call stack when we entered the try
}

impl<'a> HogVM<'a> {
    pub fn new(context: &'a ExecutionContext) -> Result<Self, VmError> {
        Ok(Self {
            stack: Vec::new(),
            stack_frames: Vec::new(),
            throw_frames: Vec::new(),
            open_upvalues: Vec::new(),
            ip: 0,
            async_steps: 0,
            ops: 0,
            telemetry: Vec::new(),
            current_symbol: None,
            current_tokens: context.chunk_tokens(&None)?,
            context,
            heap: VmHeap::new(context.max_heap_size),
        })
    }

    /// Step the virtual machine, writing some debug information to the provided output function.
    pub fn debug_step(&mut self, output: &dyn Fn(String)) -> Result<StepOutcome, VmError> {
        let op: Operation = self.next_op()?;
        let mut surrounding = Vec::new();
        let start = self.ip.saturating_sub(2);
        for i in 0..5 {
            if let Ok(op) = self.context.get_bytecode(start + i, &self.current_symbol) {
                surrounding.push(op);
            }
        }

        self.ip -= 1;
        output(format!(
            "Op ({}), module {:?}: {:?} [{:?}], Stack: {:?}",
            self.ip, self.current_symbol, op, surrounding, self.stack
        ));
        self.step()
    }

    /// Step the virtual machine one cycle.
    pub fn step(&mut self) -> Result<StepOutcome, VmError> {
        let pre_ip = self.ip;
        let op: Operation = match self.next_op() {
            Ok(op) => op,
            // The reference VMs halt gracefully when the instruction pointer runs off the
            // end of the top-level program (it has no trailing RETURN), yielding the top of
            // stack or null. Only do this at the top level — running off the end inside a
            // function/module is still malformed bytecode.
            Err(VmError::EndOfProgram(_))
                if self.stack_frames.is_empty() && self.current_symbol.is_none() =>
            {
                let result = if self.stack.is_empty() {
                    HogLiteral::Null.into()
                } else {
                    self.pop_stack()?
                };
                return Ok(StepOutcome::Finished(self.hog_to_json(&result)?));
            }
            Err(e) => return Err(e),
        };

        self.ops += 1;

        if self.context.collect_telemetry {
            // The reference's `[time, chunk, ip, "op/NAME", debug]`. Time is a placeholder (we don't
            // expose a wall clock here) and debug is unused; the chunk/ip/op trace is the substance.
            self.telemetry.push(json!([
                0,
                chunk_str(&self.current_symbol),
                self.node_ip(pre_ip, &self.current_symbol),
                op_telemetry_name(&op),
                "",
            ]));
        }

        match op {
            Operation::GetGlobal => {
                // GetGlobal is used to do 1 of 2 things, either push a value from a global variable onto the stack, or push a new
                // function reference (referred to in other impls as a "closure") onto the stack - either a native one, or a hog one
                let mut chain = Vec::new();
                let count: usize = self.next_usize()?;
                for _ in 0..count {
                    chain.push(self.pop_stack()?);
                }
                if chain.is_empty() {
                    return Err(VmError::UnknownGlobal("".to_string()));
                }

                // Copy out the `Copy` `&ExecutionContext` so the `found` borrow is tied to it, not to
                // `self` — leaving `self` free for the `&mut self` `json_to_hog` call below.
                let context = self.context;
                if let Some(found) = get_json_nested(&context.globals, &chain, self)? {
                    let val = self.json_to_hog(found)?;
                    self.push_stack(val)?;
                } else if let Some(closure) = self.get_fn_reference(&chain) {
                    self.push_stack(closure)?;
                } else if get_json_nested(&context.globals, &chain[..1], self)?.is_some() {
                    // If the first element of the chain is a global, push null onto the stack, e.g.
                    // if a program is looking for "properties.blah", and "properties" exists, but
                    // "blah" doesn't, push null onto the stack.
                    self.push_stack(HogLiteral::Null)?;
                } else {
                    // But if the first element in the chain didn't exist, this is an error (the mental model here
                    // comes from SQL, where a missing column is an error, but a missing field in a column is, or
                    // at least can be, treated as a null value).
                    return Err(VmError::UnknownGlobal(format!("{chain:?}")));
                }
            }
            // We don't implement DeclareFn, because it's not used in the current compiler - it uses
            // "callables" constructed on the stack, which are then called by constructing a "closure",
            // which is basically a function call that wraps a "callable" and populates it's argument list
            Operation::DeclareFn => return Err(VmError::NotImplemented("DeclareFn".to_string())),
            Operation::CallGlobal => {
                // The TS impl here has a bunch of special case handling for functions with particular names.
                // I'm hoping I can simplify that here by unifying the native call interface a bit
                let name = self.next_str()?;
                let arg_count: usize = self.next_usize()?;
                // NOTE - the TS implementation has a clause here that looks for the name in the
                // "declared functions" - basically, the legacy way of declaring a function before Callable
                // and closures were introduced. We leave it out because, as above, DeclareFn isn't supported
                let available_args = self.stack.len() - self.current_frame_base();
                if available_args < arg_count {
                    return Err(VmError::NotEnoughArguments(
                        name.to_string(),
                        available_args,
                        arg_count,
                    ));
                }
                let symbol = Symbol::new("stl", name);
                if self.context.has_symbol(&symbol) {
                    // Cross module calls are done in a manner very similar to CallLocal, just with some
                    // messing around with the current state module.
                    return self.prep_cross_module_call(symbol, arg_count);
                }

                let mut args = Vec::with_capacity(arg_count);
                for _ in 0..arg_count {
                    args.push(self.pop_stack()?);
                }
                if self.context.version() != 0 {
                    // In v0, the arguments were expected to be passed in
                    // stack pop order, not push order. We simulate that here
                    // but always popping, but then maybe reversing
                    args.reverse();
                }
                // This VM does native calls like so: it returns a "native call" struct, which the
                // executing environment dispatches — inline for a native, or as a suspension for a
                // registered async function (handled in the resumable driver).
                return Ok(self.prep_native_call(name.to_string(), args));
            }
            Operation::And => {
                let count: usize = self.next_usize()?;
                let mut acc: HogLiteral = true.into();
                for _ in 0..count {
                    let value = self.pop_stack()?;
                    let value = value.deref(&self.heap)?;
                    acc = acc.and(value)?;
                }
                self.push_stack(acc)?;
            }
            Operation::Or => {
                let count: usize = self.next_usize()?;
                let mut acc: HogLiteral = false.into();
                for _ in 0..count {
                    let value = self.pop_stack()?;
                    let value = value.deref(&self.heap)?;
                    acc = acc.or(value)?;
                }
                self.push_stack(acc)?;
            }
            Operation::Not => {
                let val = self.pop_stack()?;
                let result = val.deref(&self.heap)?.not()?;
                self.push_stack(result)?;
            }
            Operation::Plus => {
                let (a, b) = (self.pop_arith_operand()?, self.pop_arith_operand()?);
                self.push_stack(Num::binary_op(NumOp::Add, &a, &b)?)?;
            }
            Operation::Minus => {
                let (a, b) = (self.pop_arith_operand()?, self.pop_arith_operand()?);
                self.push_stack(Num::binary_op(NumOp::Sub, &a, &b)?)?;
            }
            Operation::Mult => {
                let (a, b) = (self.pop_arith_operand()?, self.pop_arith_operand()?);
                self.push_stack(Num::binary_op(NumOp::Mul, &a, &b)?)?;
            }
            Operation::Div => {
                let (a, b) = (self.pop_arith_operand()?, self.pop_arith_operand()?);
                self.push_stack(Num::binary_op(NumOp::Div, &a, &b)?)?;
            }
            Operation::Mod => {
                let (a, b) = (self.pop_arith_operand()?, self.pop_arith_operand()?);
                self.push_stack(Num::binary_op(NumOp::Mod, &a, &b)?)?;
            }
            Operation::Eq => {
                let (a, b) = (self.pop_stack()?, self.pop_stack()?);
                let result = self.eq_op(&a, &b)?;
                self.push_stack(result)?;
            }
            Operation::NotEq => {
                let (a, b) = (self.pop_stack()?, self.pop_stack()?);
                let result = self.eq_op(&a, &b)?.not()?;
                self.push_stack(result)?;
            }
            Operation::Gt => self.compare_op(NumOp::Gt)?,
            Operation::GtEq => self.compare_op(NumOp::Gte)?,
            Operation::Lt => self.compare_op(NumOp::Lt)?,
            Operation::LtEq => self.compare_op(NumOp::Lte)?,
            Operation::Like => {
                let (val, pat) = self.pop_like_operands()?;
                self.push_stack(like(val, pat, true)?)?;
            }
            Operation::Ilike => {
                let (val, pat) = self.pop_like_operands()?;
                self.push_stack(like(val, pat, false)?)?;
            }
            Operation::NotLike => {
                let (val, pat) = self.pop_like_operands()?;
                self.push_stack(!like(val, pat, true)?)?;
            }
            Operation::NotIlike => {
                let (val, pat) = self.pop_like_operands()?;
                self.push_stack(!like(val, pat, false)?)?;
            }
            Operation::In => {
                let (needle, haystack) = (self.pop_stack()?, self.pop_stack()?);
                let result = self.in_op(&haystack, &needle)?;
                self.push_stack(result)?;
            }
            Operation::NotIn => {
                let (needle, haystack) = (self.pop_stack()?, self.pop_stack()?);
                let result = self.in_op(&haystack, &needle)?.not()?;
                self.push_stack(result)?;
            }
            Operation::Regex => self.regex_op(true, false)?,
            Operation::NotRegex => self.regex_op(true, true)?,
            Operation::Iregex => self.regex_op(false, false)?,
            Operation::NotIregex => self.regex_op(false, true)?,
            Operation::InCohort => return Err(VmError::NotImplemented("InCohort".to_string())),
            Operation::NotInCohort => {
                return Err(VmError::NotImplemented("NotInCohort".to_string()))
            }
            Operation::True => {
                self.push_stack(true)?;
            }
            Operation::False => {
                self.push_stack(false)?;
            }
            Operation::Null => {
                self.push_stack(HogLiteral::Null)?;
            }
            Operation::String => {
                let val = self.next_str()?;
                // Constants push a shared view of the pre-decoded token — no allocation.
                self.push_stack(HogLiteral::String(HogStr::Shared(val.clone())))?;
            }
            Operation::Float => {
                let val: f64 = self.next_f64()?;
                self.push_stack(val)?;
            }
            // The reference's INTEGER opcode is untyped `pushStack(next())`; the SQL-AST compiler
            // emits it with a boolean operand. Fast-path the overwhelmingly common i64 (this is a hot
            // opcode — loop counters etc.), and fall back to pushing the raw token for anything else.
            Operation::Integer => match self.next_token()? {
                Token::Int(i) => self.push_stack(*i)?,
                Token::Bool(b) => self.push_stack(*b)?,
                Token::Float(f) => self.push_stack(*f)?,
                Token::Str(s) => self.push_stack(HogLiteral::String(HogStr::Shared(s.clone())))?,
                Token::Null => self.push_stack(HogLiteral::Null)?,
                Token::Json(token) => {
                    let val = self.json_to_hog(token)?;
                    self.push_stack(val)?;
                }
            },
            Operation::Pop => {
                self.pop_stack()?;
            }
            Operation::GetLocal => {
                let base = self.current_frame_base();
                // Usize as a positive offset from the current frame
                let offset: usize = self.next_usize()?;
                let ptr = self.hoist(
                    base.checked_add(offset)
                        .ok_or_else(|| VmError::IntegerOverflow)?,
                )?;
                self.push_stack(ptr)?;
            }
            Operation::SetLocal => {
                // Replace some item "lower" in the stack with the top one.
                let base = self.current_frame_base();
                // Usize as a positive offset from the current frame
                let offset: usize = self.next_usize()?;
                let value = self.pop_stack()?;
                self.set_stack_val(
                    base.checked_add(offset)
                        .ok_or_else(|| VmError::IntegerOverflow)?,
                    value,
                )?;
            }
            Operation::Return => {
                let result = self.pop_stack()?;
                let last_frame = self.stack_frames.pop();
                let Some(frame) = last_frame else {
                    return Ok(StepOutcome::Finished(self.hog_to_json(&result)?));
                };

                // NOTE - this diverges from the TS impl
                // if self.stack_frames.is_empty() {
                //     return Ok(StepOutcome::Finished(result.deref(&self.heap)?.clone()));
                // };
                //
                self.set_chunk(frame.ret_symbol)?;
                self.truncate_stack(frame.stack_start)?;
                self.ip = frame.ret_ptr;
                self.push_stack(result)?;
            }
            Operation::Jump => {
                // i32 to permit branching backwards
                let offset: i32 = self.next_i32()?;
                self.ip = ((self.ip as i64)
                    .checked_add(offset as i64)
                    .ok_or_else(|| VmError::IntegerOverflow)?) as usize;
            }
            Operation::JumpIfFalse => {
                // i32 to permit branching backwards
                let offset: i32 = self.next_i32()?;
                // The reference coerces with `!popStack()`, so any falsy value (0, "", null, …) branches.
                let val = self.pop_stack()?;
                if !val.deref(&self.heap)?.truthy() {
                    self.ip = ((self.ip as i64)
                        .checked_add(offset as i64)
                        .ok_or_else(|| VmError::IntegerOverflow)?)
                        as usize;
                }
            }
            Operation::JumpIfStackNotNull => {
                // i32 to permit branching backwards
                let offset: i32 = self.next_i32()?;
                // Weirdly, this operation doesn't pop the value from the stack. This is mostly a random choice.
                if !self.stack.is_empty() {
                    let item = self.clone_stack_item(self.stack.len() - 1)?;
                    let item = item.deref(&self.heap)?;
                    if !matches!(item, HogLiteral::Null) {
                        self.ip = ((self.ip as i64)
                            .checked_add(offset as i64)
                            .ok_or_else(|| VmError::IntegerOverflow)?)
                            as usize;
                    }
                }
            }
            Operation::Dict => {
                let element_count: usize = self.next_usize()?;
                let mut keys = Vec::with_capacity(element_count);
                let mut values = Vec::with_capacity(element_count);
                // The keys and values are pushed into the stack in key:value, key:value pairs, but we're
                // going to walk the stack backwards to construct the dictionary
                for _ in 0..element_count {
                    // Note we don't put references into collections ever
                    values.push(self.pop_stack()?);
                    // Numeric keys coerce to their string form (the reference keys a JS Map with
                    // the raw scalar; see values::num_key_string).
                    let key_val = self.pop_stack()?;
                    let key = match key_val.deref(&self.heap)? {
                        HogLiteral::Number(n) => crate::values::num_key_string(n),
                        lit => lit.try_as::<str>()?.to_string(),
                    };
                    keys.push(key);
                }
                // keys/values were popped in reverse (stack order), so reverse the zip to restore
                // the source insertion order in the IndexMap.
                let map: HogMap = keys.into_iter().zip(values).rev().collect();
                let obj = HogLiteral::Object(Box::new(map));
                // For the non-primitive types below (objects, arrays, "tuples"), we /always/ heap allocate them. The reason
                // is that the pattern for e.g. nestedly setting an array value is to GetLocal followed by GetProperty, followed
                // by a SetProperty. If the array is "flat", as in, element 3 is a HogLiteral rather than Value, that SetProperty
                // will do nothing, because there's nowhere to write the new value /to/ (as the stack copy of the literal array would be
                // immediately dropped). We assert in SetProperty that the target is a reference, in order to prevent this surprising no-op
                // behaviour, but that means we have to either always heap-allocate our indexable values, or hoist in GetProperty. I've decided
                // to pay the cost of heap-allocating them up-front (and therefore chasing an extra pointer), because I think it more closely
                // mirrors the semantics of e.g. JavaScript or Python, which is what hog is based around.
                let ptr = self.heap.emplace(obj)?;
                self.push_stack(ptr)?;
            }
            Operation::Array => {
                let element_count: usize = self.next_usize()?;
                let mut elements = Vec::with_capacity(element_count);
                for _ in 0..element_count {
                    elements.push(self.pop_stack()?);
                }
                // We've walked back down the stack, but the compiler expects the array to be in pushed order
                elements.reverse();
                let array = HogLiteral::Array(elements);
                // See above
                let ptr = self.heap.emplace(array)?;
                self.push_stack(ptr)?;
            }
            Operation::Tuple => {
                // A tuple behaves like an array but prints as `(a, b)` and has typeof "tuple", so it
                // gets its own literal variant rather than reusing Array.
                let element_count: usize = self.next_usize()?;
                let mut elements = Vec::with_capacity(element_count);
                for _ in 0..element_count {
                    elements.push(self.pop_stack()?);
                }
                // We've walked back down the stack, but the compiler expects the tuple in pushed order
                elements.reverse();
                let tuple = HogLiteral::Tuple(elements);
                let ptr = self.heap.emplace(tuple)?;
                self.push_stack(ptr)?;
            }
            // Both reference VMs (`getNestedValue`/`get_nested_value`) return null for a missing
            // object key or out-of-range array index — the `nullish` flag only short-circuits a
            // null *intermediate* in a multi-key chain, which these single-key opcodes never hit.
            // So plain GetProperty and GetPropertyNullish behave identically here: null on miss.
            Operation::GetProperty | Operation::GetPropertyNullish => {
                let needle = self.pop_stack()?;
                let haystack = self.pop_stack()?;
                let chain = [needle];
                let res = haystack
                    .get_nested(&chain, &self.heap)?
                    .cloned()
                    .unwrap_or(HogLiteral::Null.into());
                self.push_stack(res)?;
            }
            Operation::SetProperty => {
                // Set property is a little tricky, because it assumes the object whose property is being set is
                // a reference. We assert this here - basically, we don't allow SET_PROPERTY to be a no-op, unlike the
                // original implementation, which would allow SET_PROPERTY to be done even if the target was dropped immediately
                // after the operation
                // Value to set - this could be a literal or a reference
                let val = self.pop_stack()?;

                // Location to set it at in the target - this could be a literal or a reference, but
                // it doesn't matter which it is, we deref anyway as only literals can be object keys
                let key: HogLiteral = self.pop_stack_as()?;

                // This can be either a literal or a reference. If it's a reference, we have to deref_mut it
                // from the head, so we can modify it
                let mut target = self.pop_stack()?;
                let target = match &mut target {
                    HogValue::Ref(ptr) => self.heap.get_mut(*ptr)?,
                    HogValue::Lit(_) => {
                        // TODO - this is a divergence from the original implementation - basically, if the target you've specified isn't on the heap,
                        // we'll drop it immediately after setting the property, which is a no-op. This should never happen - we should be using GET_LOCAL
                        // to hoist properties onto the heap before calling SET_PROPERTY, I /think/, but I'm not certain. I might end up making all
                        // objects be auto-allocated on the heap, just to be certain we're never hit this case for anything set_property() would succeed for
                        return Err(VmError::ExpectedObject);
                    }
                };

                let mut allocated_bytes = val.size();
                let key_bytes = key.size();
                let freed_bytes = target.set_property(key, val)?;
                if freed_bytes == 0 {
                    allocated_bytes += key_bytes; // We inserted a new key as well as a new prop
                }

                // This doesn't assert the allocation is possible - the next one will fail, which is good enough
                let new_bytes = self
                    .heap
                    .current_bytes
                    .saturating_sub(freed_bytes)
                    .saturating_add(allocated_bytes);
                self.heap.set_current_bytes(new_bytes);
            }
            Operation::Try => {
                // i32 to permit setting a catch offset lower than the IP
                let catch_offset: i32 = self.next_i32()?;
                // The reference computes `catchIp = <TRY position> + 1 + offset`. After reading the
                // opcode and offset, self.ip points two past the TRY position, so subtract one to land
                // on the same instruction (the compiler's offset already skips the POP_TRY).
                let catch_ip = (self.ip as i64)
                    .checked_add(catch_offset as i64 - 1)
                    .ok_or_else(|| VmError::IntegerOverflow)?
                    as usize;
                let frame = ThrowFrame {
                    catch_ptr: catch_ip,
                    catch_symbol: self.current_symbol.clone(),
                    stack_start: self.stack.len(),
                    call_depth: self.stack_frames.len(),
                };
                self.throw_frames.push(frame);
            }
            Operation::PopTry => {
                if self.throw_frames.pop().is_none() {
                    return Err(VmError::UnexpectedPopTry);
                };
            }
            Operation::Throw => {
                let exception = self.pop_stack()?;
                let type_key: HogValue = HogLiteral::from("type".to_string()).into();
                let _type = exception.get_nested(&[type_key], &self.heap)?;
                let message_key = HogLiteral::from("message".to_string()).into();
                let message = exception.get_nested(&[message_key], &self.heap)?;
                // The other impls here have some special case handling that treats "Error" as a distinct type, but
                // as far as I can tell, a "hog error" is just a HogValue::Object with some specific properties, so
                // I'll just check those exist. hog is mostly duck-typed, based on the existing impls
                if _type.is_none() || message.is_none() {
                    return Err(VmError::InvalidException);
                };

                let Some(frame) = self.throw_frames.pop() else {
                    // TODO - we need some helper that'll deeply clone a HogValue::Object and product a HashMap<String, HogLiteral>, since
                    // this is escaping the VM, so heap references can't leak.
                    // let payload_key = HogLiteral::from("payload".to_string()).into();
                    // let payload = exception.get_nested(&[payload_key], &self.heap)?;
                    let _type: &str = _type.unwrap().deref(&self.heap)?.try_as()?;
                    let _message: &str = message.unwrap().deref(&self.heap)?.try_as()?;
                    return Err(VmError::UncaughtException(
                        _type.to_string(),
                        _message.to_string(),
                    ));
                };

                self.truncate_stack(frame.stack_start)?;
                self.stack_frames.truncate(frame.call_depth);
                self.ip = frame.catch_ptr;
                self.set_chunk(frame.catch_symbol)?;
                self.push_stack(exception)?;
            }
            Operation::Callable => {
                // Construct a locally callable object - this is how e.g. fn decl's work
                let name = self.next_str()?;
                let stack_arg_count: usize = self.next_usize()?;
                let captured_arg_count: usize = self.next_usize()?;
                let body_length: usize = self.next_usize()?;
                let callable: Callable = LocalCallable {
                    name: name.to_string(),
                    stack_arg_count,
                    capture_count: captured_arg_count,
                    ip: self.ip,
                    symbol: self.current_symbol.clone(), // Cross-module jumps are currently only done via CallGlobal
                }
                .into();
                self.push_stack(HogLiteral::Callable(Box::new(callable)))?;
                self.ip = self
                    .ip
                    .checked_add(body_length)
                    .ok_or_else(|| VmError::IntegerOverflow)?;
            }
            // A closure is a callable, plus some captured arguments from the scope
            // it was constructed in.
            Operation::Closure => {
                let callable: Callable = self.pop_stack_as()?;
                let capture_count: usize = self.next_usize()?;
                // Only hog-local callables are wrapped by the CLOSURE opcode; native (STL) callables
                // are produced already-closed by GetGlobal, so they should never reach here.
                let Callable::Local(unwrapped) = &callable else {
                    return Err(VmError::InvalidCall(
                        "cannot close over a native function".to_string(),
                    ));
                };
                // Because captures are an implicit part of compilation, it's never valid
                // for them to be different from the number of captures in the callable.
                if capture_count != unwrapped.capture_count {
                    return Err(VmError::InvalidCall(
                        "invalid number of heap arguments".to_string(),
                    ));
                }
                let mut captures = Vec::with_capacity(capture_count);
                for _ in 0..capture_count {
                    // Indicates whether this captured value is a local in this frame's stack, or an
                    // upvalue already captured by this frame's closure.
                    let is_local: bool = self.next_bool()?;
                    let offset: usize = self.next_usize()?;
                    if is_local {
                        let location = self
                            .current_frame_base()
                            .checked_add(offset)
                            .ok_or_else(|| VmError::IntegerOverflow)?;
                        captures.push(self.capture_upvalue(location));
                    } else {
                        // Nested closure: share the parent frame's upvalue cell.
                        captures.push(self.frame_capture(offset)?);
                    }
                }

                let closure = Closure { callable, captures };
                self.push_stack(HogLiteral::Closure(Box::new(closure)))?;
            }
            Operation::CallLocal => {
                let closure: Closure = self.pop_stack_as()?;
                let arg_count: usize = self.next_usize()?;
                // Extract the scalar callable fields up front so `closure` can be moved into the frame
                // (which now retains it for snapshotting). A native-function value dispatches inline.
                let (ip, symbol, stack_arg_count) = match &closure.callable {
                    Callable::Local(callable) => (
                        callable.ip,
                        callable.symbol.clone(),
                        callable.stack_arg_count,
                    ),
                    // A first-class native function value (e.g. `let f := base64Encode; f(x)`):
                    // pop the args (push order, reversed for v>0 like CallGlobal) and dispatch natively.
                    Callable::Stl(name) => {
                        let name = name.clone();
                        let mut args = Vec::with_capacity(arg_count);
                        for _ in 0..arg_count {
                            args.push(self.pop_stack()?);
                        }
                        if self.context.version() != 0 {
                            args.reverse();
                        }
                        return Ok(self.prep_native_call(name, args));
                    }
                };
                if arg_count > stack_arg_count {
                    return Err(VmError::InvalidCall(format!(
                        "Too many args - expected {stack_arg_count}, got {arg_count}"
                    )));
                }
                let null_args = stack_arg_count.saturating_sub(arg_count);
                // For pure-hog function calls (calls to "local" callables), we just push a null onto the stack for each missing argument,
                // then construct the stack frame, and jump to the callable's frame ip. For other kinds of calls (which we don't support yet),
                // we'll have to do something more complicated
                for _ in 0..null_args {
                    self.push_stack(HogLiteral::Null)?;
                }
                let frame = CallFrame {
                    ret_ptr: self.ip,
                    ret_symbol: self.current_symbol.clone(),
                    stack_start: self.stack.len().saturating_sub(stack_arg_count),
                    closure,
                };
                self.stack_frames.push(frame);
                self.set_chunk(symbol)?; // Prep to jump across the module boundary, if that's what we're doing
                self.ip = ip; // Do the jump
            }
            Operation::GetUpvalue => {
                let index: usize = self.next_usize()?;
                let cell = self.frame_capture(index)?;
                // Open: read through to the live stack slot. Closed: read the snapshot it owns.
                let val = {
                    let uv = cell.borrow();
                    if uv.closed {
                        uv.value.clone().unwrap_or_else(|| HogLiteral::Null.into())
                    } else {
                        self.stack
                            .get(uv.location)
                            .cloned()
                            .ok_or_else(|| VmError::StackIndexOutOfBounds)?
                    }
                };
                self.push_stack(val)?;
            }
            Operation::SetUpvalue => {
                let index: usize = self.next_usize()?;
                let cell = self.frame_capture(index)?;
                let val = self.pop_stack()?;
                // Open: write through to the live stack slot. Closed: store into the snapshot.
                let (closed, location) = {
                    let uv = cell.borrow();
                    (uv.closed, uv.location)
                };
                if closed {
                    cell.borrow_mut().value = Some(val);
                } else {
                    self.set_stack_val(location, val)?;
                }
            }
            Operation::CloseUpvalue => {
                // Reference: `stackKeepFirstElements(stack.length - 1)` — close any upvalue viewing
                // the top slot (snapshot it), then drop the slot.
                let new_len = self.stack.len().saturating_sub(1);
                self.truncate_stack(new_len)?;
            }
        }

        Ok(StepOutcome::Continue)
    }

    // Returns the first stack item in this call frames scope
    fn current_frame_base(&self) -> usize {
        if self.stack_frames.is_empty() {
            0
        } else {
            self.stack_frames[self.stack_frames.len() - 1].stack_start
        }
    }

    // The upvalue cell captured at `index` by the currently-executing closure.
    fn frame_capture(&self, index: usize) -> Result<UpvalueCell, VmError> {
        let Some(frame) = self.stack_frames.last() else {
            return Err(VmError::NoFrame);
        };
        frame
            .closure
            .captures
            .get(index)
            .cloned()
            .ok_or_else(|| VmError::CaptureOutOfBounds(index))
    }

    // Capture the stack slot at `location` as an open upvalue, deduplicating so all closures that
    // capture the same live slot share one cell (and therefore observe the same close). Matches the
    // reference's `captureUpValue`.
    fn capture_upvalue(&mut self, location: usize) -> UpvalueCell {
        if let Some(existing) = self
            .open_upvalues
            .iter()
            .find(|cell| cell.borrow().location == location)
            .cloned()
        {
            return existing;
        }
        let cell = Rc::new(RefCell::new(Upvalue {
            location,
            closed: false,
            value: None,
        }));
        self.open_upvalues.push(cell.clone());
        cell
    }

    // Close every open upvalue whose slot is at or above `from` (i.e. about to leave scope):
    // snapshot the current stack value into the cell and drop it from the open list. The closures
    // holding the cell keep the snapshot. Matches the reference's `stackKeepFirstElements` close loop.
    fn close_upvalues(&mut self, from: usize) {
        if self.open_upvalues.is_empty() {
            return;
        }
        let mut i = 0;
        while i < self.open_upvalues.len() {
            let location = self.open_upvalues[i].borrow().location;
            if location >= from {
                let snapshot = self
                    .stack
                    .get(location)
                    .cloned()
                    .unwrap_or_else(|| HogLiteral::Null.into());
                {
                    let mut uv = self.open_upvalues[i].borrow_mut();
                    uv.closed = true;
                    uv.value = Some(snapshot);
                }
                self.open_upvalues.swap_remove(i);
            } else {
                i += 1;
            }
        }
    }

    // Change the chunk being executed, keeping `current_tokens` in lockstep with
    // `current_symbol`. Every assignment of `current_symbol` must go through here.
    fn set_chunk(&mut self, symbol: Option<Symbol>) -> Result<(), VmError> {
        self.current_tokens = self.context.chunk_tokens(&symbol)?;
        self.current_symbol = symbol;
        Ok(())
    }

    // Instruction fetches read the pre-decoded token stream — enum matches, no serde and no
    // allocation per step. Error strings are built only on an actual mismatch (malformed
    // bytecode), never on the hot path. The returned borrow is tied to the context's lifetime
    // (`'a`), not to `&mut self`, so callers can keep it across further VM mutation.
    fn next_token(&mut self) -> Result<&'a Token, VmError> {
        let token = self
            .current_tokens
            .get(self.ip)
            .ok_or_else(|| VmError::EndOfProgram(self.ip))?;
        self.ip += 1;
        Ok(token)
    }

    fn invalid_token(token: &Token, expected: &str) -> VmError {
        VmError::InvalidValue(token.type_name().to_string(), expected.to_string())
    }

    fn next_op(&mut self) -> Result<Operation, VmError> {
        match self.next_token()? {
            Token::Int(i) => Operation::from_i64(*i),
            token => Err(Self::invalid_token(token, "Operation")),
        }
    }

    fn next_usize(&mut self) -> Result<usize, VmError> {
        match self.next_token()? {
            Token::Int(i) if *i >= 0 => Ok(*i as usize),
            token => Err(Self::invalid_token(token, "usize")),
        }
    }

    fn next_i32(&mut self) -> Result<i32, VmError> {
        match self.next_token()? {
            Token::Int(i) => {
                i32::try_from(*i).map_err(|_| Self::invalid_token(&Token::Int(*i), "i32"))
            }
            token => Err(Self::invalid_token(token, "i32")),
        }
    }

    fn next_bool(&mut self) -> Result<bool, VmError> {
        match self.next_token()? {
            Token::Bool(b) => Ok(*b),
            token => Err(Self::invalid_token(token, "bool")),
        }
    }

    // Serde deserialized integer tokens into f64 fetches transparently; keep accepting both.
    fn next_f64(&mut self) -> Result<f64, VmError> {
        match self.next_token()? {
            Token::Float(f) => Ok(*f),
            Token::Int(i) => Ok(*i as f64),
            token => Err(Self::invalid_token(token, "f64")),
        }
    }

    fn next_str(&mut self) -> Result<&'a Arc<str>, VmError> {
        match self.next_token()? {
            Token::Str(s) => Ok(s),
            token => Err(Self::invalid_token(token, "String")),
        }
    }

    fn pop_stack(&mut self) -> Result<HogValue, VmError> {
        if self.stack.len() <= self.current_frame_base() {
            return Err(VmError::StackUnderflow);
        }
        self.stack.pop().ok_or_else(|| VmError::StackUnderflow)
    }

    // As above, but with a handy pointer chase and a cast/unwrap. Necessarily clones
    // if the stack item was a reference.
    fn pop_stack_as<T>(&mut self) -> Result<T, VmError>
    where
        T: FromHogLiteral,
    {
        match self.pop_stack()? {
            HogValue::Lit(lit) => lit.try_into(), // Purely an optimisation to skip a clone
            other => other.deref(&self.heap)?.clone().try_into(),
        }
    }

    // The reference VM applies JS arithmetic coercion, where null behaves as 0 (`5 - null` is
    // 5); real transformations do arithmetic on absent event properties.
    fn pop_arith_operand(&mut self) -> Result<Num, VmError> {
        let val = self.pop_stack()?;
        match val.deref(&self.heap)? {
            HogLiteral::Null => Ok(Num::Integer(0)),
            lit => lit.try_as::<Num>().cloned(),
        }
    }

    // The reference VM funnels both like/ilike operands through JS String coercion (the pattern
    // via String(...), the value via RegExp.test(...)), so null, booleans, and numbers stringify
    // instead of erroring — `null like '%x%'` tests the string "null". Containers stay errors: the
    // reference would produce "[object Object]", which no real program relies on.
    fn pop_like_operands(&mut self) -> Result<(String, String), VmError> {
        let val = self.pop_stack()?;
        let pat = self.pop_stack()?;
        Ok((self.js_string_coerce(&val)?, self.js_string_coerce(&pat)?))
    }

    fn js_string_coerce(&self, value: &HogValue) -> Result<String, VmError> {
        match value.deref(&self.heap)? {
            HogLiteral::String(s) => Ok(s.to_string()),
            HogLiteral::Null => Ok("null".to_string()),
            HogLiteral::Boolean(b) => Ok(b.to_string()),
            HogLiteral::Number(n) => Ok(match n {
                Num::Integer(i) => i.to_string(),
                Num::Float(f) => format!("{f}"),
            }),
            other => Err(VmError::InvalidValue(
                other.type_name().to_string(),
                "String".to_string(),
            )),
        }
    }

    /// `=~`/`!~` (and the case-insensitive variants). The stack holds the pattern below the value.
    /// Mirrors the reference's `regexMatch`: `pattern && value ? external.match(pattern, value) :
    /// false`, where the external matcher (RE2/RegExp `.test`) JS-String-coerces the value. So a
    /// falsy pattern or value (null, `false`, `0`, `""`, `NaN`) never matches, and scalar values
    /// (numbers, booleans) stringify just like `like`/`ilike` via `js_string_coerce`. Containers
    /// still error, matching the deliberate `pop_like_operands` deviation.
    fn regex_op(&mut self, case_sensitive: bool, negate: bool) -> Result<(), VmError> {
        let val = self.pop_stack()?;
        let pat = self.pop_stack()?;
        let matched = {
            let val_lit = val.deref(&self.heap)?;
            let pat_lit = pat.deref(&self.heap)?;
            if !val_lit.truthy() || !pat_lit.truthy() {
                false
            } else {
                regex_match(
                    &self.js_string_coerce(&val)?,
                    &self.js_string_coerce(&pat)?,
                    case_sensitive,
                )?
            }
        };
        self.push_stack(HogLiteral::Boolean(matched ^ negate))
    }

    /// `Gt`/`GtEq`/`Lt`/`LtEq` arm. The default path accepts numbers and lexicographically ordered
    /// numeric arrays. `sortableSemver` returns a numeric array, so server-side property filters need
    /// this narrow array case even when they retain strict comparison semantics. Other non-number
    /// operands still error, which lets cymbal auto-disable malformed rules. Only when the context
    /// opts into coercing comparisons does a non-number operand reach [`compare_values`]. `a` is the
    /// top of the stack.
    fn compare_op(&mut self, op: NumOp) -> Result<(), VmError> {
        let a = self.pop_stack()?;
        let b = self.pop_stack()?;
        // Scope the immutable heap borrows so the result is owned before the `&mut self` push.
        let result = {
            let a_lit = a.deref(&self.heap)?;
            let b_lit = b.deref(&self.heap)?;
            if matches!(a_lit, HogLiteral::Null) || matches!(b_lit, HogLiteral::Null) {
                // SQL semantics, shared with the Node and Python VMs: a null on either side is no match.
                HogLiteral::Boolean(false)
            } else if self.context.coerce_comparisons {
                compare_values(op, a_lit, b_lit, &self.heap)?
            } else if let (HogLiteral::Array(a_values), HogLiteral::Array(b_values)) =
                (a_lit, b_lit)
            {
                let ordering = compare_numeric_arrays(a_values, b_values, &self.heap)?;
                HogLiteral::Boolean(ordering_matches(op, ordering))
            } else {
                let a_num: &Num = a_lit.try_as()?;
                let b_num: &Num = b_lit.try_as()?;
                Num::binary_op(op, a_num, b_num)?
            }
        };
        self.push_stack(result)
    }

    /// `Eq`/`NotEq` core. Both paths apply structural equality with numeric unification, so
    /// `1 == 1.0` holds for every shared-crate consumer (see [`PartialEq for Num`](crate::Num)).
    /// The flag gates only what sits on top: when the context opts into coercing comparisons (the
    /// realtime-cohort evaluator), a temporal pair compares by epoch seconds to match ClickHouse
    /// (`is_date_exact`); every non-temporal pair compares through [`Self::values_equal`].
    ///
    /// "Temporal pair" is [`temporal_seconds_pair`] — the same predicate [`compare_values`] uses for
    /// ordering, so `timestamp == toDateTime(...)` and `timestamp > toDateTime(...)` agree on which
    /// operands are dates. The Python/TS reference VMs route all six comparison opcodes through a
    /// single coercion function, so equality has to cover the bare-field string case too, not just
    /// the both-temporal one.
    fn eq_op(&self, a: &HogValue, b: &HogValue) -> Result<HogLiteral, VmError> {
        if self.context.coerce_comparisons {
            let (lhs, rhs) = (a.deref(&self.heap)?, b.deref(&self.heap)?);
            if let Some((x, y)) = temporal_seconds_pair(lhs, rhs, &self.heap) {
                return Ok((x == y).into());
            }
        }
        Ok(self.values_equal(a, b)?.into())
    }

    /// Shared by `==` and every membership test (`in`, `has`, `indexOf`, `in()`) so that they agree.
    pub(crate) fn values_equal(&self, a: &HogValue, b: &HogValue) -> Result<bool, VmError> {
        if self.context.boolean_text_equality {
            if let Some(equal) = boolean_text_equals(a.deref(&self.heap)?, b.deref(&self.heap)?) {
                return Ok(equal);
            }
        }
        Ok(*a.equals(b, &self.heap)?.try_as::<bool>()?)
    }

    fn in_op(&self, haystack: &HogValue, needle: &HogValue) -> Result<HogLiteral, VmError> {
        if !self.context.boolean_text_equality {
            return haystack.contains(needle, &self.heap);
        }
        let (HogLiteral::Array(items) | HogLiteral::Tuple(items)) = haystack.deref(&self.heap)?
        else {
            return haystack.contains(needle, &self.heap);
        };
        for item in items {
            if self.values_equal(item, needle)? {
                return Ok(true.into());
            }
        }
        Ok(false.into())
    }

    // Move a value from the stack onto the heap, replacing it with a reference to the heap value,
    // and returning a reference to it. If it was already a reference, return it.
    fn hoist(&mut self, idx: usize) -> Result<HeapReference, VmError> {
        let item = self.clone_stack_item(idx)?;
        match item {
            HogValue::Lit(lit) => {
                let ptr = self.heap.emplace(lit)?;
                self.set_stack_val(idx, ptr)?;
                Ok(ptr)
            }
            HogValue::Ref(ptr) => Ok(ptr),
        }
    }

    fn set_stack_val(&mut self, idx: usize, value: impl Into<HogValue>) -> Result<(), VmError> {
        // TODO - revisit this, idk about it
        self.stack
            .get_mut(idx)
            .ok_or_else(|| VmError::StackIndexOutOfBounds)
            .map(|v| *v = value.into())
    }

    fn truncate_stack(&mut self, new_len: usize) -> Result<(), VmError> {
        if new_len > self.stack.len() {
            return Err(VmError::StackIndexOutOfBounds);
        }
        // Any upvalue viewing a slot that's about to disappear must be closed (snapshotted) first,
        // so closures that captured it keep the value rather than a dangling stack index.
        self.close_upvalues(new_len);
        self.stack.truncate(new_len);
        Ok(())
    }

    pub(crate) fn push_stack(&mut self, value: impl Into<HogValue>) -> Result<(), VmError> {
        if self.stack.len() >= self.context.max_stack_depth {
            return Err(VmError::StackOverflow);
        }
        self.stack.push(value.into());
        Ok(())
    }

    // TODO - we don't support function imports right now - most trivial programs don't need them,
    // and filters are generally trivial
    // Resolve a global name referenced as a first-class function value. Currently supports native
    // (STL) functions — `let f := base64Encode` yields a closure that dispatches to the native fn.
    // Returns None when the chain isn't a reference to a known native function — the caller's
    // GetGlobal falls through to its next resolution strategy. A plain miss here is normal
    // control flow, so it must not construct an error (this sat on the hot path allocating
    // discarded error strings).
    fn get_fn_reference(&self, chain: &[HogValue]) -> Option<HogLiteral> {
        if chain.len() != 1 {
            return None;
        }
        let name: &str = chain[0].deref(&self.heap).ok()?.try_as().ok()?;
        if self.context.has_native(name) {
            return Some(HogLiteral::Closure(Box::new(Closure {
                callable: Callable::Stl(name.to_string()),
                captures: Vec::new(),
            })));
        }
        None
    }

    fn clone_stack_item(&self, idx: usize) -> Result<HogValue, VmError> {
        self.stack
            .get(idx)
            .cloned()
            .ok_or_else(|| VmError::StackUnderflow)
    }

    fn prep_native_call(&self, name: String, args: Vec<HogValue>) -> StepOutcome {
        StepOutcome::NativeCall(name, args)
    }

    fn prep_cross_module_call(
        &mut self,
        symbol: Symbol,
        arg_count: usize,
    ) -> Result<StepOutcome, VmError> {
        // See CallLocal for details on how this works, but effectively, a cross-module call is just a
        // local call plus a current "module/function" change
        let to_call = self.context.get_symbol(&symbol)?;
        if arg_count > to_call.arg_count() {
            return Err(VmError::InvalidCall(format!(
                "Too many args - expected {}, got {}",
                to_call.arg_count(),
                arg_count
            )));
        }
        let callee_arg_count = to_call.arg_count();
        let null_args = callee_arg_count.saturating_sub(arg_count);
        for _ in 0..null_args {
            self.push_stack(HogLiteral::Null)?;
        }
        // Cross-module calls never involve captures, but the frame still carries a synthesized
        // callable for its symbol so a snapshot can render the reference VM's per-frame closure
        // (e.g. `fn<arrayMap>` in chunk `stl/arrayMap`).
        let frame = CallFrame {
            ret_ptr: self.ip,
            ret_symbol: self.current_symbol.clone(),
            stack_start: self.stack.len().saturating_sub(callee_arg_count),
            closure: Closure {
                callable: Callable::Local(LocalCallable {
                    name: symbol.name.clone(),
                    stack_arg_count: callee_arg_count,
                    capture_count: 0,
                    ip: 0,
                    symbol: Some(symbol.clone()),
                }),
                captures: Vec::new(),
            },
        };

        self.stack_frames.push(frame);
        self.set_chunk(Some(symbol))?;
        self.ip = 0;

        Ok(StepOutcome::Continue)
    }

    // Construct a hog value from a Json object. If the json object would be heap allocated
    // as a HogValue (e.g. if it's an array or object), then it will be allocated onto the heap,
    // and a reference will be returned. Nested json objects are flattened during allocation,
    // such that e.g. [[1,2],[3,4]] would lead to an array of [HeapReference, HeapReference] being
    // put into the heap, and a HeapReference being returned.
    //
    // This is a function on the VM, rather than being standalone, because hog values don't really
    // exist outside of the context of a VM (and specifically a heap). It could be a function on the
    // heap itself, though.
    pub fn json_to_hog(&mut self, json: &JsonValue) -> Result<HogValue, VmError> {
        self.json_to_hog_impl(json, 0)
    }

    fn json_to_hog_impl(&mut self, current: &JsonValue, depth: usize) -> Result<HogValue, VmError> {
        if depth > MAX_JSON_SERDE_DEPTH {
            return Err(VmError::OutOfResource(
                "json->hog deserialization depth".to_string(),
            ));
        };

        // Clone only the scalar leaves (numbers/strings); containers are walked by reference and
        // rebuilt onto the heap.
        match current {
            JsonValue::Null => Ok(HogLiteral::Null.into()),
            JsonValue::Bool(b) => Ok(HogLiteral::Boolean(*b).into()),
            JsonValue::Number(n) => Ok(HogLiteral::Number(n.clone().into()).into()),
            JsonValue::String(s) => Ok(HogLiteral::from(s.clone()).into()),
            JsonValue::Array(arr) => {
                let mut values = Vec::new();
                for value in arr {
                    values.push(self.json_to_hog_impl(value, depth + 1)?);
                }
                let to_emplace = HogLiteral::Array(values);
                let ptr = self.heap.emplace(to_emplace)?;
                Ok(ptr.into())
            }
            JsonValue::Object(obj) => {
                let mut map = HogMap::default();
                for (key, value) in obj.iter() {
                    map.insert(key.clone(), self.json_to_hog_impl(value, depth + 1)?);
                }
                let to_emplace = HogLiteral::Object(Box::new(map));
                let ptr = self.heap.emplace(to_emplace)?;
                Ok(ptr.into())
            }
        }
    }

    // Convert back from an arbitrary HogValue to a json Value. Again, this function exists on
    // the VM, because HogValues don't really exist in any other context.
    pub fn hog_to_json(&self, value: &HogValue) -> Result<JsonValue, VmError> {
        self.hog_to_json_impl(value, 0)
    }

    fn hog_to_json_impl(&self, value: &HogValue, depth: usize) -> Result<JsonValue, VmError> {
        if depth > MAX_JSON_SERDE_DEPTH {
            return Err(VmError::OutOfResource(
                "hog->json serialization depth".to_string(),
            ));
        };

        let val = value.deref(&self.heap)?;
        match val {
            HogLiteral::Null => Ok(JsonValue::Null),
            HogLiteral::Boolean(b) => Ok(JsonValue::Bool(*b)),
            HogLiteral::Number(n) => Ok(JsonValue::Number(n.clone().try_into()?)),
            HogLiteral::String(s) => Ok(JsonValue::String(s.to_string())),
            HogLiteral::Array(arr) | HogLiteral::Tuple(arr) => {
                let mut json_arr = Vec::new();
                for elem in arr {
                    json_arr.push(self.hog_to_json_impl(elem, depth + 1)?);
                }
                Ok(JsonValue::Array(json_arr))
            }
            HogLiteral::Object(obj) => {
                let mut map = serde_json::Map::new();
                for (key, value) in obj.iter() {
                    map.insert(key.clone(), self.hog_to_json_impl(value, depth + 1)?);
                }
                Ok(JsonValue::Object(map))
            }
            HogLiteral::Callable(_) => Err(VmError::NotImplemented(
                "Callable serialisation".to_string(),
            )),
            HogLiteral::Closure(_) => {
                Err(VmError::NotImplemented("Closure serialisation".to_string()))
            }
        }
    }

    /// Capture the full live state as a serializable [`VmSnapshot`]. The program/STL are not part of
    /// the snapshot — [`resume`] re-supplies them via the `ExecutionContext`. The upvalue graph
    /// (`Rc<RefCell<Upvalue>>`, shared between the open list and closures) is flattened to ids.
    pub fn snapshot(&self) -> Result<VmSnapshot, VmError> {
        let mut registry = CellRegistry::default();
        // Intern open upvalues first (stable low ids), then everything reachable from the stack and
        // frame captures, then chase closed-cell values for nested closures.
        for cell in &self.open_upvalues {
            registry.intern(cell);
        }
        for value in &self.stack {
            collect_cells(value, &self.heap, &mut registry)?;
        }
        for frame in &self.stack_frames {
            for cell in &frame.closure.captures {
                registry.intern(cell);
            }
        }
        close_over_cells(&self.heap, &mut registry)?;

        // Read each cell's data out before serializing (so the value walk isn't aliasing the
        // registry it borrows; it is already complete, so no new ids are minted).
        let cell_data: Vec<(usize, bool, Option<HogValue>)> = registry
            .cells()
            .iter()
            .map(|c| {
                let b = c.borrow();
                (b.location, b.closed, b.value.clone())
            })
            .collect();

        let header_len = self.context.program_header_len();
        let mut stack = Vec::with_capacity(self.stack.len());
        for value in &self.stack {
            stack.push(value_to_json(value, &self.heap, &mut registry, header_len)?);
        }

        let mut upvalues = Vec::with_capacity(cell_data.len());
        for (id, (location, closed, value)) in cell_data.into_iter().enumerate() {
            let value_json = match (closed, &value) {
                (true, Some(v)) => value_to_json(v, &self.heap, &mut registry, header_len)?,
                _ => JsonValue::Null,
            };
            upvalues.push(UpvalueJson {
                marker: true,
                // 1-based, matching the reference (`id: sortedUpValues.length + 1`).
                id: id + 1,
                location,
                closed,
                value: value_json,
            });
        }

        // The reference keeps upvalues sorted by location so its capture early-break stays valid on
        // resume; each entry keeps its id so closure references still resolve.
        upvalues.sort_by_key(|u| u.location);

        let call_stack = self.build_call_stack(&mut registry, header_len)?;

        let throw_stack = self
            .throw_frames
            .iter()
            .map(|t| ThrowFrameJson {
                // The reference's callStackLen counts its frames — our depth plus the root frame.
                call_stack_len: t.call_depth + 1,
                stack_len: t.stack_start,
                catch_ip: self.node_ip(t.catch_ptr, &t.catch_symbol),
            })
            .collect();

        // The reference stores `bytecodes.root` as just `{ bytecode }` (globals are re-supplied via
        // options on resume), so omit globals to match its wire shape.
        Ok(VmSnapshot {
            bytecodes: json!({ "root": { "bytecode": self.context.program_tokens() } }),
            stack,
            upvalues,
            call_stack,
            throw_stack,
            declared_functions: json!({}),
            ops: self.ops,
            async_steps: self.async_steps,
            sync_duration: 0,
            max_mem_used: self.heap.peak_bytes,
            telemetry: self
                .context
                .collect_telemetry
                .then(|| self.telemetry.clone()),
        })
    }

    /// Build the reference's `callStack` (root frame first, active frame last). Frame `d` < N draws
    /// its ip/chunk from the frame pushed when depth `d` called `d+1` (which stored `d`'s resume
    /// point) and its closure/stackStart from that same callee frame; depth N is the live position.
    fn build_call_stack(
        &self,
        registry: &mut CellRegistry,
        header_len: usize,
    ) -> Result<Vec<CallFrameJson>, VmError> {
        let n = self.stack_frames.len();
        let mut frames = Vec::with_capacity(n + 1);
        for d in 0..=n {
            let (rust_ip, symbol) = if d == n {
                (self.ip, self.current_symbol.clone())
            } else {
                (
                    self.stack_frames[d].ret_ptr,
                    self.stack_frames[d].ret_symbol.clone(),
                )
            };
            let (closure, stack_start, arg_count) = if d == 0 {
                (root_closure_json(), 0, 0)
            } else {
                let frame = &self.stack_frames[d - 1];
                let arg_count = match &frame.closure.callable {
                    Callable::Local(lc) => lc.stack_arg_count,
                    Callable::Stl(_) => 0,
                };
                (
                    closure_to_json(&frame.closure, registry, header_len),
                    frame.stack_start,
                    arg_count,
                )
            };
            frames.push(CallFrameJson {
                closure,
                ip: self.node_ip(rust_ip, &symbol),
                chunk: chunk_str(&symbol),
                stack_start,
                arg_count,
            });
        }
        Ok(frames)
    }

    /// Our `ip` is body-relative; the reference's per-frame `ip` indexes the full bytecode array
    /// (header included) for the root chunk. Module chunks (STL bodies) carry no header.
    fn node_ip(&self, rust_ip: usize, symbol: &Option<Symbol>) -> usize {
        rust_ip
            + if symbol.is_none() {
                self.context.program_header_len()
            } else {
                0
            }
    }

    fn rust_ip(node_ip: usize, chunk: &str, context: &ExecutionContext) -> usize {
        node_ip.saturating_sub(if chunk == "root" {
            context.program_header_len()
        } else {
            0
        })
    }

    /// Rebuild a VM from a [`VmSnapshot`] against `context`. Inverse of [`Self::snapshot`]: cells are
    /// created empty first so closure captures resolve by id, containers are re-emplaced onto a fresh
    /// heap, and the open-upvalue list is reconstructed from the still-open cells.
    pub fn restore(
        context: &'a ExecutionContext,
        snapshot: &VmSnapshot,
    ) -> Result<HogVM<'a>, VmError> {
        let mut vm = HogVM::new(context)?;
        let header_len = context.program_header_len();

        let cells = rebuild_cells(&snapshot.upvalues, &mut vm.heap, header_len)?;

        let mut stack = Vec::with_capacity(snapshot.stack.len());
        for value in &snapshot.stack {
            stack.push(value_from_json(value, &mut vm.heap, &cells, header_len)?);
        }

        // Inverse of `build_call_stack`: the reference callStack has N+1 frames (root + N); we keep N
        // frames plus the live ip/symbol. Frame i bundles depth-i's return point (callStack[i]'s
        // ip/chunk) with depth-(i+1)'s closure and stack window.
        let call_stack = &snapshot.call_stack;
        let Some(active) = call_stack.last() else {
            return Err(VmError::Other("snapshot callStack is empty".to_string()));
        };
        vm.ip = Self::rust_ip(active.ip, &active.chunk, context);
        vm.set_chunk(chunk_to_symbol(&active.chunk))?;

        let n = call_stack.len() - 1;
        let mut stack_frames = Vec::with_capacity(n);
        for i in 0..n {
            stack_frames.push(CallFrame {
                ret_ptr: Self::rust_ip(call_stack[i].ip, &call_stack[i].chunk, context),
                ret_symbol: chunk_to_symbol(&call_stack[i].chunk),
                stack_start: call_stack[i + 1].stack_start,
                closure: closure_from_json(&call_stack[i + 1].closure, &cells, header_len)?,
            });
        }

        let mut throw_frames = Vec::with_capacity(snapshot.throw_stack.len());
        for t in &snapshot.throw_stack {
            // callStackLen counts reference frames (root + our depth); recover our depth and the
            // chunk that depth ran in so we can restore catch_symbol.
            let call_depth = t.call_stack_len.saturating_sub(1);
            let chunk = call_stack
                .get(call_depth)
                .map(|f| f.chunk.as_str())
                .unwrap_or("root");
            throw_frames.push(ThrowFrame {
                catch_ptr: Self::rust_ip(t.catch_ip, chunk, context),
                catch_symbol: chunk_to_symbol(chunk),
                stack_start: t.stack_len,
                call_depth,
            });
        }

        let open_upvalues = cells
            .values()
            .filter(|c| !c.borrow().closed)
            .cloned()
            .collect();

        vm.stack = stack;
        vm.stack_frames = stack_frames;
        vm.throw_frames = throw_frames;
        vm.open_upvalues = open_upvalues;
        vm.async_steps = snapshot.async_steps;
        vm.ops = snapshot.ops;
        // Carry the recorded high-water mark across resume — rebuilding the heap only accounts for
        // live values, so without this a restore→snapshot round-trip would lose the original peak.
        vm.heap.peak_bytes = vm.heap.peak_bytes.max(snapshot.max_mem_used);
        // Carry the trace across resume so a multi-suspension run accumulates one continuous trace.
        vm.telemetry = snapshot.telemetry.clone().unwrap_or_default();
        Ok(vm)
    }
}

// Helper function to simply run a program until it either finishes or returns an error
pub fn sync_execute(context: &ExecutionContext, print_debug: bool) -> Result<JsonValue, VmFailure> {
    let fail = |e, vm: Option<&HogVM>, step: usize| VmFailure {
        error: e,
        ip: vm.map_or(0, |vm| vm.ip),
        stack: vm.map_or(Vec::new(), |vm| vm.stack.clone()),
        step,
    };

    let mut vm = HogVM::new(context).map_err(|e| fail(e, None, 0))?;

    let mut i = 0;
    while i < context.max_steps {
        let res = if print_debug {
            vm.debug_step(&|s| println!("{s}"))
                .map_err(|e| fail(e, Some(&vm), i))?
        } else {
            vm.step().map_err(|e| fail(e, Some(&vm), i))?
        };

        match res {
            StepOutcome::Continue => {}
            StepOutcome::Finished(res) => return Ok(res),
            StepOutcome::NativeCall(name, args) => {
                // Sync execution can't suspend: an async function name isn't a registered native, so
                // this surfaces as an `UnknownFunction` error here — the correct outcome for a sync
                // consumer that hits an async call. Resumable execution handles it via [`resume`].
                match context.execute_native_function_call(&mut vm, &name, args) {
                    Ok(_) => {}
                    Err(err) => return Err(fail(err, Some(&vm), i)),
                };
            }
        }
        i += 1;
    }

    let err = VmError::OutOfResource("steps".to_string());

    Err(fail(err, Some(&vm), i))
}

/// Drive a fresh program to completion or its first async suspension — the resumable counterpart to
/// [`sync_execute`]. On a registered async call it returns [`Resumable::Suspended`] with a
/// serializable snapshot; the host performs the side effect and calls [`resume`].
pub fn execute_resumable(context: &ExecutionContext) -> Result<Resumable, VmFailure> {
    let vm = HogVM::new(context).map_err(|e| failure(e, None, 0))?;
    drive_resumable(vm, context)
}

/// Resume a snapshot with the result of the async function it suspended on. The result is pushed as
/// the value the suspended `CALL_GLOBAL` yields, and execution continues from where it left off.
pub fn resume(
    context: &ExecutionContext,
    state: &VmSnapshot,
    async_result: JsonValue,
) -> Result<Resumable, VmFailure> {
    let mut vm = HogVM::restore(context, state).map_err(|e| failure(e, None, 0))?;
    let value = vm
        .json_to_hog(&async_result)
        .map_err(|e| failure(e, Some(&vm), 0))?;
    vm.push_stack(value).map_err(|e| failure(e, Some(&vm), 0))?;
    drive_resumable(vm, context)
}

fn drive_resumable(mut vm: HogVM, context: &ExecutionContext) -> Result<Resumable, VmFailure> {
    let mut i = 0;
    while i < context.max_steps {
        let res = vm.step().map_err(|e| failure(e, Some(&vm), i))?;
        match res {
            StepOutcome::Continue => {}
            StepOutcome::Finished(result) => return Ok(Resumable::Finished(result)),
            StepOutcome::NativeCall(name, args) => {
                // A registered async function suspends instead of running inline: snapshot the state
                // (args already popped, `ip` past the call) and hand it back for the host to perform
                // the side effect and `resume`. Everything else is a normal inline native call.
                if context.is_async(&name) {
                    if vm.async_steps >= context.max_async_steps {
                        return Err(failure(
                            VmError::OutOfResource(format!(
                                "async steps ({})",
                                context.max_async_steps
                            )),
                            Some(&vm),
                            i,
                        ));
                    }
                    vm.async_steps += 1;
                    let mut json_args = Vec::with_capacity(args.len());
                    for arg in &args {
                        json_args.push(vm.hog_to_json(arg).map_err(|e| failure(e, Some(&vm), i))?);
                    }
                    let state = vm.snapshot().map_err(|e| failure(e, Some(&vm), i))?;
                    return Ok(Resumable::Suspended {
                        function: name,
                        args: json_args,
                        state: Box::new(state),
                    });
                }
                context
                    .execute_native_function_call(&mut vm, &name, args)
                    .map_err(|e| failure(e, Some(&vm), i))?;
            }
        }
        i += 1;
    }
    Err(failure(
        VmError::OutOfResource("steps".to_string()),
        Some(&vm),
        i,
    ))
}

fn failure(error: VmError, vm: Option<&HogVM>, step: usize) -> VmFailure {
    VmFailure {
        error,
        ip: vm.map_or(0, |vm| vm.ip),
        stack: vm.map_or(Vec::new(), |vm| vm.stack.clone()),
        step,
    }
}

fn compare_numeric_arrays(
    a: &[HogValue],
    b: &[HogValue],
    heap: &VmHeap,
) -> Result<Ordering, VmError> {
    for value in a.iter().chain(b) {
        let _: &Num = value.deref(heap)?.try_as()?;
    }

    for (a_value, b_value) in a.iter().zip(b) {
        let a_num: &Num = a_value.deref(heap)?.try_as()?;
        let b_num: &Num = b_value.deref(heap)?.try_as()?;
        let ordering = a_num.compare(b_num);
        if ordering != Ordering::Equal {
            return Ok(ordering);
        }
    }
    Ok(a.len().cmp(&b.len()))
}

fn ordering_matches(op: NumOp, ordering: Ordering) -> bool {
    match op {
        NumOp::Gt => ordering.is_gt(),
        NumOp::Gte => ordering.is_ge(),
        NumOp::Lt => ordering.is_lt(),
        NumOp::Lte => ordering.is_le(),
        _ => unreachable!("ordering comparison requires a comparison operation"),
    }
}

/// The reference's telemetry opcode label: `"<num>/<SCREAMING_SNAKE_NAME>"` (e.g. `2/CALL_GLOBAL`).
fn op_telemetry_name(op: &Operation) -> String {
    let pascal = format!("{op:?}");
    let mut name = String::with_capacity(pascal.len() + 4);
    for (i, c) in pascal.char_indices() {
        if c.is_ascii_uppercase() && i != 0 {
            name.push('_');
        }
        name.push(c.to_ascii_uppercase());
    }
    format!("{}/{}", *op as u8, name)
}