Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): treat hog vm index errors as broken evaluation source (#114636)

posthog[bot]b7a7bc9adb8b
diff --git a/posthog/temporal/ai_observability/evaluation_hog.py b/posthog/temporal/ai_observability/evaluation_hog.py
index a236cd10803a..795a77219551 100644
--- a/posthog/temporal/ai_observability/evaluation_hog.py
+++ b/posthog/temporal/ai_observability/evaluation_hog.py
@@ -48,6 +48,10 @@ logger = structlog.get_logger(__name__)
# Such a source fails on every unit, and `finalize_hog_eval_result` makes that a terminal
# hog_error which disables the evaluation. Neither error may count as an input error, or the
# broken source would skip every unit silently and blame the customer's data.
+#
+# The VM does raise IndexError for one source shape: a `throw` after a function returned from
+# inside `try`. The VM keeps the returned frame's handler and jumps into a frame that no longer
+# exists. `execute_hog_eval_bytecode` treats that IndexError as a broken source, like HogVMException.
HOG_INPUT_ERROR_TYPES: tuple[type[Exception], ...] = (
TypeError,
AttributeError,
@@ -182,6 +186,8 @@ def execute_hog_eval_bytecode(
return {"verdict": None, "reasoning": "", "error": "Memory limit exceeded"}
except HogVMException as e:
return {"verdict": None, "reasoning": "", "error": f"Runtime error: {e}"}
+ except IndexError as e:
+ return {"verdict": None, "reasoning": "", "error": f"Runtime error: {type(e).__name__}: {e}"}
except HOG_INPUT_ERROR_TYPES as e:
return {
"verdict": None,
diff --git a/posthog/temporal/ai_observability/test_evaluation_hog.py b/posthog/temporal/ai_observability/test_evaluation_hog.py
index 24fa9abac0b7..a70668ae1c81 100644
--- a/posthog/temporal/ai_observability/test_evaluation_hog.py
+++ b/posthog/temporal/ai_observability/test_evaluation_hog.py
@@ -33,6 +33,25 @@ MISSING_ARGUMENT_SOURCE = """
return jsonParse()
"""
+# The VM keeps the handler of `parse` after `parse` returns inside `try`, so the later uncaught throw
+# jumps into a frame that no longer exists and the VM raises IndexError.
+THROW_AFTER_RETURN_IN_TRY_SOURCE = """
+fun parse(text) {
+ try {
+ return jsonParse(text)
+ } catch (e) {
+ return null
+ }
+}
+fun fail(reason) {
+ throw Error(reason)
+}
+let parsed := parse('{"score": 1}')
+let score := parsed.score
+let threshold := 2
+return fail('score below threshold')
+"""
+
EVALUATION = {"id": "01890000-0000-0000-0000-000000000000", "team_id": 1}
@@ -199,12 +218,18 @@ class TestHogInputErrorClassification:
assert result["verdict"] is False
# A source that fails on every unit must not be classified as an input error, or it skips
- # forever while blaming the customer's data. A wrong-arity builtin is a broken source: the VM
- # raises HogVMException, which is neither an input error nor our bug. finalize_hog_eval_result
- # turns it into a terminal hog_error that disables the evaluation and tells the user, instead of
- # paging us once per unit.
- def test_wrong_arity_builtin_is_a_broken_source(self) -> None:
- result = run_source(MISSING_ARGUMENT_SOURCE)
+ # forever while blaming the customer's data. A wrong-arity builtin or an uncaught throw is a
+ # broken source: the VM raises HogVMException or IndexError, which is neither an input error nor our bug.
+ # finalize_hog_eval_result turns it into a terminal hog_error that disables the evaluation and
+ # tells the user, instead of paging us once per unit.
+ @parameterized.expand(
+ [
+ ("wrong_arity_builtin", MISSING_ARGUMENT_SOURCE),
+ ("throw_after_return_in_try", THROW_AFTER_RETURN_IN_TRY_SOURCE),
+ ]
+ )
+ def test_broken_source_disables_the_evaluation(self, _name: str, source: str) -> None:
+ result = run_source(source)
assert result["error"] is not None
assert "user_input_error" not in result