:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): treat hog vm index errors as broken evaluation source (#114636)
posthog[bot]b7a7bc9adb8b
diff --git a/posthog/temporal/ai_observability/evaluation_hog.py b/posthog/temporal/ai_observability/evaluation_hog.pyindex a236cd10803a..795a77219551 100644--- a/posthog/temporal/ai_observability/evaluation_hog.py+++ b/posthog/temporal/ai_observability/evaluation_hog.py@@ -48,6 +48,10 @@ logger = structlog.get_logger(__name__)# Such a source fails on every unit, and `finalize_hog_eval_result` makes that a terminal# hog_error which disables the evaluation. Neither error may count as an input error, or the# broken source would skip every unit silently and blame the customer's data.+#+# The VM does raise IndexError for one source shape: a `throw` after a function returned from+# inside `try`. The VM keeps the returned frame's handler and jumps into a frame that no longer+# exists. `execute_hog_eval_bytecode` treats that IndexError as a broken source, like HogVMException.HOG_INPUT_ERROR_TYPES: tuple[type[Exception], ...] = (TypeError,AttributeError,@@ -182,6 +186,8 @@ def execute_hog_eval_bytecode(return {"verdict": None, "reasoning": "", "error": "Memory limit exceeded"}except HogVMException as e:return {"verdict": None, "reasoning": "", "error": f"Runtime error: {e}"}+ except IndexError as e:+ return {"verdict": None, "reasoning": "", "error": f"Runtime error: {type(e).__name__}: {e}"}except HOG_INPUT_ERROR_TYPES as e:return {"verdict": None,diff --git a/posthog/temporal/ai_observability/test_evaluation_hog.py b/posthog/temporal/ai_observability/test_evaluation_hog.pyindex 24fa9abac0b7..a70668ae1c81 100644--- a/posthog/temporal/ai_observability/test_evaluation_hog.py+++ b/posthog/temporal/ai_observability/test_evaluation_hog.py@@ -33,6 +33,25 @@ MISSING_ARGUMENT_SOURCE = """return jsonParse()"""+# The VM keeps the handler of `parse` after `parse` returns inside `try`, so the later uncaught throw+# jumps into a frame that no longer exists and the VM raises IndexError.+THROW_AFTER_RETURN_IN_TRY_SOURCE = """+fun parse(text) {+ try {+ return jsonParse(text)+ } catch (e) {+ return null+ }+}+fun fail(reason) {+ throw Error(reason)+}+let parsed := parse('{"score": 1}')+let score := parsed.score+let threshold := 2+return fail('score below threshold')+"""+EVALUATION = {"id": "01890000-0000-0000-0000-000000000000", "team_id": 1}@@ -199,12 +218,18 @@ class TestHogInputErrorClassification:assert result["verdict"] is False# A source that fails on every unit must not be classified as an input error, or it skips- # forever while blaming the customer's data. A wrong-arity builtin is a broken source: the VM- # raises HogVMException, which is neither an input error nor our bug. finalize_hog_eval_result- # turns it into a terminal hog_error that disables the evaluation and tells the user, instead of- # paging us once per unit.- def test_wrong_arity_builtin_is_a_broken_source(self) -> None:- result = run_source(MISSING_ARGUMENT_SOURCE)+ # forever while blaming the customer's data. A wrong-arity builtin or an uncaught throw is a+ # broken source: the VM raises HogVMException or IndexError, which is neither an input error nor our bug.+ # finalize_hog_eval_result turns it into a terminal hog_error that disables the evaluation and+ # tells the user, instead of paging us once per unit.+ @parameterized.expand(+ [+ ("wrong_arity_builtin", MISSING_ARGUMENT_SOURCE),+ ("throw_after_return_in_try", THROW_AFTER_RETURN_IN_TRY_SOURCE),+ ]+ )+ def test_broken_source_disables_the_evaluation(self, _name: str, source: str) -> None:+ result = run_source(source)assert result["error"] is not Noneassert "user_input_error" not in result