:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): preserve generation output before sampling judge input (#122025)
Bernat Torresacb63f6e22d8
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex b7d089f7825f..890d5644fb73 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -35,7 +35,8 @@ If the assembled fallback session still exceeds 500,000 characters, the evaluati## Generation formattingGeneration evaluations extract message text without the per-message character cutoff.-They sample the combined input, tool definitions, and output only when that text exceeds 150,000 characters, with a final character slice enforcing the limit.+They reserve space for the complete output, then sample input and tool definitions into the remaining space within the 150,000-character limit.+If the output and section labels alone exceed that limit, the run is skipped with reason `generation_output_too_long` without calling the judge.Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_evaluation.py), [session judge](../../posthog/temporal/ai_observability/run_session_evaluation.py), and [generation judge](../../posthog/temporal/ai_observability/evaluation_llm_judge.py).diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.pyindex 1ff34d898afe..6879d7e6b7d8 100644--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py@@ -537,12 +537,22 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActisystem_prompt = build_system_prompt(prompt, allows_na, output_type=output_type, output_config=output_config)- sections = [f"Input: {input_data}"]+ output_section = f"\n\nOutput: {output_data}"+ context_budget = EVENT_TEXT_MAX_CHARS - len("Input: ") - len(output_section)+ if context_budget < 0:+ return build_skipped_evaluation_result(+ output_type=output_type,+ allows_na=allows_na,+ reasoning="Generation output exceeded the judge's text limit; evaluation skipped.",+ skip_reason="generation_output_too_long",+ )++ sections = [input_data]if tools_data:sections.append(f"Tools available:\n{tools_data}")- sections.append(f"Output: {output_data}")- user_prompt = "\n\n".join(sections)- user_prompt = truncate_by_line_sampling(user_prompt, EVENT_TEXT_MAX_CHARS)+ # Keep the full answer being graded outside the sampled context.+ context = truncate_by_line_sampling("\n\n".join(sections), context_budget)+ user_prompt = f"Input: {context}{output_section}"return call_llm_judge(evaluation=evaluation,diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.pyindex 87e5a12f5402..bf7da3f00279 100644--- a/posthog/temporal/ai_observability/test_run_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_evaluation.py@@ -1620,6 +1620,7 @@ class TestRunEvaluationWorkflow:raw_size = sum(len(message["content"]) for message in oversized_input)assert len(sent_prompt) <= EVENT_TEXT_MAX_CHARSassert len(sent_prompt) < raw_size+ assert sent_prompt.endswith("\n\nOutput: assistant: ok")@pytest.mark.parametrize("output_config,expected_verdict,expected_applicable",@@ -4474,6 +4475,68 @@ class TestJudgePromptAssembly:regressions in section ordering, tool catalog inclusion, and tool_call_idcorrelation that the unit tests for the helpers can't catch alone."""+ @pytest.mark.parametrize(+ "output",+ [+ pytest.param("The result is ready.\nDetails: all checks passed.\nEnd of response.", id="multiline"),+ pytest.param("x" * (EVENT_TEXT_MAX_CHARS - len("Input: \n\nOutput: ")), id="no_context_budget"),+ pytest.param("x" * (EVENT_TEXT_MAX_CHARS - len("Input: \n\nOutput: ") - 1), id="one_context_character"),+ ],+ )+ def test_prompt_bounds_tools_without_sampling_output(self, output: str) -> None:+ evaluation = {+ "evaluation_type": "llm_judge",+ "evaluation_config": {"prompt": "Is this correct?"},+ "output_type": "boolean",+ }+ event_data = create_mock_event_data(+ 1,+ properties={+ "$ai_input": "Check the result.",+ "$ai_output_choices": output,+ "$ai_tools": [{"type": "function", "function": {"name": "check_result", "description": "x" * 600_000}}],+ },+ )++ with patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as judge:+ execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))++ user_prompt = judge.call_args.kwargs["user_prompt"]+ assert len(user_prompt) <= EVENT_TEXT_MAX_CHARS+ assert user_prompt.endswith(f"\n\nOutput: {output}")+ assert user_prompt.startswith("Input: ")++ @pytest.mark.parametrize("output_type", ["boolean", "numeric", "categorical"])+ @pytest.mark.parametrize("allows_na", [False, True])+ def test_prompt_skips_output_that_cannot_fit(self, output_type: str, allows_na: bool) -> None:+ evaluation = {+ "evaluation_type": "llm_judge",+ "evaluation_config": {"prompt": "Is this correct?"},+ "output_type": output_type,+ "output_config": {+ "allows_na": allows_na,+ **({"options": [{"key": "correct", "label": "Correct"}]} if output_type == "categorical" else {}),+ },+ }+ event_data = create_mock_event_data(+ 1,+ properties={+ "$ai_input": "",+ "$ai_output_choices": "x" * (EVENT_TEXT_MAX_CHARS - len("Input: \n\nOutput: ") + 1),+ },+ )++ with patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as judge:+ result = execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))++ assert judge.call_count == 0+ assert result == build_skipped_evaluation_result(+ output_type=output_type,+ allows_na=allows_na,+ reasoning="Generation output exceeded the judge's text limit; evaluation skipped.",+ skip_reason="generation_output_too_long",+ )+def test_prompt_contains_input_tools_and_output_sections_in_order(self):evaluation = {"id": "eval-123",