Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): preserve generation output before sampling judge input (#122025)

Bernat Torresacb63f6e22d8
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index b7d089f7825f..890d5644fb73 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -35,7 +35,8 @@ If the assembled fallback session still exceeds 500,000 characters, the evaluati
## Generation formatting
Generation evaluations extract message text without the per-message character cutoff.
-They sample the combined input, tool definitions, and output only when that text exceeds 150,000 characters, with a final character slice enforcing the limit.
+They reserve space for the complete output, then sample input and tool definitions into the remaining space within the 150,000-character limit.
+If the output and section labels alone exceed that limit, the run is skipped with reason `generation_output_too_long` without calling the judge.
Implementation: [trace judge](../../posthog/temporal/ai_observability/run_trace_evaluation.py), [session judge](../../posthog/temporal/ai_observability/run_session_evaluation.py), and [generation judge](../../posthog/temporal/ai_observability/evaluation_llm_judge.py).
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 1ff34d898afe..6879d7e6b7d8 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -537,12 +537,22 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActi
system_prompt = build_system_prompt(prompt, allows_na, output_type=output_type, output_config=output_config)
- sections = [f"Input: {input_data}"]
+ output_section = f"\n\nOutput: {output_data}"
+ context_budget = EVENT_TEXT_MAX_CHARS - len("Input: ") - len(output_section)
+ if context_budget < 0:
+ return build_skipped_evaluation_result(
+ output_type=output_type,
+ allows_na=allows_na,
+ reasoning="Generation output exceeded the judge's text limit; evaluation skipped.",
+ skip_reason="generation_output_too_long",
+ )
+
+ sections = [input_data]
if tools_data:
sections.append(f"Tools available:\n{tools_data}")
- sections.append(f"Output: {output_data}")
- user_prompt = "\n\n".join(sections)
- user_prompt = truncate_by_line_sampling(user_prompt, EVENT_TEXT_MAX_CHARS)
+ # Keep the full answer being graded outside the sampled context.
+ context = truncate_by_line_sampling("\n\n".join(sections), context_budget)
+ user_prompt = f"Input: {context}{output_section}"
return call_llm_judge(
evaluation=evaluation,
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 87e5a12f5402..bf7da3f00279 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -1620,6 +1620,7 @@ class TestRunEvaluationWorkflow:
raw_size = sum(len(message["content"]) for message in oversized_input)
assert len(sent_prompt) <= EVENT_TEXT_MAX_CHARS
assert len(sent_prompt) < raw_size
+ assert sent_prompt.endswith("\n\nOutput: assistant: ok")
@pytest.mark.parametrize(
"output_config,expected_verdict,expected_applicable",
@@ -4474,6 +4475,68 @@ class TestJudgePromptAssembly:
regressions in section ordering, tool catalog inclusion, and tool_call_id
correlation that the unit tests for the helpers can't catch alone."""
+ @pytest.mark.parametrize(
+ "output",
+ [
+ pytest.param("The result is ready.\nDetails: all checks passed.\nEnd of response.", id="multiline"),
+ pytest.param("x" * (EVENT_TEXT_MAX_CHARS - len("Input: \n\nOutput: ")), id="no_context_budget"),
+ pytest.param("x" * (EVENT_TEXT_MAX_CHARS - len("Input: \n\nOutput: ") - 1), id="one_context_character"),
+ ],
+ )
+ def test_prompt_bounds_tools_without_sampling_output(self, output: str) -> None:
+ evaluation = {
+ "evaluation_type": "llm_judge",
+ "evaluation_config": {"prompt": "Is this correct?"},
+ "output_type": "boolean",
+ }
+ event_data = create_mock_event_data(
+ 1,
+ properties={
+ "$ai_input": "Check the result.",
+ "$ai_output_choices": output,
+ "$ai_tools": [{"type": "function", "function": {"name": "check_result", "description": "x" * 600_000}}],
+ },
+ )
+
+ with patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as judge:
+ execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))
+
+ user_prompt = judge.call_args.kwargs["user_prompt"]
+ assert len(user_prompt) <= EVENT_TEXT_MAX_CHARS
+ assert user_prompt.endswith(f"\n\nOutput: {output}")
+ assert user_prompt.startswith("Input: ")
+
+ @pytest.mark.parametrize("output_type", ["boolean", "numeric", "categorical"])
+ @pytest.mark.parametrize("allows_na", [False, True])
+ def test_prompt_skips_output_that_cannot_fit(self, output_type: str, allows_na: bool) -> None:
+ evaluation = {
+ "evaluation_type": "llm_judge",
+ "evaluation_config": {"prompt": "Is this correct?"},
+ "output_type": output_type,
+ "output_config": {
+ "allows_na": allows_na,
+ **({"options": [{"key": "correct", "label": "Correct"}]} if output_type == "categorical" else {}),
+ },
+ }
+ event_data = create_mock_event_data(
+ 1,
+ properties={
+ "$ai_input": "",
+ "$ai_output_choices": "x" * (EVENT_TEXT_MAX_CHARS - len("Input: \n\nOutput: ") + 1),
+ },
+ )
+
+ with patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as judge:
+ result = execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))
+
+ assert judge.call_count == 0
+ assert result == build_skipped_evaluation_result(
+ output_type=output_type,
+ allows_na=allows_na,
+ reasoning="Generation output exceeded the judge's text limit; evaluation skipped.",
+ skip_reason="generation_output_too_long",
+ )
+
def test_prompt_contains_input_tools_and_output_sections_in_order(self):
evaluation = {
"id": "eval-123",