Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): name skipped judge attachments and tell the judge not to guess (#127469)

Bernat Torres6d5abdeb0749
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 08a413a44c0d..74ff1526d9b3 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -44,12 +44,13 @@ They reserve space for the complete output, then sample input and tool definitio
If the output and section labels alone exceed that limit, the run is skipped with reason `generation_output_too_long` without calling the judge.
An exception applies when the judge sees images (the `llm-analytics-judge-images` flag is on and the evaluation does not turn attachments off).
Then each image becomes an `[image N]` marker and each PDF a `[file N]` marker instead of its base64 or link, so an output with large inline attachments can fit and the run continues.
-The attachment labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 attachments, plus one summary line for the rest.
+The attachment labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 attachments, plus one summary line that names the rest.
PDFs reach OpenAI, Anthropic and Gemini judges, up to 10 MB each.
WAV and MP3 audio becomes an `[audio N]` marker and reaches Gemini judges and OpenAI audio models, always inline.
MP4, MPEG, QuickTime and WebM video becomes a `[video N]` marker and reaches Gemini judges, up to 15 MB each.
OpenRouter judges get each type that OpenRouter's model catalogue lists as an input of their model. PDFs need the `file` input, so OpenRouter never parses them with its paid OCR engine. Everything except images goes inline, and a model the catalogue doesn't list gets images only.
-Attachments a judge can't read take none of its 20 slots, and one summary line counts them.
+Attachments a judge can't read take none of its 20 slots, and one summary line names them.
+When any attachment is left out, the judge is told it can't see it and not to guess what it contains.
Generation evaluation and LLM tagger user prompts repair UTF-16 surrogate sequences after sampling.
Lone surrogates become replacement characters (`�`), while valid pairs become their Unicode character.
diff --git a/posthog/temporal/ai_observability/judge_attachments.py b/posthog/temporal/ai_observability/judge_attachments.py
index c8aee9e764c4..bb8ac3e8b7d3 100644
--- a/posthog/temporal/ai_observability/judge_attachments.py
+++ b/posthog/temporal/ai_observability/judge_attachments.py
@@ -51,6 +51,16 @@ _NOT_ATTACHED_LABELS: dict[Outcome, str] = {
}
+# Judges asked whether an answer matches a missing attachment otherwise tend to take the answer's word for it.
+_NOT_ATTACHED_NOTE = "You can't see the attachments marked not attached. Don't guess what they contain."
+
+
+def _names(labels: list[str]) -> str:
+ # Capped: these lines sit outside the transcript's text budget, and a transcript can hold thousands of markers.
+ rest = len(labels) - MAX_ATTACHMENTS
+ return ", ".join(labels[:MAX_ATTACHMENTS]) + (f", and {rest} more" if rest > 0 else "")
+
+
_NUMBERED_MARKER_RE = re.compile(r"\[(image|file|audio|video) \d+\]")
@@ -173,14 +183,21 @@ class JudgeAttachments:
else:
parts.extend([{"type": "text", "text": f"{image.label}:"}, attachment.part])
# One line each, not one per attachment: they sit outside the transcript's text budget.
- if over_limit := len(readable) - len(attachments):
- outcomes["over_limit"] = over_limit
- parts.append({"type": "text", "text": f"Not attached, over the judge's attachment limit: {over_limit}."})
- if unreadable := len(images) - len(readable):
- outcomes["unreadable"] = unreadable
+ if over_limit := [image.label for image in readable if image.marker not in attachments]:
+ outcomes["over_limit"] = len(over_limit)
+ parts.append(
+ {"type": "text", "text": f"Not attached, over the judge's attachment limit: {_names(over_limit)}."}
+ )
+ if unreadable := [image.label for image in images if transports[image.mime_type] is None]:
+ outcomes["unreadable"] = len(unreadable)
parts.append(
- {"type": "text", "text": f"Not attached, this judge can't read their file type: {unreadable}."}
+ {
+ "type": "text",
+ "text": f"Not attached, this judge can't read their file type: {_names(unreadable)}.",
+ }
)
+ if over_limit or unreadable or any(attachment.part is None for attachment in attachments.values()):
+ parts.append({"type": "text", "text": _NOT_ATTACHED_NOTE})
parts.append({"type": "text", "text": user_prompt})
logger.info("judge_attachments_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)
return parts
diff --git a/posthog/temporal/ai_observability/test_judge_attachments.py b/posthog/temporal/ai_observability/test_judge_attachments.py
index e1d9052c6f88..dd939d4a453e 100644
--- a/posthog/temporal/ai_observability/test_judge_attachments.py
+++ b/posthog/temporal/ai_observability/test_judge_attachments.py
@@ -4,7 +4,7 @@ import pytest
from unittest.mock import patch
from posthog.storage.object_storage import ObjectStorageError
-from posthog.temporal.ai_observability.judge_attachments import MAX_ATTACHMENTS, JudgeAttachments
+from posthog.temporal.ai_observability.judge_attachments import _NOT_ATTACHED_NOTE, MAX_ATTACHMENTS, JudgeAttachments
from products.ai_observability.backend.llm.attachments import (
MAX_IMAGE_BYTES,
@@ -80,7 +80,7 @@ class TestJudgeAttachmentsContent:
assert {"type": "text", "text": "Image 1:"} in content
assert {"type": "text", "text": "Image 2:"} not in content
- assert {"type": "text", "text": "Not attached, over the judge's attachment limit: 1."} in content
+ assert {"type": "text", "text": "Not attached, over the judge's attachment limit: Image 2."} in content
def test_attaches_the_newest_images_when_over_the_limit(self):
images = JudgeAttachments(team_id=7)
@@ -89,10 +89,11 @@ class TestJudgeAttachmentsContent:
content = images.content(prompt, "openai", "gpt-5.5")
assert content[1] == {"type": "text", "text": "Image 2:"}
- assert content[-4:-1] == [
+ assert content[-5:-1] == [
{"type": "text", "text": f"Image {MAX_ATTACHMENTS + 1}:"},
AttachmentPart(mime_type="image/png", data=base64.b64decode(f"{MAX_ATTACHMENTS:02}" * 4)),
- {"type": "text", "text": "Not attached, over the judge's attachment limit: 1."},
+ {"type": "text", "text": "Not attached, over the judge's attachment limit: Image 1."},
+ {"type": "text", "text": _NOT_ATTACHED_NOTE},
]
@pytest.mark.parametrize(
@@ -142,7 +143,7 @@ class TestJudgeAttachmentsContent:
"gemini",
"gemini-3-flash",
"image%2Fgif",
- "Not attached, this judge can't read their file type: 1.",
+ "Not attached, this judge can't read their file type: Image 1.",
None,
id="format_rejected_by_provider",
),
@@ -174,7 +175,7 @@ class TestJudgeAttachmentsContent:
"anthropic",
"claude-haiku-4-5",
"video%2Fmp4",
- "Not attached, this judge can't read their file type: 1.",
+ "Not attached, this judge can't read their file type: Video 1.",
None,
id="video_on_a_provider_without_video_input",
),
@@ -190,7 +191,7 @@ class TestJudgeAttachmentsContent:
"openai",
"gpt-5.5",
"audio%2Fwav",
- "Not attached, this judge can't read their file type: 1.",
+ "Not attached, this judge can't read their file type: Audio 1.",
None,
id="audio_on_an_openai_model_without_audio_input",
),
@@ -198,7 +199,7 @@ class TestJudgeAttachmentsContent:
"openai",
"gpt-4o-audio-preview",
"image%2Fpng",
- "Not attached, this judge can't read their file type: 1.",
+ "Not attached, this judge can't read their file type: Image 1.",
None,
id="image_on_an_openai_audio_model",
),
@@ -222,7 +223,7 @@ class TestJudgeAttachmentsContent:
"openrouter",
"openai/gpt-5-mini",
"video%2Fmp4",
- "Not attached, this judge can't read their file type: 1.",
+ "Not attached, this judge can't read their file type: Video 1.",
None,
id="video_on_an_openrouter_model_without_video_input",
),
@@ -230,7 +231,7 @@ class TestJudgeAttachmentsContent:
"openrouter",
"example/unlisted",
"application%2Fpdf",
- "Not attached, this judge can't read their file type: 1.",
+ "Not attached, this judge can't read their file type: File 1.",
None,
id="pdf_on_a_model_the_openrouter_catalogue_does_not_list",
),
@@ -263,7 +264,7 @@ class TestJudgeAttachmentsContent:
"text": "The transcript refers to these attachments as [image N], [file N], [audio N] or [video N].",
},
{"type": "text", "text": expected_label},
- *([expected_part] if expected_part else []),
+ *([expected_part] if expected_part else [{"type": "text", "text": _NOT_ATTACHED_NOTE}]),
{"type": "text", "text": prompt},
]
if expected_part:
@@ -392,14 +393,15 @@ class TestJudgeAttachmentsContent:
assert "Image 1:" in labels
assert {f"Video {n}:" for n in range(12 - attached, 12)} <= labels
assert f"Video {11 - attached}:" not in labels
- assert f"Not attached, over the judge's attachment limit: {11 - attached}." in labels
+ dropped = ", ".join(f"Video {n}" for n in range(1, 12 - attached))
+ assert f"Not attached, over the judge's attachment limit: {dropped}." in labels
def test_attachments_the_judge_cannot_read_take_no_slot(self, openrouter_catalogue):
images = JudgeAttachments(team_id=7)
image = images.marker(_inline("aa"))
pdfs = " ".join(
images.marker(MediaBlock(modality="file", url=f"phaiblob://v1/sha256/{i:064x}?mime=application%2Fpdf"))
- for i in range(MAX_ATTACHMENTS)
+ for i in range(MAX_ATTACHMENTS + 1)
)
content = images.content(f"user: {image} {pdfs}", "openrouter", "example/image-only")
@@ -407,7 +409,12 @@ class TestJudgeAttachmentsContent:
assert content[1:4] == [
{"type": "text", "text": "Image 1:"},
AttachmentPart(mime_type="image/png", data=base64.b64decode("aa" * 4)),
- {"type": "text", "text": f"Not attached, this judge can't read their file type: {MAX_ATTACHMENTS}."},
+ {
+ "type": "text",
+ "text": "Not attached, this judge can't read their file type: "
+ + ", ".join(f"File {n}" for n in range(1, MAX_ATTACHMENTS + 1))
+ + ", and 1 more.",
+ },
]
# Each lookup reads the whole OpenRouter catalogue, so it runs once per file type, not per attachment.
assert openrouter_catalogue.call_count == 2