:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): name skipped judge attachments and tell the judge not to guess (#127469)
Bernat Torres6d5abdeb0749
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex 08a413a44c0d..74ff1526d9b3 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -44,12 +44,13 @@ They reserve space for the complete output, then sample input and tool definitioIf the output and section labels alone exceed that limit, the run is skipped with reason `generation_output_too_long` without calling the judge.An exception applies when the judge sees images (the `llm-analytics-judge-images` flag is on and the evaluation does not turn attachments off).Then each image becomes an `[image N]` marker and each PDF a `[file N]` marker instead of its base64 or link, so an output with large inline attachments can fit and the run continues.-The attachment labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 attachments, plus one summary line for the rest.+The attachment labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 attachments, plus one summary line that names the rest.PDFs reach OpenAI, Anthropic and Gemini judges, up to 10 MB each.WAV and MP3 audio becomes an `[audio N]` marker and reaches Gemini judges and OpenAI audio models, always inline.MP4, MPEG, QuickTime and WebM video becomes a `[video N]` marker and reaches Gemini judges, up to 15 MB each.OpenRouter judges get each type that OpenRouter's model catalogue lists as an input of their model. PDFs need the `file` input, so OpenRouter never parses them with its paid OCR engine. Everything except images goes inline, and a model the catalogue doesn't list gets images only.-Attachments a judge can't read take none of its 20 slots, and one summary line counts them.+Attachments a judge can't read take none of its 20 slots, and one summary line names them.+When any attachment is left out, the judge is told it can't see it and not to guess what it contains.Generation evaluation and LLM tagger user prompts repair UTF-16 surrogate sequences after sampling.Lone surrogates become replacement characters (`�`), while valid pairs become their Unicode character.diff --git a/posthog/temporal/ai_observability/judge_attachments.py b/posthog/temporal/ai_observability/judge_attachments.pyindex c8aee9e764c4..bb8ac3e8b7d3 100644--- a/posthog/temporal/ai_observability/judge_attachments.py+++ b/posthog/temporal/ai_observability/judge_attachments.py@@ -51,6 +51,16 @@ _NOT_ATTACHED_LABELS: dict[Outcome, str] = {}+# Judges asked whether an answer matches a missing attachment otherwise tend to take the answer's word for it.+_NOT_ATTACHED_NOTE = "You can't see the attachments marked not attached. Don't guess what they contain."+++def _names(labels: list[str]) -> str:+ # Capped: these lines sit outside the transcript's text budget, and a transcript can hold thousands of markers.+ rest = len(labels) - MAX_ATTACHMENTS+ return ", ".join(labels[:MAX_ATTACHMENTS]) + (f", and {rest} more" if rest > 0 else "")++_NUMBERED_MARKER_RE = re.compile(r"\[(image|file|audio|video) \d+\]")@@ -173,14 +183,21 @@ class JudgeAttachments:else:parts.extend([{"type": "text", "text": f"{image.label}:"}, attachment.part])# One line each, not one per attachment: they sit outside the transcript's text budget.- if over_limit := len(readable) - len(attachments):- outcomes["over_limit"] = over_limit- parts.append({"type": "text", "text": f"Not attached, over the judge's attachment limit: {over_limit}."})- if unreadable := len(images) - len(readable):- outcomes["unreadable"] = unreadable+ if over_limit := [image.label for image in readable if image.marker not in attachments]:+ outcomes["over_limit"] = len(over_limit)+ parts.append(+ {"type": "text", "text": f"Not attached, over the judge's attachment limit: {_names(over_limit)}."}+ )+ if unreadable := [image.label for image in images if transports[image.mime_type] is None]:+ outcomes["unreadable"] = len(unreadable)parts.append(- {"type": "text", "text": f"Not attached, this judge can't read their file type: {unreadable}."}+ {+ "type": "text",+ "text": f"Not attached, this judge can't read their file type: {_names(unreadable)}.",+ })+ if over_limit or unreadable or any(attachment.part is None for attachment in attachments.values()):+ parts.append({"type": "text", "text": _NOT_ATTACHED_NOTE})parts.append({"type": "text", "text": user_prompt})logger.info("judge_attachments_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)return partsdiff --git a/posthog/temporal/ai_observability/test_judge_attachments.py b/posthog/temporal/ai_observability/test_judge_attachments.pyindex e1d9052c6f88..dd939d4a453e 100644--- a/posthog/temporal/ai_observability/test_judge_attachments.py+++ b/posthog/temporal/ai_observability/test_judge_attachments.py@@ -4,7 +4,7 @@ import pytestfrom unittest.mock import patchfrom posthog.storage.object_storage import ObjectStorageError-from posthog.temporal.ai_observability.judge_attachments import MAX_ATTACHMENTS, JudgeAttachments+from posthog.temporal.ai_observability.judge_attachments import _NOT_ATTACHED_NOTE, MAX_ATTACHMENTS, JudgeAttachmentsfrom products.ai_observability.backend.llm.attachments import (MAX_IMAGE_BYTES,@@ -80,7 +80,7 @@ class TestJudgeAttachmentsContent:assert {"type": "text", "text": "Image 1:"} in contentassert {"type": "text", "text": "Image 2:"} not in content- assert {"type": "text", "text": "Not attached, over the judge's attachment limit: 1."} in content+ assert {"type": "text", "text": "Not attached, over the judge's attachment limit: Image 2."} in contentdef test_attaches_the_newest_images_when_over_the_limit(self):images = JudgeAttachments(team_id=7)@@ -89,10 +89,11 @@ class TestJudgeAttachmentsContent:content = images.content(prompt, "openai", "gpt-5.5")assert content[1] == {"type": "text", "text": "Image 2:"}- assert content[-4:-1] == [+ assert content[-5:-1] == [{"type": "text", "text": f"Image {MAX_ATTACHMENTS + 1}:"},AttachmentPart(mime_type="image/png", data=base64.b64decode(f"{MAX_ATTACHMENTS:02}" * 4)),- {"type": "text", "text": "Not attached, over the judge's attachment limit: 1."},+ {"type": "text", "text": "Not attached, over the judge's attachment limit: Image 1."},+ {"type": "text", "text": _NOT_ATTACHED_NOTE},]@pytest.mark.parametrize(@@ -142,7 +143,7 @@ class TestJudgeAttachmentsContent:"gemini","gemini-3-flash","image%2Fgif",- "Not attached, this judge can't read their file type: 1.",+ "Not attached, this judge can't read their file type: Image 1.",None,id="format_rejected_by_provider",),@@ -174,7 +175,7 @@ class TestJudgeAttachmentsContent:"anthropic","claude-haiku-4-5","video%2Fmp4",- "Not attached, this judge can't read their file type: 1.",+ "Not attached, this judge can't read their file type: Video 1.",None,id="video_on_a_provider_without_video_input",),@@ -190,7 +191,7 @@ class TestJudgeAttachmentsContent:"openai","gpt-5.5","audio%2Fwav",- "Not attached, this judge can't read their file type: 1.",+ "Not attached, this judge can't read their file type: Audio 1.",None,id="audio_on_an_openai_model_without_audio_input",),@@ -198,7 +199,7 @@ class TestJudgeAttachmentsContent:"openai","gpt-4o-audio-preview","image%2Fpng",- "Not attached, this judge can't read their file type: 1.",+ "Not attached, this judge can't read their file type: Image 1.",None,id="image_on_an_openai_audio_model",),@@ -222,7 +223,7 @@ class TestJudgeAttachmentsContent:"openrouter","openai/gpt-5-mini","video%2Fmp4",- "Not attached, this judge can't read their file type: 1.",+ "Not attached, this judge can't read their file type: Video 1.",None,id="video_on_an_openrouter_model_without_video_input",),@@ -230,7 +231,7 @@ class TestJudgeAttachmentsContent:"openrouter","example/unlisted","application%2Fpdf",- "Not attached, this judge can't read their file type: 1.",+ "Not attached, this judge can't read their file type: File 1.",None,id="pdf_on_a_model_the_openrouter_catalogue_does_not_list",),@@ -263,7 +264,7 @@ class TestJudgeAttachmentsContent:"text": "The transcript refers to these attachments as [image N], [file N], [audio N] or [video N].",},{"type": "text", "text": expected_label},- *([expected_part] if expected_part else []),+ *([expected_part] if expected_part else [{"type": "text", "text": _NOT_ATTACHED_NOTE}]),{"type": "text", "text": prompt},]if expected_part:@@ -392,14 +393,15 @@ class TestJudgeAttachmentsContent:assert "Image 1:" in labelsassert {f"Video {n}:" for n in range(12 - attached, 12)} <= labelsassert f"Video {11 - attached}:" not in labels- assert f"Not attached, over the judge's attachment limit: {11 - attached}." in labels+ dropped = ", ".join(f"Video {n}" for n in range(1, 12 - attached))+ assert f"Not attached, over the judge's attachment limit: {dropped}." in labelsdef test_attachments_the_judge_cannot_read_take_no_slot(self, openrouter_catalogue):images = JudgeAttachments(team_id=7)image = images.marker(_inline("aa"))pdfs = " ".join(images.marker(MediaBlock(modality="file", url=f"phaiblob://v1/sha256/{i:064x}?mime=application%2Fpdf"))- for i in range(MAX_ATTACHMENTS)+ for i in range(MAX_ATTACHMENTS + 1))content = images.content(f"user: {image} {pdfs}", "openrouter", "example/image-only")@@ -407,7 +409,12 @@ class TestJudgeAttachmentsContent:assert content[1:4] == [{"type": "text", "text": "Image 1:"},AttachmentPart(mime_type="image/png", data=base64.b64decode("aa" * 4)),- {"type": "text", "text": f"Not attached, this judge can't read their file type: {MAX_ATTACHMENTS}."},+ {+ "type": "text",+ "text": "Not attached, this judge can't read their file type: "+ + ", ".join(f"File {n}" for n in range(1, MAX_ATTACHMENTS + 1))+ + ", and 1 more.",+ },]# Each lookup reads the whole OpenRouter catalogue, so it runs once per file type, not per attachment.assert openrouter_catalogue.call_count == 2