:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): let llm judges see pdfs, audio and vercel ai sdk file parts (#126272)
Bernat Torrese333c7285b84
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex 9ce5903e0806..214f93384d47 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -43,8 +43,11 @@ Generation evaluations extract message text without the per-message character cuThey reserve space for the complete output, then sample input and tool definitions into the remaining space within the 150,000-character limit.If the output and section labels alone exceed that limit, the run is skipped with reason `generation_output_too_long` without calling the judge.An exception applies when the judge sees images (the `llm-analytics-judge-images` flag is on and the evaluation does not turn attachments off).-Then each image becomes an `[image N]` marker instead of its base64 or link, so an output with large inline images can fit and the run continues.-The image labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 images, plus one summary line for the rest.+Then each image becomes an `[image N]` marker and each PDF a `[file N]` marker instead of its base64 or link, so an output with large inline attachments can fit and the run continues.+The attachment labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 attachments, plus one summary line for the rest.+PDFs reach only OpenAI, Anthropic and Gemini judges, up to 10 MB each.+WAV and MP3 audio becomes an `[audio N]` marker and reaches Gemini judges and OpenAI audio models, always inline.+Attachments a judge can't read take none of its 20 slots, and one summary line counts them.Generation evaluation and LLM tagger user prompts repair UTF-16 surrogate sequences after sampling.Lone surrogates become replacement characters (`�`), while valid pairs become their Unicode character.diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.pyindex 2b474a65e0b4..9cb5a12ddbf9 100644--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py@@ -41,7 +41,7 @@ from posthog.temporal.ai_observability.evaluation_types import (build_generation_not_found_result,build_skipped_evaluation_result,)-from posthog.temporal.ai_observability.judge_images import ImagePrompt, build_image_prompt+from posthog.temporal.ai_observability.judge_attachments import AttachmentPrompt, build_attachment_promptfrom posthog.temporal.ai_observability.message_utils import extract_text_from_messages, format_tool_definitionsfrom posthog.temporal.ai_observability.metrics import (increment_errors,@@ -56,6 +56,7 @@ from posthog.temporal.common.errors import NonReportableApplicationError, NonRepfrom posthog.temporal.common.utils import close_db_connectionsfrom products.ai_observability.backend.llm import DEFAULT_MODEL_BY_PROVIDER, Client, CompletionRequest, Usage+from products.ai_observability.backend.llm.attachments import AttachmentPartfrom products.ai_observability.backend.llm.decisions import (DecisionClient,DecisionEndpointBlockedError,@@ -80,7 +81,6 @@ from products.ai_observability.backend.llm.errors import (UnsupportedModelError,provider_error_detail,)-from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.llm.providers.openai import OPENAI_DECISIONS_BASE_URLfrom products.ai_observability.backend.llm.providers.openai_decisions import OpenAIDecisionAdapterfrom products.ai_observability.backend.llm.providers.openrouter import OPENROUTER_DECISIONS_BASE_URL@@ -450,11 +450,11 @@ def _build_unparsable_response_skip_result(_IMAGE_REJECTION_STATUSES = frozenset({400, 413, 415, 422})-class JudgeImagesRejectedError(Exception):+class JudgeAttachmentsRejectedError(Exception):"""Reported without the provider's message, which can quote a live presigned link."""-def _image_rejection_status(error: Exception) -> int | None:+def _attachment_rejection_status(error: Exception) -> int | None:"""The status of a provider rejection an image can cause, raw or wrapped by an adapter."""for candidate in (error, error.__cause__):if (@@ -585,14 +585,14 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActisystem_prompt = build_system_prompt(prompt, allows_na, output_type=output_type, output_config=output_config)user_prompt = _build_user_prompt(io.input_raw, io.output_raw, tools_data)- image_prompt = build_image_prompt(+ attachment_prompt = build_attachment_prompt(evaluation,user_prompt,lambda media_marker: _build_user_prompt(io.input_raw, io.output_raw, tools_data, media_marker=media_marker),)- if user_prompt is None and image_prompt is not None:+ if user_prompt is None and attachment_prompt is not None:# Inline base64 can push the plain text over the limit when the marked text still fits.- user_prompt = image_prompt.unnumbered_text()+ user_prompt = attachment_prompt.unnumbered_text()if user_prompt is None:return build_skipped_evaluation_result(output_type=output_type,@@ -607,7 +607,7 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActiuser_prompt=user_prompt,allows_na=allows_na,backfill=inputs.backfill_id is not None,- image_prompt=image_prompt,+ attachment_prompt=attachment_prompt,)@@ -650,7 +650,7 @@ def call_llm_judge(user_prompt: str,allows_na: bool,backfill: bool = False,- image_prompt: ImagePrompt | None = None,+ attachment_prompt: AttachmentPrompt | None = None,) -> EvaluationActivityResult:"""Resolve the judge model/key for `evaluation` and run a single judge completion.@@ -859,8 +859,12 @@ def call_llm_judge(),)else:- content = image_prompt.images.content(image_prompt.text, provider, model) if image_prompt else user_prompt- sent_images = isinstance(content, list) and any(isinstance(part, ImagePart) for part in content)+ content = (+ attachment_prompt.attachments.content(attachment_prompt.text, provider, model)+ if attachment_prompt+ else user_prompt+ )+ sent_attachments = isinstance(content, list) and any(isinstance(part, AttachmentPart) for part in content)request = CompletionRequest(model=model,system=system_prompt,@@ -873,13 +877,13 @@ def call_llm_judge(try:response = client.complete(request)except Exception as e:- status = _image_rejection_status(e) if sent_images else None+ status = _attachment_rejection_status(e) if sent_attachments else Noneif status is None:raise# Handle the rejection outside the except: the provider's error can quote a presigned link, and# an active exception reaches the logs through capture_exception and chains onto a retry failure.if status is not None:- # A model without image input, or an image link the provider could not download.+ # A model without image or PDF input, or a link the provider could not download.# Grade the plain transcript rather than skip the run.context = {"evaluation_id": evaluation["id"],@@ -887,11 +891,14 @@ def call_llm_judge("provider": provider,"model": model,"status": status,+ "sent_pdfs": any(isinstance(part, AttachmentPart) and part.is_pdf for part in content),}- logger.warning("judge_images_rejected", **context)+ logger.warning("judge_attachments_rejected", **context)if not is_byok:- # On a PostHog key this can mean broken presigned links, so keep it visible.- capture_exception(JudgeImagesRejectedError(f"Provider rejected judge images ({status})"), context)+ # On a PostHog key this can mean broken presigned links or a refused PDF, so keep it visible.+ capture_exception(+ JudgeAttachmentsRejectedError(f"Provider rejected judge images ({status})"), context+ )response = client.complete(replace(request, messages=[{"role": "user", "content": user_prompt}]))except (DecisionEndpointBlockedError, ProviderConfigurationError) as e:increment_user_errors("endpoint_blocked", provider=provider)diff --git a/posthog/temporal/ai_observability/judge_images.py b/posthog/temporal/ai_observability/judge_attachments.pysimilarity index 52%rename from posthog/temporal/ai_observability/judge_images.pyrename to posthog/temporal/ai_observability/judge_attachments.pyindex 121420fed86b..ec6b971083e9 100644--- a/posthog/temporal/ai_observability/judge_images.py+++ b/posthog/temporal/ai_observability/judge_attachments.py@@ -1,4 +1,4 @@-"""Images an LLM judge sees next to the transcript that refers to them as `[image N]`."""+"""Images, PDFs and audio an LLM judge gets next to the transcript, which refers to them by markers such as `[image N]`."""import reimport base64@@ -15,15 +15,23 @@ from posthog.models.team import Teamfrom posthog.storage.object_storage import ObjectStorageErrorfrom products.ai_observability.backend.facade.blobs import BlobPointer-from products.ai_observability.backend.llm.images import IMAGE_MIME_TYPES, MAX_IMAGE_BYTES, ImagePart, image_transport+from products.ai_observability.backend.llm.attachments import (+ AUDIO_FORMATS,+ IMAGE_MIME_TYPES,+ MAX_IMAGE_BYTES,+ MAX_PDF_BYTES,+ PDF_MIME_TYPE,+ AttachmentPart,+ attachment_transport,+)from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, MediaMarkerlogger = structlog.get_logger(__name__)-JUDGE_IMAGES_FLAG = "llm-analytics-judge-images"+JUDGE_ATTACHMENTS_FLAG = "llm-analytics-judge-images"-# Each image costs tokens on the customer's key, and above 20 images Anthropic tightens its per-image dimension limit.-MAX_IMAGES = 20+# Each attachment costs tokens on the customer's key, and above 20 images Anthropic tightens its per-image dimension limit.+MAX_ATTACHMENTS = 20# Bytes sent inline in one judge request. Links don't count: the provider downloads them.MAX_INLINE_BYTES = 15_000_000# Covers the judge call and its retries. Sign right before the call: a link signed with temporary@@ -38,44 +46,54 @@ _NOT_ATTACHED_LABELS: dict[Outcome, str] = {}-_NUMBERED_MARKER_RE = re.compile(r"\[image \d+\]")+_NUMBERED_MARKER_RE = re.compile(r"\[(image|file|audio) \d+\]")@frozen-class ImagePrompt:- """The judge transcript with `[image N]` markers, and the images those markers refer to."""+class AttachmentPrompt:+ """The judge transcript with attachment markers, and the attachments they refer to."""text: str- images: "JudgeImages"+ attachments: "JudgeAttachments"def unnumbered_text(self) -> str:- """The transcript for a judge call that sends no images, so no marker points at a missing image."""- return _NUMBERED_MARKER_RE.sub("[image]", self.text)+ """The transcript for a judge call that sends no attachments, so no marker points at a missing one."""+ return _NUMBERED_MARKER_RE.sub(r"[\1]", self.text)@frozenclass _Attachment:outcome: Outcome- part: ImagePart | None = None+ part: AttachmentPart | None = Noneinline_bytes: int = 0@frozen-class _Image:+class _Media:+ kind: Literal["image", "file", "audio"]number: intmime_type: str# An offloaded image has a pointer, a small one stays inline as base64.pointer: BlobPointer | Nonebase64_data: str | None+ @property+ def marker(self) -> str:+ return f"[{self.kind} {self.number}]"-class JudgeImages:- """Numbers the images found while formatting a transcript, then attaches them to the judge request."""+ @property+ def label(self) -> str:+ return f"{self.kind.capitalize()} {self.number}"+++class JudgeAttachments:+ """Numbers the attachments found while formatting a transcript, then attaches them to the judge request."""def __init__(self, team_id: int) -> None:self._team_id = team_id- self._images: list[_Image] = []- self._numbers: dict[str, int] = {}+ self._media: list[_Media] = []+ self._media_by_key: dict[str, _Media] = {}+ self._counts: Counter[str] = Counter()# After one storage failure the rest would fail too, each after its own timeout and retry.self._storage_failed = False@@ -84,65 +102,81 @@ class JudgeImages:mime_type = _normalize_mime_type(media.mime_type or (pointer.mime_type if pointer else None))# Links from the customer's app keep a plain marker: we never fetch them.key = pointer.sha256 if pointer else media.base64_data- if media.modality != "image" or mime_type not in IMAGE_MIME_TYPES or key is None:+ kind = _kind(media.modality, mime_type)+ if kind is None or mime_type is None or key is None:return media.marker()- if (number := self._numbers.get(key)) is None:- number = self._numbers[key] = len(self._images) + 1- self._images.append(- _Image(number=number, mime_type=mime_type, pointer=pointer, base64_data=media.base64_data)+ if (image := self._media_by_key.get(key)) is None:+ self._counts[kind] += 1+ image = self._media_by_key[key] = _Media(+ kind=kind,+ number=self._counts[kind],+ mime_type=mime_type,+ pointer=pointer,+ base64_data=media.base64_data,)- return f"[image {number}]"+ self._media.append(image)+ return image.markerdef content(self, user_prompt: str, provider: str, model: str) -> str | list[Any]:- """The user message content: each referenced image under its label, then the transcript."""- # Line sampling can drop a marker from a long transcript, and then its image only costs tokens.- images = [image for image in self._images if f"[image {image.number}]" in user_prompt]+ """The user message content: each referenced attachment under its label, then the transcript."""+ # Line sampling can drop a marker from a long transcript, and then its attachment only costs tokens.+ images = [image for image in self._media if image.marker in user_prompt]if not images:return user_prompt+ # Attachments this judge can't read take no slot, so they can't push out ones it can.+ readable = [image for image in images if attachment_transport(provider, model, image.mime_type) is not None]# Keep the images referenced last: the newest input and the graded output, even when it repeats an old image.- ranked = sorted(images, key=lambda image: user_prompt.rfind(f"[image {image.number}]"))- attachments: dict[int, _Attachment] = {}+ ranked = sorted(readable, key=lambda image: user_prompt.rfind(image.marker))+ attachments: dict[str, _Attachment] = {}inline_bytes = 0- for image in reversed(ranked[-MAX_IMAGES:]):+ for image in reversed(ranked[-MAX_ATTACHMENTS:]):attachment = self._attach(image, provider, model, MAX_INLINE_BYTES - inline_bytes)inline_bytes += attachment.inline_bytes- attachments[image.number] = attachment+ attachments[image.marker] = attachment- parts: list[Any] = [{"type": "text", "text": "The transcript refers to these images as [image N]."}]+ parts: list[Any] = [+ {"type": "text", "text": "The transcript refers to these attachments as [image N], [file N] or [audio N]."}+ ]outcomes: Counter[str] = Counter()for image in images:- if image.number not in attachments:+ if image.marker not in attachments:continue- attachment = attachments[image.number]- outcomes[attachment.outcome] += 1+ attachment = attachments[image.marker]+ outcomes[f"{image.kind}_{attachment.outcome}"] += 1if attachment.part is None:label = _NOT_ATTACHED_LABELS[attachment.outcome]- parts.append({"type": "text", "text": f"Image {image.number}: {label}."})+ parts.append({"type": "text", "text": f"{image.label}: {label}."})else:- parts.extend([{"type": "text", "text": f"Image {image.number}:"}, attachment.part])- # One line, not one per image: it sits outside the transcript's text budget.- if over_limit := len(images) - len(attachments):+ parts.extend([{"type": "text", "text": f"{image.label}:"}, attachment.part])+ # One line each, not one per attachment: they sit outside the transcript's text budget.+ if over_limit := len(readable) - len(attachments):outcomes["over_limit"] = over_limit- parts.append({"type": "text", "text": f"Images not attached, over the judge's image limit: {over_limit}."})+ parts.append({"type": "text", "text": f"Not attached, over the judge's attachment limit: {over_limit}."})+ if unreadable := len(images) - len(readable):+ outcomes["unreadable"] = unreadable+ parts.append(+ {"type": "text", "text": f"Not attached, this judge can't read their file type: {unreadable}."}+ )parts.append({"type": "text", "text": user_prompt})- logger.info("judge_images_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)+ logger.info("judge_attachments_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)return parts- def _attach(self, image: _Image, provider: str, model: str, inline_budget: int) -> _Attachment:- transport = image_transport(provider, model, image.mime_type)+ def _attach(self, image: _Media, provider: str, model: str, inline_budget: int) -> _Attachment:+ transport = attachment_transport(provider, model, image.mime_type)if transport is None:return _Attachment(outcome="unavailable")- provider_limit = MAX_IMAGE_BYTES.get(provider)+ provider_limit = MAX_PDF_BYTES if image.kind == "file" else MAX_IMAGE_BYTES.get(provider)if image.pointer is None:- if len(image.base64_data or "") * 3 // 4 > min(inline_budget, provider_limit or inline_budget):+ encoded = image.base64_data or ""+ if len(encoded) * 3 // 4 - encoded[-2:].count("=") > min(inline_budget, provider_limit or inline_budget):return _Attachment(outcome="too_large")try:data = base64.b64decode(image.base64_data or "", validate=True)except ValueError: # binascii.Error, or a non-ASCII stringreturn _Attachment(outcome="unavailable")return _Attachment(- outcome="inline", part=ImagePart(mime_type=image.mime_type, data=data), inline_bytes=len(data)+ outcome="inline", part=AttachmentPart(mime_type=image.mime_type, data=data), inline_bytes=len(data))if self._storage_failed:return _Attachment(outcome="unavailable")@@ -156,7 +190,7 @@ class JudgeImages:return _Attachment(outcome="unavailable")def _attach_blob(- self, image: _Image, pointer: BlobPointer, transport: Literal["url", "inline"], max_bytes: int | None+ self, image: _Media, pointer: BlobPointer, transport: Literal["url", "inline"], max_bytes: int | None) -> _Attachment:stored_size = 0if max_bytes is not None:@@ -171,20 +205,38 @@ class JudgeImages:url = pointer.presigned_url(self._team_id, PRESIGNED_URL_SECONDS)if url is None:return _Attachment(outcome="unavailable")- return _Attachment(outcome="link", part=ImagePart(mime_type=image.mime_type, url=url))+ return _Attachment(outcome="link", part=AttachmentPart(mime_type=image.mime_type, url=url))# ponytail: sequential reads, a thread pool if traces with many images get slowdata = pointer.read(self._team_id)if data is None:return _Attachment(outcome="unavailable")- part = ImagePart(mime_type=image.mime_type, data=data)+ part = AttachmentPart(mime_type=image.mime_type, data=data)return _Attachment(outcome="inline", part=part, inline_bytes=stored_size)+def _kind(modality: str, mime_type: str | None) -> Literal["image", "file", "audio"] | None:+ if modality == "image" and mime_type in IMAGE_MIME_TYPES:+ return "image"+ if modality == "audio" and mime_type in AUDIO_FORMATS:+ return "audio"+ if mime_type == PDF_MIME_TYPE:+ return "file"+ return None+++_MIME_ALIASES = {+ "image/jpg": "image/jpeg",+ "audio/mp3": "audio/mpeg",+ "audio/x-wav": "audio/wav",+ "audio/wave": "audio/wav",+}++def _normalize_mime_type(mime_type: str | None) -> str | None:if mime_type is None:return Nonenormalized = mime_type.strip().lower()- return "image/jpeg" if normalized == "image/jpg" else normalized+ return _MIME_ALIASES.get(normalized, normalized)@lru_cache(maxsize=4096)@@ -193,7 +245,7 @@ def _project_uuid(team_id: int) -> str | None:return str(project_uuid) if project_uuid else None-def judge_sees_images(evaluation: dict[str, Any]) -> bool:+def judge_sees_attachments(evaluation: dict[str, Any]) -> bool:if evaluation["evaluation_config"].get("include_media") is False:return False# Local evaluation keeps the check off the network. An inconclusive result means off.@@ -204,7 +256,7 @@ def judge_sees_images(evaluation: dict[str, Any]) -> bool:return Falsereturn bool(posthoganalytics.feature_enabled(- JUDGE_IMAGES_FLAG,+ JUDGE_ATTACHMENTS_FLAG,project_uuid,groups={"project": project_uuid},group_properties={"project": {"id": str(evaluation["team_id"]), "uuid": project_uuid}},@@ -216,15 +268,19 @@ def judge_sees_images(evaluation: dict[str, Any]) -> bool:return False-def build_image_prompt(+def build_attachment_prompt(evaluation: dict[str, Any], plain_prompt: str | None, render: Callable[[MediaMarker], str | None]-) -> ImagePrompt | None:- """The marked prompt and its images, or None when this judge run gets the plain prompt only.+) -> AttachmentPrompt | None:+ """The marked prompt and its attachments, or None when this judge run gets the plain prompt only.`render` builds the same transcript as the plain prompt, with media blocks rendered by the marker it gets."""- if not judge_sees_images(evaluation):+ if not judge_sees_attachments(evaluation):return None- images = JudgeImages(evaluation["team_id"])- marked = render(images.marker)- return ImagePrompt(text=marked, images=images) if marked is not None and marked != plain_prompt else None+ attachments = JudgeAttachments(evaluation["team_id"])+ marked = render(attachments.marker)+ return (+ AttachmentPrompt(text=marked, attachments=attachments)+ if marked is not None and marked != plain_prompt+ else None+ )diff --git a/posthog/temporal/ai_observability/run_session_evaluation.py b/posthog/temporal/ai_observability/run_session_evaluation.pyindex 8b683d743139..57c2600e2b87 100644--- a/posthog/temporal/ai_observability/run_session_evaluation.py+++ b/posthog/temporal/ai_observability/run_session_evaluation.py@@ -42,7 +42,7 @@ from posthog.temporal.ai_observability.evaluation_payload import (should_skip_for_payload,)from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult, build_skipped_evaluation_result-from posthog.temporal.ai_observability.judge_images import build_image_prompt+from posthog.temporal.ai_observability.judge_attachments import build_attachment_promptfrom posthog.temporal.ai_observability.message_utils import extract_text_from_messagesfrom posthog.temporal.ai_observability.run_trace_evaluation import TRACE_EVENTS_LOOKBACKfrom posthog.temporal.common.utils import close_db_connections@@ -493,12 +493,12 @@ def execute_session_llm_judge_activity(inputs: ExecuteSessionEvaluationInputs) -traces = outcome.tracestranscript = format_session_for_judge(traces)- image_prompt = build_image_prompt(+ attachment_prompt = build_attachment_prompt(evaluation, transcript, lambda media_marker: format_session_for_judge(traces, media_marker))- if transcript is None and image_prompt is not None:+ if transcript is None and attachment_prompt is not None:# Inline base64 can push the plain transcript over the limit when the marked one still fits.- transcript = image_prompt.unnumbered_text()+ transcript = attachment_prompt.unnumbered_text()if transcript is None:return build_session_skip_result(allows_na, "session_too_long_to_judge", output_type=output_type)@@ -510,7 +510,7 @@ def execute_session_llm_judge_activity(inputs: ExecuteSessionEvaluationInputs) -user_prompt=transcript,allows_na=allows_na,backfill=inputs.backfill_id is not None,- image_prompt=image_prompt,+ attachment_prompt=attachment_prompt,)diff --git a/posthog/temporal/ai_observability/run_trace_evaluation.py b/posthog/temporal/ai_observability/run_trace_evaluation.pyindex 3e3b71439dee..b204541ad9d0 100644--- a/posthog/temporal/ai_observability/run_trace_evaluation.py+++ b/posthog/temporal/ai_observability/run_trace_evaluation.py@@ -63,7 +63,7 @@ from posthog.temporal.ai_observability.evaluation_workflow_activities import (emit_internal_telemetry_activity,fetch_evaluation_activity,)-from posthog.temporal.ai_observability.judge_images import build_image_prompt+from posthog.temporal.ai_observability.judge_attachments import build_attachment_promptfrom posthog.temporal.ai_observability.message_utils import extract_text_from_messagesfrom posthog.temporal.ai_observability.metrics import increment_emit_event_outcome, increment_errorsfrom posthog.temporal.ai_observability.run_evaluation import (@@ -708,7 +708,7 @@ def execute_trace_llm_judge_activity(inputs: ExecuteTraceEvaluationInputs) -> Evuser_prompt=user_prompt,allows_na=allows_na,backfill=inputs.backfill_id is not None,- image_prompt=build_image_prompt(+ attachment_prompt=build_attachment_prompt(evaluation, user_prompt, lambda media_marker: format_trace_for_judge(trace, media_marker)),)diff --git a/posthog/temporal/ai_observability/test_judge_attachments.py b/posthog/temporal/ai_observability/test_judge_attachments.pynew file mode 100644index 000000000000..d6d9b6d7d151--- /dev/null+++ b/posthog/temporal/ai_observability/test_judge_attachments.py@@ -0,0 +1,319 @@+import base64++import pytest+from unittest.mock import patch++from posthog.storage.object_storage import ObjectStorageError+from posthog.temporal.ai_observability.judge_attachments import MAX_ATTACHMENTS, JudgeAttachments++from products.ai_observability.backend.llm.attachments import MAX_IMAGE_BYTES, MAX_PDF_BYTES, AttachmentPart+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, parse_media_block++OBJECT_STORAGE = "products.ai_observability.backend.ai_blobs.object_storage"+++def _pointer(digit: str, mime: str = "image%2Fjpeg") -> MediaBlock:+ modality = "audio" if mime.startswith("audio") else "image"+ return MediaBlock(modality=modality, url=f"phaiblob://v1/sha256/{digit * 64}?mime={mime}&size=63357")+++def _inline(digit: str, length: int = 4) -> MediaBlock:+ return MediaBlock(modality="image", mime_type="image/png", base64_data=digit * length)+++class TestJudgeAttachmentsMarkers:+ def test_numbers_each_kind_and_reuses_the_number_for_a_repeat(self):+ images = JudgeAttachments(team_id=7)+ inline = MediaBlock(modality="image", mime_type="image/png", base64_data="aGk=")++ pdf = _pointer("b", mime="application%2Fpdf")++ markers = [images.marker(media) for media in (_pointer("a"), pdf, inline, _pointer("a"))]++ assert markers == ["[image 1]", "[file 1]", "[image 2]", "[image 1]"]++ @pytest.mark.parametrize(+ ("media", "marker"),+ [+ pytest.param(MediaBlock(modality="audio", base64_data="aGk="), "[audio]", id="not_an_image"),+ pytest.param(_pointer("a", mime="image%2Fsvg%2Bxml"), "[image]", id="unsupported_format"),+ pytest.param(+ MediaBlock(modality="image", mime_type="image/png", url="https://example.com/a.png"),+ "[image: https://example.com/a.png]",+ id="customer_link",+ ),+ pytest.param(+ parse_media_block({"file_data": {"mime_type": "image/png", "file_uri": "gs://bucket/a.png"}}),+ "[image]",+ id="gemini_cloud_storage_uri",+ ),+ ],+ )+ def test_images_it_cannot_attach_keep_an_unnumbered_marker(self, media: MediaBlock, marker: str):+ assert JudgeAttachments(team_id=7).marker(media) == marker+++class TestJudgeAttachmentsContent:+ def test_an_image_referenced_again_at_the_end_stays_attached(self):+ images = JudgeAttachments(team_id=7)+ markers = [images.marker(_inline(f"{i:02}")) for i in range(MAX_ATTACHMENTS + 1)]+ # The output repeats the first input image after every newer one.+ content = images.content(" ".join([*markers, markers[0]]), "openai", "gpt-5.5")++ assert {"type": "text", "text": "Image 1:"} in content+ assert {"type": "text", "text": "Image 2:"} not in content+ assert {"type": "text", "text": "Not attached, over the judge's attachment limit: 1."} in content++ def test_attaches_the_newest_images_when_over_the_limit(self):+ images = JudgeAttachments(team_id=7)+ prompt = " ".join(images.marker(_inline(f"{i:02}")) for i in range(MAX_ATTACHMENTS + 1))++ content = images.content(prompt, "openai", "gpt-5.5")++ assert content[1] == {"type": "text", "text": "Image 2:"}+ assert content[-4:-1] == [+ {"type": "text", "text": f"Image {MAX_ATTACHMENTS + 1}:"},+ AttachmentPart(mime_type="image/png", data=base64.b64decode(f"{MAX_ATTACHMENTS:02}" * 4)),+ {"type": "text", "text": "Not attached, over the judge's attachment limit: 1."},+ ]++ @pytest.mark.parametrize(+ ("provider", "model", "mime", "expected_label", "expected_part"),+ [+ pytest.param(+ "openai",+ "gpt-5.5",+ "image%2FJPG",+ "Image 1:",+ AttachmentPart(mime_type="image/jpeg", url="https://signed"),+ id="link",+ ),+ pytest.param(+ "together_ai",+ "any",+ "image%2FJPG",+ "Image 1:",+ AttachmentPart(mime_type="image/jpeg", data=b"hi"),+ id="inline",+ ),+ pytest.param(+ "gemini",+ "gemini-2.0-flash",+ "image%2FJPG",+ "Image 1:",+ AttachmentPart(mime_type="image/jpeg", data=b"hi"),+ id="gemini_2_0",+ ),+ pytest.param(+ "anthropic",+ "claude-haiku-4-5",+ "application%2Fpdf",+ "File 1:",+ AttachmentPart(mime_type="application/pdf", url="https://signed"),+ id="pdf_link",+ ),+ pytest.param(+ "openai",+ "gpt-5.5",+ "application%2Fpdf",+ "File 1:",+ AttachmentPart(mime_type="application/pdf", data=b"hi"),+ id="pdf_inline_on_openai",+ ),+ pytest.param(+ "gemini",+ "gemini-3-flash",+ "image%2Fgif",+ "Not attached, this judge can't read their file type: 1.",+ None,+ id="format_rejected_by_provider",+ ),+ pytest.param(+ "gemini",+ "gemini-2.5-flash",+ "audio%2Fwav",+ "Audio 1:",+ AttachmentPart(mime_type="audio/wav", data=b"hi"),+ id="audio_inline_on_gemini",+ ),+ pytest.param(+ "openai",+ "gpt-4o-audio-preview",+ "audio%2Fmp3",+ "Audio 1:",+ AttachmentPart(mime_type="audio/mpeg", data=b"hi"),+ id="audio_on_an_openai_audio_model",+ ),+ pytest.param(+ "openai",+ "gpt-5.5",+ "audio%2Fwav",+ "Not attached, this judge can't read their file type: 1.",+ None,+ id="audio_on_an_openai_model_without_audio_input",+ ),+ pytest.param(+ "openai",+ "gpt-4o-audio-preview",+ "image%2Fpng",+ "Not attached, this judge can't read their file type: 1.",+ None,+ id="image_on_an_openai_audio_model",+ ),+ pytest.param(+ "openrouter",+ "any",+ "application%2Fpdf",+ "Not attached, this judge can't read their file type: 1.",+ None,+ id="pdf_unsupported_provider",+ ),+ ],+ )+ def test_attaches_each_attachment_under_its_label(+ self, provider: str, model: str, mime: str, expected_label: str, expected_part: AttachmentPart | None+ ):+ images = JudgeAttachments(team_id=7)+ prompt = f"Input: user: {images.marker(_pointer('a', mime=mime))} What does this chart show?"++ with (+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed") as presign,+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 2}),+ patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"hi") as read,+ ):+ content = images.content(prompt, provider, model)++ assert content == [+ {"type": "text", "text": "The transcript refers to these attachments as [image N], [file N] or [audio N]."},+ {"type": "text", "text": expected_label},+ *([expected_part] if expected_part else []),+ {"type": "text", "text": prompt},+ ]+ if expected_part:+ storage_call = presign if expected_part.url else read+ assert storage_call.call_args.args[0] == f"aio/7/sha256/{'a' * 64}"++ @pytest.mark.parametrize(+ ("provider", "model", "media", "storage_error"),+ [+ pytest.param("openai", "gpt-5.5", _pointer("a"), None, id="blob_missing_for_a_link"),+ pytest.param("together_ai", "any", _pointer("a"), None, id="blob_missing_for_inline"),+ pytest.param(+ "together_ai", "any", _pointer("a"), ObjectStorageError("head_object failed"), id="storage_error"+ ),+ pytest.param(+ "gemini",+ "gemini-3-flash",+ MediaBlock(modality="image", mime_type="image/png", base64_data="aGk"),+ None,+ id="invalid_inline_base64",+ ),+ pytest.param(+ "gemini",+ "gemini-3-flash",+ MediaBlock(modality="image", mime_type="image/png", base64_data="aGk\u00e9"),+ None,+ id="non_ascii_inline_base64",+ ),+ ],+ )+ def test_reports_an_image_it_could_not_attach(+ self, provider: str, model: str, media: MediaBlock, storage_error: Exception | None+ ):+ images = JudgeAttachments(team_id=7)+ prompt = f"user: {images.marker(media)}"++ with (+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value=None),+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value=None, side_effect=storage_error),+ ):+ content = images.content(prompt, provider, model)++ assert content[1] == {"type": "text", "text": "Image 1: could not be attached."}++ def test_stops_calling_storage_after_a_storage_error(self):+ images = JudgeAttachments(team_id=7)+ prompt = f"user: {images.marker(_pointer('a'))} {images.marker(_pointer('b'))}"++ with patch(f"{OBJECT_STORAGE}.head_object_strict", side_effect=ObjectStorageError("timed out")) as head:+ content = images.content(prompt, "together_ai", "any")++ assert head.call_count == 1+ assert {"type": "text", "text": "Image 1: could not be attached."} in content+ assert {"type": "text", "text": "Image 2: could not be attached."} in content++ @pytest.mark.parametrize(+ ("provider", "media", "too_large", "reads"),+ [+ pytest.param("openai", _pointer, set(), 0, id="links_have_no_byte_cap"),+ pytest.param("anthropic", _pointer, {1, 2}, 0, id="provider_limit_on_links"),+ # The pointers claim 63 KB, so this also checks that the budget uses the stored size.+ pytest.param("together_ai", _pointer, {1}, 1, id="stored_size_for_inline_providers"),+ pytest.param(+ "openai", lambda digit: _inline(digit, length=13_400_000), {1}, 0, id="inline_images_on_link_providers"+ ),+ ],+ )+ def test_caps_image_bytes(self, provider: str, media, too_large: set[int], reads: int):+ images = JudgeAttachments(team_id=7)+ prompt = f"user: {images.marker(media('a'))} {images.marker(media('b'))}"++ with (+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed"),+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 10_000_000}),+ patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"x" * 10_000_000) as read,+ ):+ content = images.content(prompt, provider, "any")++ assert {+ number+ for number in (1, 2)+ if {"type": "text", "text": f"Image {number}: not attached, too large."} in content+ } == too_large+ # An image over the budget is never downloaded.+ assert read.call_count == reads++ def test_caps_pdf_bytes_on_links(self):+ images = JudgeAttachments(team_id=7)+ prompt = f"user: {images.marker(_pointer('a', mime='application%2Fpdf'))}"++ with (+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed") as presign,+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": MAX_PDF_BYTES + 1}),+ ):+ # Gemini has no image byte limit, so only the PDF cap can reject this one.+ content = images.content(prompt, "gemini", "gemini-3-flash")++ assert {"type": "text", "text": "File 1: not attached, too large."} in content+ presign.assert_not_called()++ def test_attachments_the_judge_cannot_read_take_no_slot(self):+ images = JudgeAttachments(team_id=7)+ image = images.marker(_inline("aa"))+ pdfs = " ".join(+ images.marker(MediaBlock(modality="file", url=f"phaiblob://v1/sha256/{i:064x}?mime=application%2Fpdf"))+ for i in range(MAX_ATTACHMENTS)+ )++ content = images.content(f"user: {image} {pdfs}", "openrouter", "openai/gpt-5-mini")++ assert content[1:4] == [+ {"type": "text", "text": "Image 1:"},+ AttachmentPart(mime_type="image/png", data=base64.b64decode("aa" * 4)),+ {"type": "text", "text": f"Not attached, this judge can't read their file type: {MAX_ATTACHMENTS}."},+ ]++ def test_an_inline_image_exactly_at_the_provider_limit_is_attached(self):+ images = JudgeAttachments(team_id=7)+ encoded = base64.b64encode(b"x" * MAX_IMAGE_BYTES["anthropic"]).decode()+ prompt = f"user: {images.marker(MediaBlock(modality='image', mime_type='image/png', base64_data=encoded))}"++ content = images.content(prompt, "anthropic", "claude-haiku-4-5")++ assert content[1] == {"type": "text", "text": "Image 1:"}++ def test_skips_images_whose_marker_was_sampled_out_of_the_prompt(self):+ images = JudgeAttachments(team_id=7)+ images.marker(_pointer("a"))++ assert images.content("user: hello", "openai", "gpt-5.5") == "user: hello"diff --git a/posthog/temporal/ai_observability/test_judge_images.py b/posthog/temporal/ai_observability/test_judge_images.pydeleted file mode 100644index 3d755b001e4f..000000000000--- a/posthog/temporal/ai_observability/test_judge_images.py+++ /dev/null@@ -1,192 +0,0 @@-import base64--import pytest-from unittest.mock import patch--from posthog.storage.object_storage import ObjectStorageError-from posthog.temporal.ai_observability.judge_images import MAX_IMAGES, JudgeImages--from products.ai_observability.backend.llm.images import ImagePart-from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, parse_media_block--OBJECT_STORAGE = "products.ai_observability.backend.ai_blobs.object_storage"---def _pointer(digit: str, mime: str = "image%2Fjpeg") -> MediaBlock:- return MediaBlock(modality="image", url=f"phaiblob://v1/sha256/{digit * 64}?mime={mime}&size=63357")---def _inline(digit: str, length: int = 4) -> MediaBlock:- return MediaBlock(modality="image", mime_type="image/png", base64_data=digit * length)---class TestJudgeImagesMarkers:- def test_numbers_images_and_reuses_the_number_for_a_repeat(self):- images = JudgeImages(team_id=7)- inline = MediaBlock(modality="image", mime_type="image/png", base64_data="aGk=")-- markers = [images.marker(media) for media in (_pointer("a"), inline, _pointer("a"))]-- assert markers == ["[image 1]", "[image 2]", "[image 1]"]-- @pytest.mark.parametrize(- ("media", "marker"),- [- pytest.param(MediaBlock(modality="audio", base64_data="aGk="), "[audio]", id="not_an_image"),- pytest.param(_pointer("a", mime="image%2Fsvg%2Bxml"), "[image]", id="unsupported_format"),- pytest.param(- MediaBlock(modality="image", mime_type="image/png", url="https://example.com/a.png"),- "[image: https://example.com/a.png]",- id="customer_link",- ),- pytest.param(- parse_media_block({"file_data": {"mime_type": "image/png", "file_uri": "gs://bucket/a.png"}}),- "[image]",- id="gemini_cloud_storage_uri",- ),- ],- )- def test_images_it_cannot_attach_keep_an_unnumbered_marker(self, media: MediaBlock, marker: str):- assert JudgeImages(team_id=7).marker(media) == marker---class TestJudgeImagesContent:- def test_an_image_referenced_again_at_the_end_stays_attached(self):- images = JudgeImages(team_id=7)- markers = [images.marker(_inline(f"{i:02}")) for i in range(MAX_IMAGES + 1)]- # The output repeats the first input image after every newer one.- content = images.content(" ".join([*markers, markers[0]]), "openai", "gpt-5.5")-- assert {"type": "text", "text": "Image 1:"} in content- assert {"type": "text", "text": "Image 2:"} not in content- assert {"type": "text", "text": "Images not attached, over the judge's image limit: 1."} in content-- def test_attaches_the_newest_images_when_over_the_limit(self):- images = JudgeImages(team_id=7)- prompt = " ".join(images.marker(_inline(f"{i:02}")) for i in range(MAX_IMAGES + 1))-- content = images.content(prompt, "openai", "gpt-5.5")-- assert content[1] == {"type": "text", "text": "Image 2:"}- assert content[-4:-1] == [- {"type": "text", "text": f"Image {MAX_IMAGES + 1}:"},- ImagePart(mime_type="image/png", data=base64.b64decode(f"{MAX_IMAGES:02}" * 4)),- {"type": "text", "text": "Images not attached, over the judge's image limit: 1."},- ]-- @pytest.mark.parametrize(- ("provider", "model", "expected_image"),- [- pytest.param("openai", "gpt-5.5", ImagePart(mime_type="image/jpeg", url="https://signed"), id="link"),- pytest.param("together_ai", "any", ImagePart(mime_type="image/jpeg", data=b"hi"), id="inline"),- pytest.param("gemini", "gemini-2.0-flash", ImagePart(mime_type="image/jpeg", data=b"hi"), id="gemini_2_0"),- ],- )- def test_attaches_each_image_under_its_label(self, provider: str, model: str, expected_image: ImagePart):- images = JudgeImages(team_id=7)- prompt = f"Input: user: {images.marker(_pointer('a', mime='image%2FJPG'))} What does this chart show?"-- with (- patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed") as presign,- patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 2}),- patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"hi") as read,- ):- content = images.content(prompt, provider, model)-- assert content == [- {"type": "text", "text": "The transcript refers to these images as [image N]."},- {"type": "text", "text": "Image 1:"},- expected_image,- {"type": "text", "text": prompt},- ]- storage_call = presign if expected_image.url else read- assert storage_call.call_args.args[0] == f"aio/7/sha256/{'a' * 64}"-- @pytest.mark.parametrize(- ("provider", "model", "media", "storage_error"),- [- pytest.param("openai", "gpt-5.5", _pointer("a"), None, id="blob_missing_for_a_link"),- pytest.param("together_ai", "any", _pointer("a"), None, id="blob_missing_for_inline"),- pytest.param(- "together_ai", "any", _pointer("a"), ObjectStorageError("head_object failed"), id="storage_error"- ),- pytest.param(- "gemini", "gemini-3-flash", _pointer("a", mime="image%2Fgif"), None, id="format_rejected_by_provider"- ),- pytest.param(- "gemini",- "gemini-3-flash",- MediaBlock(modality="image", mime_type="image/png", base64_data="aGk"),- None,- id="invalid_inline_base64",- ),- pytest.param(- "gemini",- "gemini-3-flash",- MediaBlock(modality="image", mime_type="image/png", base64_data="aGk\u00e9"),- None,- id="non_ascii_inline_base64",- ),- ],- )- def test_reports_an_image_it_could_not_attach(- self, provider: str, model: str, media: MediaBlock, storage_error: Exception | None- ):- images = JudgeImages(team_id=7)- prompt = f"user: {images.marker(media)}"-- with (- patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value=None),- patch(f"{OBJECT_STORAGE}.head_object_strict", return_value=None, side_effect=storage_error),- ):- content = images.content(prompt, provider, model)-- assert content[1] == {"type": "text", "text": "Image 1: could not be attached."}-- def test_stops_calling_storage_after_a_storage_error(self):- images = JudgeImages(team_id=7)- prompt = f"user: {images.marker(_pointer('a'))} {images.marker(_pointer('b'))}"-- with patch(f"{OBJECT_STORAGE}.head_object_strict", side_effect=ObjectStorageError("timed out")) as head:- content = images.content(prompt, "together_ai", "any")-- assert head.call_count == 1- assert {"type": "text", "text": "Image 1: could not be attached."} in content- assert {"type": "text", "text": "Image 2: could not be attached."} in content-- @pytest.mark.parametrize(- ("provider", "media", "too_large", "reads"),- [- pytest.param("openai", _pointer, set(), 0, id="links_have_no_byte_cap"),- pytest.param("anthropic", _pointer, {1, 2}, 0, id="provider_limit_on_links"),- # The pointers claim 63 KB, so this also checks that the budget uses the stored size.- pytest.param("together_ai", _pointer, {1}, 1, id="stored_size_for_inline_providers"),- pytest.param(- "openai", lambda digit: _inline(digit, length=13_400_000), {1}, 0, id="inline_images_on_link_providers"- ),- ],- )- def test_caps_image_bytes(self, provider: str, media, too_large: set[int], reads: int):- images = JudgeImages(team_id=7)- prompt = f"user: {images.marker(media('a'))} {images.marker(media('b'))}"-- with (- patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed"),- patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 10_000_000}),- patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"x" * 10_000_000) as read,- ):- content = images.content(prompt, provider, "any")-- assert {- number- for number in (1, 2)- if {"type": "text", "text": f"Image {number}: not attached, too large."} in content- } == too_large- # An image over the budget is never downloaded.- assert read.call_count == reads-- def test_skips_images_whose_marker_was_sampled_out_of_the_prompt(self):- images = JudgeImages(team_id=7)- images.marker(_pointer("a"))-- assert images.content("user: hello", "openai", "gpt-5.5") == "user: hello"diff --git a/posthog/temporal/ai_observability/test_message_utils.py b/posthog/temporal/ai_observability/test_message_utils.pyindex bcdc89602c96..78de7915e0f1 100644--- a/posthog/temporal/ai_observability/test_message_utils.py+++ b/posthog/temporal/ai_observability/test_message_utils.py@@ -4,7 +4,7 @@ import jsonimport pytest-from posthog.temporal.ai_observability.judge_images import JudgeImages+from posthog.temporal.ai_observability.judge_attachments import JudgeAttachmentsfrom posthog.temporal.ai_observability.message_utils import (extract_text_from_messages,format_tool_definitions,@@ -113,20 +113,89 @@ class TestExtractTextFromMessages:pytest.param({"type": "image", "data": "aGk=", "mime_type": "image/png"}, "[image 1]", id="gemini_interactions"),- pytest.param({"inlineData": {"mimeType": "audio/wav", "data": "aGk="}}, "[audio]", id="gemini_audio"),+ pytest.param({"inlineData": {"mimeType": "audio/wav", "data": "aGk="}}, "[audio 1]", id="gemini_audio"),+ pytest.param(+ {"type": "input_audio", "input_audio": {"data": "aGk=", "format": "wav"}},+ "[audio 1]",+ id="openai_audio_wav",+ ),+ pytest.param(+ {+ "type": "input_audio",+ "input_audio": {"data": f"phaiblob://v1/sha256/{'b' * 64}?mime=audio%2Fmp3", "format": "mp3"},+ },+ "[audio 1]",+ id="openai_audio_offloaded",+ ),pytest.param({"type": "input_audio", "input_audio": {"data": "aGk="}}, "[audio]", id="openai_audio"),pytest.param({"type": "document", "source": {"type": "base64", "data": "aGk="}}, "[file]", id="file"),+ pytest.param(+ {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "aGk="}},+ "[file 1]",+ id="anthropic_pdf",+ ),+ pytest.param(+ {"type": "file", "file": {"filename": "a.pdf", "file_data": "data:application/pdf;base64,aGk="}},+ "[file 1]",+ id="openai_chat_pdf",+ ),+ pytest.param(+ {"type": "input_file", "filename": "a.pdf", "file_data": "data:application/pdf;base64,aGk="},+ "[file 1]",+ id="openai_responses_pdf",+ ),+ pytest.param(+ {"type": "file", "file": {"file_id": "file-abc"}}, "[file]", id="openai_uploaded_file_without_bytes"+ ),+ pytest.param(+ {"type": "file", "mediaType": "image/png", "file": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng"},+ "[image 1]",+ id="vercel_image",+ ),+ pytest.param(+ {"type": "file", "mediaType": "application/pdf", "data": "data:application/pdf;base64,aGk="},+ "[file 1]",+ id="vercel_pdf",+ ),+ pytest.param(+ {"type": "document", "mime_type": "application/pdf", "data": "aGk="}, "[file 1]", id="gemini_pdf"+ ),+ pytest.param(+ {"type": "file", "mediaType": "application/pdf", "data": "data:application/octet-stream;base64,aGk="},+ "[file 1]",+ id="declared_pdf_beats_a_generic_data_uri",+ ),+ pytest.param(+ {"type": "file", "mediaType": "APPLICATION/OCTET-STREAM", "data": "data:application/pdf;base64,aGk="},+ "[file 1]",+ id="data_uri_pdf_beats_a_generic_declared_type",+ ),+ pytest.param(+ {"type": "file", "file": {"filename": "report.PDF", "file_data": "aGk="}},+ "[file 1]",+ id="pdf_known_only_by_its_filename",+ ),+ pytest.param(+ {"type": "image", "uri": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng"},+ "[image 1]",+ id="otel_style_image_uri",+ ),+ pytest.param(+ {"type": "file", "mediaType": "image/png", "file": "raw files not supported"},+ "[image]",+ id="vercel_sdk_placeholder",+ ),],)- def test_judge_images_replace_media_blocks_with_markers(self, block, marker):+ def test_judge_attachments_replace_media_blocks_with_markers(self, block, marker):messages = [{"role": "user", "content": [block, {"type": "text", "text": "What does this chart show?"}]}]assert (- extract_text_from_messages(messages, JudgeImages(team_id=7).marker)+ extract_text_from_messages(messages, JudgeAttachments(team_id=7).marker)== f"user: {marker} What does this chart show?")- def test_judge_images_number_otel_parts_and_tool_result_images(self):+ def test_judge_attachments_number_otel_parts_and_tool_result_images(self):pointer = f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1"image = {"type": "image_url", "image_url": {"url": pointer}}messages = [@@ -143,12 +212,12 @@ class TestExtractTextFromMessages:]assert (- extract_text_from_messages(messages, JudgeImages(team_id=7).marker)+ extract_text_from_messages(messages, JudgeAttachments(team_id=7).marker)== "user: [image 1]\nuser: [image 1]\ntool[c1]: [image 1]")assert extract_text_from_messages(messages).startswith("user: [image]\n")- def test_media_blocks_keep_their_text_without_judge_images(self):+ def test_media_blocks_keep_their_text_without_judge_attachments(self):block = {"type": "file", "file": {"filename": "report.pdf"}}assert "report.pdf" in extract_text_from_messages([{"role": "user", "content": [block]}])diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.pyindex 5ae5ee7572aa..5282732d3575 100644--- a/posthog/temporal/ai_observability/test_run_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_evaluation.py@@ -28,13 +28,14 @@ from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Workerfrom posthog.api.capture import CaptureInternalErrorfrom posthog.constants import AvailableFeaturefrom posthog.models import Organization, OrganizationMembership, Team, User-from posthog.temporal.ai_observability.judge_images import ImagePrompt, JudgeImages+from posthog.temporal.ai_observability.judge_attachments import AttachmentPrompt, JudgeAttachmentsfrom posthog.temporal.ai_observability.sentiment.extraction import truncate_to_head_tailfrom posthog.temporal.ai_observability.sentiment.schema import SentimentResultfrom posthog.temporal.common.errors import NonReportableErrorfrom posthog.temporal.common.posthog_client import is_expected_activity_failurefrom products.access_control.backend.models.access_control import AccessControl+from products.ai_observability.backend.llm.attachments import AttachmentPartfrom products.ai_observability.backend.llm.errors import (RESPONSE_LIMIT_MESSAGE,AuthenticationError,@@ -51,7 +52,6 @@ from products.ai_observability.backend.llm.errors import (StructuredOutputParseError,UnsupportedModelError,)-from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.models.evaluation_config import EvaluationConfigfrom products.ai_observability.backend.models.evaluation_directories import EvaluationDirectoryfrom products.ai_observability.backend.models.evaluations import Evaluation@@ -1169,9 +1169,9 @@ def _adapter_rejection(cause: Exception | None) -> ProviderRequestRejectedError:def test_judge_falls_back_to_the_plain_prompt_only_when_an_attached_image_is_rejected(error: Exception, presigned_url: str | None, retries: bool) -> None:- images = JudgeImages(team_id=1)+ images = JudgeAttachments(team_id=1)pointer = MediaBlock(modality="image", url=f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1")- image_prompt = ImagePrompt(text=f"user: {images.marker(pointer)} Polite?", images=images)+ attachment_prompt = AttachmentPrompt(text=f"user: {images.marker(pointer)} Polite?", attachments=images)with (patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,@@ -1198,7 +1198,7 @@ def test_judge_falls_back_to_the_plain_prompt_only_when_an_attached_image_is_rejsystem_prompt="",user_prompt="user: Polite?",allows_na=False,- image_prompt=image_prompt,+ attachment_prompt=attachment_prompt,)if retries:@@ -1624,7 +1624,7 @@ class TestRunEvaluationWorkflow:with (patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client") as mock_client_class,patch(- "posthog.temporal.ai_observability.judge_images.posthoganalytics.feature_enabled",+ "posthog.temporal.ai_observability.judge_attachments.posthoganalytics.feature_enabled",return_value=flag_enabled,) as feature_enabled,patch(@@ -1644,7 +1644,7 @@ class TestRunEvaluationWorkflow:# Team ids repeat across regions, so the flag is checked against the project uuid.assert feature_enabled.call_args.kwargs["groups"] == {"project": str(team.uuid)}if attaches_image:- assert ImagePart(mime_type="image/jpeg", url="https://signed") in content+ assert AttachmentPart(mime_type="image/jpeg", url="https://signed") in contentassert "user: [image 1] What does this chart show?" in content[-1]["text"]assert "assistant: [image 2] Sales rose in March." in content[-1]["text"]else:@@ -4738,17 +4738,17 @@ class TestJudgePromptAssembly:with (patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as judge,patch(- "posthog.temporal.ai_observability.judge_images.posthoganalytics.feature_enabled",+ "posthog.temporal.ai_observability.judge_attachments.posthoganalytics.feature_enabled",return_value=True,),- patch("posthog.temporal.ai_observability.judge_images._project_uuid", return_value="uuid"),+ patch("posthog.temporal.ai_observability.judge_attachments._project_uuid", return_value="uuid"),):execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))- image_prompt = judge.call_args.kwargs["image_prompt"]- assert image_prompt is not None+ attachment_prompt = judge.call_args.kwargs["attachment_prompt"]+ assert attachment_prompt is not None# Decision judges and the text-only retry use this prompt, and they send no images.- assert judge.call_args.kwargs["user_prompt"] == image_prompt.unnumbered_text()+ assert judge.call_args.kwargs["user_prompt"] == attachment_prompt.unnumbered_text()assert "[image 1]" not in judge.call_args.kwargs["user_prompt"]@pytest.mark.parametrize(diff --git a/posthog/temporal/ai_observability/test_run_session_evaluation.py b/posthog/temporal/ai_observability/test_run_session_evaluation.pyindex 70a4b7cb3608..0ab649083086 100644--- a/posthog/temporal/ai_observability/test_run_session_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_session_evaluation.py@@ -20,7 +20,7 @@ from posthog.temporal.ai_observability.evaluation_llm_judge import (get_output_type_config,)from posthog.temporal.ai_observability.evaluation_payload import payload_budget_bytes-from posthog.temporal.ai_observability.judge_images import JudgeImages+from posthog.temporal.ai_observability.judge_attachments import JudgeAttachmentsfrom posthog.temporal.ai_observability.run_session_evaluation import (_SESSION_EVENT_COUNT_SQL,AI_EVENTS_RETENTION_DAYS,@@ -46,6 +46,7 @@ from posthog.temporal.ai_observability.run_trace_evaluation import ()from products.ai_observability.backend.hog import compile_ai_observability_hog+from products.ai_observability.backend.llm.attachments import AttachmentPartfrom products.cdp.backend.validation import compile_hogFROZEN_NOW = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC)@@ -148,7 +149,7 @@ class TestFormatSessionForJudge:for trace in traces:trace.events[0].properties["$ai_input"] = [{"role": "user", "content": [image]}]- rendered = format_session_for_judge(traces, JudgeImages(team_id=1).marker)+ rendered = format_session_for_judge(traces, JudgeAttachments(team_id=1).marker)assert rendered is not Noneassert rendered.count("[image 1]") == 2@@ -676,6 +677,39 @@ class TestExecuteSessionActivities:# The whole point of the truncation-as-skip choice: never grade a partial transcript.mock_call_llm_judge.assert_not_called()+ def test_judge_gets_the_session_images(self) -> None:+ trace = _trace("trace-1", cost=0, latency=0)+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}}+ trace.events[0].properties["$ai_input"] = [{"role": "user", "content": [image]}]+ with (+ patch(+ "posthog.temporal.ai_observability.run_session_evaluation.fetch_session_for_evaluation",+ return_value=SessionFetchOutcome(traces=[trace], skip_reason=None, event_count=1),+ ),+ patch("posthog.temporal.ai_observability.judge_attachments.judge_sees_attachments", return_value=True),+ patch("posthog.temporal.ai_observability.run_session_evaluation.call_llm_judge") as mock_call_llm_judge,+ ):+ execute_session_llm_judge_activity(+ ExecuteSessionEvaluationInputs(+ evaluation={+ "team_id": 1,+ "evaluation_type": "llm_judge",+ "evaluation_config": {"prompt": "Did the user accomplish their goal?"},+ "output_type": "boolean",+ "output_config": {},+ },+ team_id=1,+ session_id="s-1",+ window_start=FROZEN_NOW.isoformat(),+ )+ )++ attachment_prompt = mock_call_llm_judge.call_args.kwargs["attachment_prompt"]+ assert "[image 1]" in attachment_prompt.text+ assert attachment_prompt.attachments.content(attachment_prompt.text, "openai", "gpt-5.5")[2] == AttachmentPart(+ mime_type="image/png", data=b"hi"+ )+def test_judge_skips_without_judging_when_the_rendered_transcript_would_overflow(self):# The fetch itself succeeds (session_truncated wasn't raised there), but the session has# enough traces that format_session_for_judge's char budget can't fit all of them. This isdiff --git a/posthog/temporal/ai_observability/test_run_trace_evaluation.py b/posthog/temporal/ai_observability/test_run_trace_evaluation.pyindex 3b94246a8d4e..4267993810d6 100644--- a/posthog/temporal/ai_observability/test_run_trace_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_trace_evaluation.py@@ -23,7 +23,7 @@ from posthog.hogql_queries.ai.trace_query_runner import TraceQueryRunnerfrom posthog.models import Organization, PropertyDefinition, Team, Userfrom posthog.models.ai_events.test_util import bulk_create_ai_eventsfrom posthog.models.event.util import bulk_create_events-from posthog.temporal.ai_observability.judge_images import JudgeImages+from posthog.temporal.ai_observability.judge_attachments import JudgeAttachmentsfrom posthog.temporal.ai_observability.run_session_evaluation import fetch_session_for_evaluationfrom posthog.temporal.ai_observability.trace_summarization.fetch_and_format import _fetch_and_format_trace@@ -31,6 +31,7 @@ from products.access_control.backend.facade.contracts import PropertyAccessLevelfrom products.access_control.backend.models.access_control import AccessControlfrom products.access_control.backend.models.property_access_control import PropertyAccessControlfrom products.access_control.backend.property_access_control import restriction_cache_scope+from products.ai_observability.backend.llm.attachments import AttachmentPartfrom products.ai_observability.backend.models.evaluation_config import EvaluationConfigfrom products.ai_observability.backend.models.evaluations import Evaluationfrom products.ai_observability.backend.models.provider_keys import LLMProviderKey@@ -194,7 +195,7 @@ class TestFormatTraceForJudge:)plain = format_trace_for_judge(trace)- marked = format_trace_for_judge(trace, JudgeImages(team_id=1).marker)+ marked = format_trace_for_judge(trace, JudgeAttachments(team_id=1).marker)assert "[image 1]" not in plain# The message and the tool result repeat one image, and the OTel part adds a second.@@ -919,6 +920,37 @@ class TestExecuteTraceLLMJudgeActivity:assert result["verdict"] is Trueassert result["reasoning"] == "Resolved both questions"+ @pytest.mark.django_db(transaction=True)+ def test_sends_the_trace_images_to_a_judge_with_images(+ self, setup_data: dict[str, Any], active_key_config: None+ ) -> None:+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}}+ trace = create_trace([], inputState=[{"role": "user", "content": [image, {"type": "text", "text": "Chart?"}]}])+ with (+ patch(+ "posthog.temporal.ai_observability.run_trace_evaluation.fetch_trace_for_evaluation",+ return_value=TraceFetchOutcome(trace=trace, skip_reason=None, event_count=1),+ ),+ patch("posthog.temporal.ai_observability.judge_attachments.judge_sees_attachments", return_value=True),+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client") as mock_client_class,+ ):+ mock_client = mock_client_class.return_value+ mock_client.complete.return_value.parsed = BooleanEvalResult(verdict=True, reasoning="ok")+ mock_client.complete.return_value.usage = MagicMock(input_tokens=1, output_tokens=1, total_tokens=2)++ execute_trace_llm_judge_activity(+ ExecuteTraceEvaluationInputs(+ evaluation=evaluation_dict(setup_data),+ team_id=setup_data["team"].id,+ trace_id="trace-123",+ window_start=FROZEN_NOW.isoformat(),+ )+ )++ content = mock_client.complete.call_args[0][0].messages[0]["content"]+ assert AttachmentPart(mime_type="image/png", data=b"hi") in content+ assert "[image 1]" in content[-1]["text"]+@pytest.mark.parametrize("trace_state",[diff --git a/products/ai_observability/backend/llm/attachments.py b/products/ai_observability/backend/llm/attachments.pynew file mode 100644index 000000000000..6a43a21a2794--- /dev/null+++ b/products/ai_observability/backend/llm/attachments.py@@ -0,0 +1,85 @@+import base64+from collections.abc import Callable+from dataclasses import field+from typing import Any, Literal++from posthog.dataclasses import frozen++IMAGE_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/webp", "image/gif"})+PDF_MIME_TYPE = "application/pdf"+# The formats OpenAI's chat API takes as input audio, which Gemini reads too.+AUDIO_FORMATS = {"audio/wav": "wav", "audio/mpeg": "mp3"}+# Providers whose image blocks take a link that they download themselves.+_URL_PROVIDERS = frozenset({"openai", "azure_openai", "anthropic", "openrouter", "gemini"})+# Providers that read PDFs natively. OpenRouter is left out: it parses PDFs for other models with a paid OCR engine.+_PDF_PROVIDERS = frozenset({"openai", "anthropic", "gemini"})+# Raw bytes per image. Anthropic allows 10 MB base64-encoded, about 7.5 MB raw, so keep a margin under it.+MAX_IMAGE_BYTES = {"anthropic": 7_000_000}+# Raw bytes per PDF for every provider. Each page costs about as many tokens as an image, and links have no other cap.+MAX_PDF_BYTES = 10_000_000+++@frozen+class AttachmentPart:+ """An image, PDF or audio clip in a message's content list. Each adapter converts it to its provider's block.++ Set `url` for a link the provider downloads, or `data` for inline bytes.+ """++ mime_type: str+ url: str | None = None+ data: bytes | None = field(default=None, repr=False)++ def base64(self) -> str:+ return base64.b64encode(self.data or b"").decode()++ def data_uri(self) -> str:+ return f"data:{self.mime_type};base64,{self.base64()}"++ @property+ def is_pdf(self) -> bool:+ return self.mime_type == PDF_MIME_TYPE++ @property+ def audio_format(self) -> str | None:+ return AUDIO_FORMATS.get(self.mime_type)+++def attachment_transport(provider: str, model: str, mime_type: str) -> Literal["url", "inline"] | None:+ """How a provider accepts an attachment of this type: a link, inline base64, or not at all."""+ openai_audio_model = provider == "openai" and "audio" in model+ if mime_type in AUDIO_FORMATS:+ # OpenAI takes audio only as inline base64, and only on its audio models, so audio always goes inline.+ return "inline" if provider == "gemini" or openai_audio_model else None+ if openai_audio_model:+ # OpenAI's audio models reject images and PDFs, and the rejection would drop the audio too.+ return None+ is_pdf = mime_type == PDF_MIME_TYPE+ if not is_pdf and mime_type not in IMAGE_MIME_TYPES:+ return None+ if is_pdf and provider not in _PDF_PROVIDERS:+ return None+ if provider == "gemini":+ if mime_type == "image/gif":+ return None+ # The Gemini 2.0 family can't fetch external links.+ return "inline" if model.startswith("gemini-2.0") else "url"+ if is_pdf and provider == "openai":+ # The chat API takes a PDF only as inline file data.+ return "inline"+ return "url" if provider in _URL_PROVIDERS else "inline"+++def convert_attachment_parts(+ messages: list[dict[str, Any]], convert: Callable[[AttachmentPart], Any]+) -> list[dict[str, Any]]:+ """Replace each AttachmentPart in the messages' content lists with the provider block that `convert` returns."""+ return [+ {+ **message,+ "content": [convert(part) if isinstance(part, AttachmentPart) else part for part in message["content"]],+ }+ if isinstance(message.get("content"), list)+ else message+ for message in messages+ ]diff --git a/products/ai_observability/backend/llm/images.py b/products/ai_observability/backend/llm/images.pydeleted file mode 100644index 9d54236ca84d..000000000000--- a/products/ai_observability/backend/llm/images.py+++ /dev/null@@ -1,52 +0,0 @@-import base64-from collections.abc import Callable-from dataclasses import field-from typing import Any, Literal--from posthog.dataclasses import frozen--IMAGE_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/webp", "image/gif"})-# Providers whose image blocks take a link that they download themselves.-_URL_PROVIDERS = frozenset({"openai", "azure_openai", "anthropic", "openrouter", "gemini"})-# Raw bytes per image. Anthropic allows 10 MB base64-encoded, about 7.5 MB raw, so keep a margin under it.-MAX_IMAGE_BYTES = {"anthropic": 7_000_000}---@frozen-class ImagePart:- """An image in a message's content list. Each adapter converts it to its provider's image block.-- Set `url` for a link the provider downloads, or `data` for inline bytes.- """-- mime_type: str- url: str | None = None- data: bytes | None = field(default=None, repr=False)-- def base64(self) -> str:- return base64.b64encode(self.data or b"").decode()-- def data_uri(self) -> str:- return f"data:{self.mime_type};base64,{self.base64()}"---def image_transport(provider: str, model: str, mime_type: str) -> Literal["url", "inline"] | None:- """How a provider accepts an image of this type: a link, inline base64, or not at all."""- if mime_type not in IMAGE_MIME_TYPES:- return None- if provider == "gemini":- if mime_type == "image/gif":- return None- # The Gemini 2.0 family can't fetch external links.- return "inline" if model.startswith("gemini-2.0") else "url"- return "url" if provider in _URL_PROVIDERS else "inline"---def convert_image_parts(messages: list[dict[str, Any]], convert: Callable[[ImagePart], Any]) -> list[dict[str, Any]]:- """Replace each ImagePart in the messages' content lists with the provider block that `convert` returns."""- return [- {**message, "content": [convert(part) if isinstance(part, ImagePart) else part for part in message["content"]]}- if isinstance(message.get("content"), list)- else message- for message in messages- ]diff --git a/products/ai_observability/backend/llm/providers/anthropic.py b/products/ai_observability/backend/llm/providers/anthropic.pyindex 3cd96efa68a5..115b302767e8 100644--- a/products/ai_observability/backend/llm/providers/anthropic.py+++ b/products/ai_observability/backend/llm/providers/anthropic.py@@ -17,6 +17,7 @@ from posthoganalytics.ai.anthropic.anthropic import WrappedMessagesfrom posthoganalytics.client import Client as PostHogClientfrom pydantic import BaseModel+from products.ai_observability.backend.llm.attachments import AttachmentPart, convert_attachment_partsfrom products.ai_observability.backend.llm.errors import (AuthenticationError,ContextWindowExceededError,@@ -34,7 +35,6 @@ from products.ai_observability.backend.llm.errors import (is_output_limit_error_message,stream_error_chunk,)-from products.ai_observability.backend.llm.images import ImagePart, convert_image_partsfrom products.ai_observability.backend.llm.providers.azure_openai import is_allowed_azure_endpointfrom products.ai_observability.backend.llm.types import (AnalyticsContext,@@ -230,7 +230,7 @@ class AnthropicAdapter:effective_api_key = api_key or self._get_default_api_key()client = self._create_client(effective_api_key, analytics)- messages: Any = convert_image_parts(request.messages, _anthropic_image_block)+ messages: Any = convert_attachment_parts(request.messages, _anthropic_attachment_block)system_prompt = request.system or ""use_structured = request.response_format and issubclass(request.response_format, BaseModel)@@ -360,7 +360,7 @@ class AnthropicAdapter:tools = self._convert_tools(request.tools) if request.tools else Nonesystem = request.system or ""- messages = convert_image_parts(request.messages, _anthropic_image_block)+ messages = convert_attachment_parts(request.messages, _anthropic_attachment_block)# Handle cache control for supported modelssystem_prompt: list[TextBlockParam] = []@@ -641,7 +641,8 @@ class AnthropicAdapter:return prepared_messages-def _anthropic_image_block(part: ImagePart) -> dict[str, Any]:+def _anthropic_attachment_block(part: AttachmentPart) -> dict[str, Any]:+ block_type = "document" if part.is_pdf else "image"if part.url is not None:- return {"type": "image", "source": {"type": "url", "url": part.url}}- return {"type": "image", "source": {"type": "base64", "media_type": part.mime_type, "data": part.base64()}}+ return {"type": block_type, "source": {"type": "url", "url": part.url}}+ return {"type": block_type, "source": {"type": "base64", "media_type": part.mime_type, "data": part.base64()}}diff --git a/products/ai_observability/backend/llm/providers/openai.py b/products/ai_observability/backend/llm/providers/openai.pyindex b69b662732c0..90e6316ae8b8 100644--- a/products/ai_observability/backend/llm/providers/openai.py+++ b/products/ai_observability/backend/llm/providers/openai.py@@ -20,6 +20,7 @@ from posthoganalytics.ai.openai import ()from pydantic import BaseModel, ValidationError+from products.ai_observability.backend.llm.attachments import AttachmentPart, convert_attachment_partsfrom products.ai_observability.backend.llm.errors import (AuthenticationError,ContentFilteredError,@@ -36,7 +37,6 @@ from products.ai_observability.backend.llm.errors import (is_output_limit_error_message,stream_error_chunk,)-from products.ai_observability.backend.llm.images import ImagePart, convert_image_partsfrom products.ai_observability.backend.llm.types import (AnalyticsContext,CompletionRequest,@@ -311,7 +311,7 @@ You must respond with valid JSON that matches this schema:Return ONLY the JSON object, no other text or markdown formatting."""enhanced_messages = [{"role": "system", "content": enhanced_system}]- enhanced_messages.extend(convert_image_parts(request.messages, _openai_image_block))+ enhanced_messages.extend(convert_attachment_parts(request.messages, _openai_attachment_block))create_response = client.chat.completions.create(model=request.model,@@ -520,7 +520,7 @@ Return ONLY the JSON object, no other text or markdown formatting."""messages = []if request.system:messages.append({"role": "system", "content": request.system})- messages.extend(convert_image_parts(request.messages, _openai_image_block))+ messages.extend(convert_attachment_parts(request.messages, _openai_attachment_block))return messagesdef _build_analytics_kwargs(self, analytics: AnalyticsContext, client) -> dict:@@ -573,5 +573,10 @@ Return ONLY the JSON object, no other text or markdown formatting."""return result-def _openai_image_block(part: ImagePart) -> dict[str, Any]:+def _openai_attachment_block(part: AttachmentPart) -> dict[str, Any]:+ if part.audio_format is not None:+ return {"type": "input_audio", "input_audio": {"data": part.base64(), "format": part.audio_format}}+ if part.is_pdf:+ # The chat API takes a PDF only as inline data, and requires a file name with it.+ return {"type": "file", "file": {"filename": "attachment.pdf", "file_data": part.data_uri()}}return {"type": "image_url", "image_url": {"url": part.url or part.data_uri()}}diff --git a/products/ai_observability/backend/llm/providers/test/test_anthropic.py b/products/ai_observability/backend/llm/providers/test/test_anthropic.pyindex 56d0a632a23c..3a5fd718195f 100644--- a/products/ai_observability/backend/llm/providers/test/test_anthropic.py+++ b/products/ai_observability/backend/llm/providers/test/test_anthropic.py@@ -5,6 +5,7 @@ import httpximport anthropicfrom parameterized import parameterized+from products.ai_observability.backend.llm.attachments import AttachmentPartfrom products.ai_observability.backend.llm.errors import (ContextWindowExceededError,ModelNotFoundError,@@ -12,7 +13,6 @@ from products.ai_observability.backend.llm.errors import (OutputTokenLimitError,ProviderConfigurationError,)-from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.llm.providers.anthropic import (AnthropicAdapter,AnthropicConfig,@@ -173,22 +173,30 @@ class TestAnthropicStreamOutputSchema:assert "output_config" not in self._stream_kwargs(None)-class TestAnthropicImageParts:+class TestAnthropicAttachmentParts:@parameterized.expand([("link",- ImagePart(mime_type="image/png", url="https://example.com/a.png"),+ AttachmentPart(mime_type="image/png", url="https://example.com/a.png"),{"type": "url", "url": "https://example.com/a.png"},),("inline",- ImagePart(mime_type="image/png", data=b"hi"),+ AttachmentPart(mime_type="image/png", data=b"hi"),{"type": "base64", "media_type": "image/png", "data": "aGk="},),+ (+ "pdf_link",+ AttachmentPart(mime_type="application/pdf", url="https://example.com/a.pdf"),+ {"type": "url", "url": "https://example.com/a.pdf"},+ "document",+ ),])- def test_image_parts_are_sent_as_image_blocks(self, _name: str, image: ImagePart, expected_source: dict):+ def test_attachment_parts_are_sent_as_blocks(+ self, _name: str, image: AttachmentPart, expected_source: dict, expected_type: str = "image"+ ):request = CompletionRequest(model="claude-haiku-4-5",messages=[{"role": "user", "content": [image, {"type": "text", "text": "hi"}]}],@@ -202,7 +210,7 @@ class TestAnthropicImageParts:AnthropicAdapter().complete(request, api_key="sk-ant-test", analytics=AnalyticsContext(capture=False))assert mock_client.messages.create.call_args.kwargs["messages"][0]["content"] == [- {"type": "image", "source": expected_source},+ {"type": expected_type, "source": expected_source},{"type": "text", "text": "hi"},]diff --git a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.pyindex 93cce2da4829..7280577a605a 100644--- a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py+++ b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py@@ -12,6 +12,7 @@ from posthoganalytics.ai.openai import ()from pydantic import BaseModel, ValidationError, model_validator+from products.ai_observability.backend.llm.attachments import AttachmentPartfrom products.ai_observability.backend.llm.errors import (ContentFilteredError,ContextWindowExceededError,@@ -19,7 +20,6 @@ from products.ai_observability.backend.llm.errors import (QuotaExceededError,StructuredOutputParseError,)-from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.llm.providers.openai import OpenAIAdapter, OpenAIConfigfrom products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema@@ -178,23 +178,40 @@ class TestOpenAIStreamOutputSchema:assert "response_format" not in self._stream_kwargs(None)-class TestOpenAIImageParts:+class TestOpenAIAttachmentParts:@pytest.mark.parametrize(- ("image", "expected_url"),+ ("part", "expected_block"),[pytest.param(- ImagePart(mime_type="image/png", url="https://example.com/a.png"),- "https://example.com/a.png",+ AttachmentPart(mime_type="image/png", url="https://example.com/a.png"),+ {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},id="link",),- pytest.param(ImagePart(mime_type="image/png", data=b"hi"), "data:image/png;base64,aGk=", id="inline"),+ pytest.param(+ AttachmentPart(mime_type="image/png", data=b"hi"),+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}},+ id="inline",+ ),+ pytest.param(+ AttachmentPart(mime_type="application/pdf", data=b"hi"),+ {+ "type": "file",+ "file": {"filename": "attachment.pdf", "file_data": "data:application/pdf;base64,aGk="},+ },+ id="pdf",+ ),+ pytest.param(+ AttachmentPart(mime_type="audio/mpeg", data=b"hi"),+ {"type": "input_audio", "input_audio": {"data": "aGk=", "format": "mp3"}},+ id="audio",+ ),],)@pytest.mark.parametrize("json_fallback", [False, True], ids=["plain", "json_fallback"])- def test_image_parts_are_sent_as_image_url_blocks(self, image: ImagePart, expected_url: str, json_fallback: bool):+ def test_attachment_parts_are_sent_as_blocks(self, part: AttachmentPart, expected_block: dict, json_fallback: bool):request = CompletionRequest(model="gpt-4.1",- messages=[{"role": "user", "content": [image, {"type": "text", "text": "hi"}]}],+ messages=[{"role": "user", "content": [part, {"type": "text", "text": "hi"}]}],provider="openai",response_format=_Verdict if json_fallback else None,)@@ -209,7 +226,7 @@ class TestOpenAIImageParts:sent = mock_client.chat.completions.create.call_args.kwargs["messages"]assert next(message for message in sent if message["role"] == "user")["content"] == [- {"type": "image_url", "image_url": {"url": expected_url}},+ expected_block,{"type": "text", "text": "hi"},]diff --git a/products/ai_observability/backend/providers/formatters/gemini_formatter.py b/products/ai_observability/backend/providers/formatters/gemini_formatter.pyindex 209e5f27aaa8..7d1f857281ca 100644--- a/products/ai_observability/backend/providers/formatters/gemini_formatter.py+++ b/products/ai_observability/backend/providers/formatters/gemini_formatter.py@@ -4,7 +4,7 @@ from typing import Any, castfrom google.genai.types import Blob, Content, ContentListUnion, FileData, FunctionCall, FunctionResponse, Part-from products.ai_observability.backend.llm.images import ImagePart+from products.ai_observability.backend.llm.attachments import AttachmentPartfrom products.ai_observability.backend.providers.formatters.anthropic_typeguards import (is_base64_image_param,is_image_block_param,@@ -85,7 +85,7 @@ def _tool_response_payload(content: Any, *, is_error: bool = False) -> dict[str,return value if isinstance(value, dict) else {"result": value}-def _gemini_image_part(image: ImagePart) -> Part:+def _gemini_attachment_part(image: AttachmentPart) -> Part:if image.url is not None:return Part(file_data=FileData(file_uri=image.url, mime_type=image.mime_type))return Part(inline_data=Blob(data=image.data or b"", mime_type=image.mime_type))@@ -103,8 +103,8 @@ def convert_anthropic_messages_to_gemini(messages: list[dict[str, Any]]) -> Contparts.append(Part(text=message["content"]))elif isinstance(message["content"], list):for block in message["content"]:- if isinstance(block, ImagePart):- parts.append(_gemini_image_part(block))+ if isinstance(block, AttachmentPart):+ parts.append(_gemini_attachment_part(block))elif is_text_block_param(block):parts.append(Part(text=block["text"]))elif is_image_block_param(block):diff --git a/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py b/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.pyindex 676458ec055f..c63ad13fb4e6 100644--- a/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py+++ b/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py@@ -5,7 +5,7 @@ import pytestfrom google.genai.types import Blob, Content, FileData, FunctionCall, FunctionResponse, Partfrom parameterized import parameterized-from products.ai_observability.backend.llm.images import ImagePart+from products.ai_observability.backend.llm.attachments import AttachmentPartfrom products.ai_observability.backend.providers.formatters.gemini_formatter import (MessageConversionError,convert_anthropic_messages_to_gemini,@@ -97,17 +97,17 @@ class TestConvertAnthropicMessagesToGemini:[("link",- ImagePart(mime_type="image/png", url="https://example.com/a.png"),+ AttachmentPart(mime_type="image/png", url="https://example.com/a.png"),Part(file_data=FileData(file_uri="https://example.com/a.png", mime_type="image/png")),),("inline",- ImagePart(mime_type="image/png", data=b"hi"),+ AttachmentPart(mime_type="image/png", data=b"hi"),Part(inline_data=Blob(data=b"hi", mime_type="image/png")),),])- def test_image_part_becomes_gemini_part(self, _name: str, image: ImagePart, expected: Part):+ def test_image_part_becomes_gemini_part(self, _name: str, image: AttachmentPart, expected: Part):contents = _convert([{"role": "user", "content": [image]}])assert _part(contents[0], 0) == expecteddiff --git a/products/ai_observability/backend/text_repr/formatters/media_blocks.py b/products/ai_observability/backend/text_repr/formatters/media_blocks.pyindex 210d9eb50ab0..6605338ca21e 100644--- a/products/ai_observability/backend/text_repr/formatters/media_blocks.py+++ b/products/ai_observability/backend/text_repr/formatters/media_blocks.py@@ -1,5 +1,6 @@import refrom collections.abc import Callable+from dataclasses import replacefrom posthog.dataclasses import frozen@@ -8,6 +9,9 @@ _AUDIO_TYPES = frozenset({"audio", "input_audio"})_FILE_TYPES = frozenset({"document", "file", "input_file"})_GEMINI_DATA_KEYS = ("inline_data", "inlineData", "file_data", "fileData")_OTEL_TYPES = frozenset({"blob", "uri"})+_GENERIC_MIME_TYPE = "application/octet-stream"+_MIME_KEYS = ("mediaType", "mimeType", "mime_type", "media_type")+_FILE_SOURCE_KEYS = ("file_data", "file_url", "data", "uri", "url")_LINK_SCHEMES = ("https://", "http://", "phaiblob://")# Without full capture the SDKs replace the bytes with a placeholder such as `[base64 image redacted]`.# Same pattern as the frontend's `isRedactedMediaSentinel`.@@ -62,9 +66,9 @@ def parse_media_block(block: dict) -> MediaBlock | None:if block_type in _IMAGE_TYPES:return _image_block(block)if block_type in _AUDIO_TYPES:- return MediaBlock(modality="audio")+ return _audio_block(block)if block_type in _FILE_TYPES:- return MediaBlock(modality="file")+ return _file_block(block)return None@@ -97,7 +101,47 @@ def _image_block(block: dict) -> MediaBlock:)# The Gemini Interactions API puts `data` and `mime_type` on the block itself.mime_type = block.get("mime_type") or block.get("mimeType")- return _with_source("image", mime_type if isinstance(mime_type, str) else None, image_url or block.get("data"))+ return _with_source(+ "image", mime_type if isinstance(mime_type, str) else None, image_url or block.get("data") or block.get("uri")+ )+++def _audio_block(block: dict) -> MediaBlock:+ # OpenAI chat names the format instead of a MIME type: `{"input_audio": {"data": ..., "format": "wav"}}`.+ audio = block.get("input_audio")+ if isinstance(audio, dict):+ audio_format = audio.get("format")+ mime_type = f"audio/{audio_format}" if isinstance(audio_format, str) and audio_format else None+ return _with_source("audio", mime_type, audio.get("data"))+ source = block.get("source")+ fields = source if isinstance(source, dict) else block+ mime_type = next((fields[key] for key in _MIME_KEYS if isinstance(fields.get(key), str)), None)+ value = next((fields[key] for key in _FILE_SOURCE_KEYS if fields.get(key)), None)+ return _with_source("audio", mime_type, value)+++def _file_block(block: dict) -> MediaBlock:+ source = block.get("source")+ if isinstance(source, dict):+ # Anthropic document.+ mime_type, value = source.get("media_type"), source.get("data") or source.get("url")+ filename = None+ else:+ file = block.get("file")+ # OpenAI chat nests the file, and the Responses API, Gemini and the Vercel AI SDK put it on the block.+ # Vercel records `{"type": "file", "mediaType": ..., "file": <data>}`.+ fields = file if isinstance(file, dict) else block+ mime_type = next((block[key] for key in _MIME_KEYS if isinstance(block.get(key), str)), None)+ value = next((fields[key] for key in _FILE_SOURCE_KEYS if fields.get(key)), None)+ if value is None and isinstance(file, str):+ value = file+ filename = fields.get("filename")+ media = _with_source("file", mime_type if isinstance(mime_type, str) else None, value)+ if media.mime_type is None and isinstance(filename, str) and filename.lower().endswith(".pdf"):+ media = replace(media, mime_type="application/pdf")+ # The Vercel AI SDK sends images, audio and video as file parts too.+ modality = media.mime_type.split("/")[0] if media.mime_type else ""+ return replace(media, modality=modality) if modality in ("image", "audio", "video") else mediadef _with_source(modality: str, mime_type: str | None, value: object) -> MediaBlock:@@ -107,9 +151,14 @@ def _with_source(modality: str, mime_type: str | None, value: object) -> MediaBl# Unwrap one envelope only. Its payload is bytes or a wrapped pointer, never another data URI,# and unwrapping nested prefixes would copy the payload once per level.header, value = value[len("data:") :].split(";base64,", 1)- mime_type = header or mime_type+ # A declared type beats a generic one in the data URI, as in the trace view.+ if header and (mime_type is None or mime_type.strip().lower() == _GENERIC_MIME_TYPE):+ mime_type = headerif _REDACTED_RE.match(value) or value.startswith("data:"):return MediaBlock(modality=modality, mime_type=mime_type)if value.startswith(_LINK_SCHEMES):return MediaBlock(modality=modality, mime_type=mime_type, url=value)+ # Base64 never holds a space, so a value with one is an SDK placeholder such as `raw files not supported`.+ if " " in value:+ return MediaBlock(modality=modality, mime_type=mime_type)return MediaBlock(modality=modality, mime_type=mime_type, base64_data=value)diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsxindex 3dee7c4c5f35..4d9018869803 100644--- a/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx+++ b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx@@ -41,7 +41,9 @@ export function EvaluationModelPanel(): JSX.Element | null {data-attr="llma-evaluation-include-attachments"/><p className="text-muted text-sm">- The judge sees the images in what it evaluates. Other file types are not supported yet.+ The judge sees the images, PDFs and audio in what it evaluates. PDFs need an OpenAI, Anthropic+ or Gemini judge, and audio a Gemini judge or an OpenAI audio model. Other file types are not+ supported yet.</p></LemonField.Pure>)}