Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): let llm judges see pdfs, audio and vercel ai sdk file parts (#126272)

Bernat Torrese333c7285b84
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 9ce5903e0806..214f93384d47 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -43,8 +43,11 @@ Generation evaluations extract message text without the per-message character cu
They reserve space for the complete output, then sample input and tool definitions into the remaining space within the 150,000-character limit.
If the output and section labels alone exceed that limit, the run is skipped with reason `generation_output_too_long` without calling the judge.
An exception applies when the judge sees images (the `llm-analytics-judge-images` flag is on and the evaluation does not turn attachments off).
-Then each image becomes an `[image N]` marker instead of its base64 or link, so an output with large inline images can fit and the run continues.
-The image labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 images, plus one summary line for the rest.
+Then each image becomes an `[image N]` marker and each PDF a `[file N]` marker instead of its base64 or link, so an output with large inline attachments can fit and the run continues.
+The attachment labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 attachments, plus one summary line for the rest.
+PDFs reach only OpenAI, Anthropic and Gemini judges, up to 10 MB each.
+WAV and MP3 audio becomes an `[audio N]` marker and reaches Gemini judges and OpenAI audio models, always inline.
+Attachments a judge can't read take none of its 20 slots, and one summary line counts them.
Generation evaluation and LLM tagger user prompts repair UTF-16 surrogate sequences after sampling.
Lone surrogates become replacement characters (`�`), while valid pairs become their Unicode character.
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 2b474a65e0b4..9cb5a12ddbf9 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -41,7 +41,7 @@ from posthog.temporal.ai_observability.evaluation_types import (
build_generation_not_found_result,
build_skipped_evaluation_result,
)
-from posthog.temporal.ai_observability.judge_images import ImagePrompt, build_image_prompt
+from posthog.temporal.ai_observability.judge_attachments import AttachmentPrompt, build_attachment_prompt
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages, format_tool_definitions
from posthog.temporal.ai_observability.metrics import (
increment_errors,
@@ -56,6 +56,7 @@ from posthog.temporal.common.errors import NonReportableApplicationError, NonRep
from posthog.temporal.common.utils import close_db_connections
from products.ai_observability.backend.llm import DEFAULT_MODEL_BY_PROVIDER, Client, CompletionRequest, Usage
+from products.ai_observability.backend.llm.attachments import AttachmentPart
from products.ai_observability.backend.llm.decisions import (
DecisionClient,
DecisionEndpointBlockedError,
@@ -80,7 +81,6 @@ from products.ai_observability.backend.llm.errors import (
UnsupportedModelError,
provider_error_detail,
)
-from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.llm.providers.openai import OPENAI_DECISIONS_BASE_URL
from products.ai_observability.backend.llm.providers.openai_decisions import OpenAIDecisionAdapter
from products.ai_observability.backend.llm.providers.openrouter import OPENROUTER_DECISIONS_BASE_URL
@@ -450,11 +450,11 @@ def _build_unparsable_response_skip_result(
_IMAGE_REJECTION_STATUSES = frozenset({400, 413, 415, 422})
-class JudgeImagesRejectedError(Exception):
+class JudgeAttachmentsRejectedError(Exception):
"""Reported without the provider's message, which can quote a live presigned link."""
-def _image_rejection_status(error: Exception) -> int | None:
+def _attachment_rejection_status(error: Exception) -> int | None:
"""The status of a provider rejection an image can cause, raw or wrapped by an adapter."""
for candidate in (error, error.__cause__):
if (
@@ -585,14 +585,14 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActi
system_prompt = build_system_prompt(prompt, allows_na, output_type=output_type, output_config=output_config)
user_prompt = _build_user_prompt(io.input_raw, io.output_raw, tools_data)
- image_prompt = build_image_prompt(
+ attachment_prompt = build_attachment_prompt(
evaluation,
user_prompt,
lambda media_marker: _build_user_prompt(io.input_raw, io.output_raw, tools_data, media_marker=media_marker),
)
- if user_prompt is None and image_prompt is not None:
+ if user_prompt is None and attachment_prompt is not None:
# Inline base64 can push the plain text over the limit when the marked text still fits.
- user_prompt = image_prompt.unnumbered_text()
+ user_prompt = attachment_prompt.unnumbered_text()
if user_prompt is None:
return build_skipped_evaluation_result(
output_type=output_type,
@@ -607,7 +607,7 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActi
user_prompt=user_prompt,
allows_na=allows_na,
backfill=inputs.backfill_id is not None,
- image_prompt=image_prompt,
+ attachment_prompt=attachment_prompt,
)
@@ -650,7 +650,7 @@ def call_llm_judge(
user_prompt: str,
allows_na: bool,
backfill: bool = False,
- image_prompt: ImagePrompt | None = None,
+ attachment_prompt: AttachmentPrompt | None = None,
) -> EvaluationActivityResult:
"""Resolve the judge model/key for `evaluation` and run a single judge completion.
@@ -859,8 +859,12 @@ def call_llm_judge(
),
)
else:
- content = image_prompt.images.content(image_prompt.text, provider, model) if image_prompt else user_prompt
- sent_images = isinstance(content, list) and any(isinstance(part, ImagePart) for part in content)
+ content = (
+ attachment_prompt.attachments.content(attachment_prompt.text, provider, model)
+ if attachment_prompt
+ else user_prompt
+ )
+ sent_attachments = isinstance(content, list) and any(isinstance(part, AttachmentPart) for part in content)
request = CompletionRequest(
model=model,
system=system_prompt,
@@ -873,13 +877,13 @@ def call_llm_judge(
try:
response = client.complete(request)
except Exception as e:
- status = _image_rejection_status(e) if sent_images else None
+ status = _attachment_rejection_status(e) if sent_attachments else None
if status is None:
raise
# Handle the rejection outside the except: the provider's error can quote a presigned link, and
# an active exception reaches the logs through capture_exception and chains onto a retry failure.
if status is not None:
- # A model without image input, or an image link the provider could not download.
+ # A model without image or PDF input, or a link the provider could not download.
# Grade the plain transcript rather than skip the run.
context = {
"evaluation_id": evaluation["id"],
@@ -887,11 +891,14 @@ def call_llm_judge(
"provider": provider,
"model": model,
"status": status,
+ "sent_pdfs": any(isinstance(part, AttachmentPart) and part.is_pdf for part in content),
}
- logger.warning("judge_images_rejected", **context)
+ logger.warning("judge_attachments_rejected", **context)
if not is_byok:
- # On a PostHog key this can mean broken presigned links, so keep it visible.
- capture_exception(JudgeImagesRejectedError(f"Provider rejected judge images ({status})"), context)
+ # On a PostHog key this can mean broken presigned links or a refused PDF, so keep it visible.
+ capture_exception(
+ JudgeAttachmentsRejectedError(f"Provider rejected judge images ({status})"), context
+ )
response = client.complete(replace(request, messages=[{"role": "user", "content": user_prompt}]))
except (DecisionEndpointBlockedError, ProviderConfigurationError) as e:
increment_user_errors("endpoint_blocked", provider=provider)
diff --git a/posthog/temporal/ai_observability/judge_images.py b/posthog/temporal/ai_observability/judge_attachments.py
similarity index 52%
rename from posthog/temporal/ai_observability/judge_images.py
rename to posthog/temporal/ai_observability/judge_attachments.py
index 121420fed86b..ec6b971083e9 100644
--- a/posthog/temporal/ai_observability/judge_images.py
+++ b/posthog/temporal/ai_observability/judge_attachments.py
@@ -1,4 +1,4 @@
-"""Images an LLM judge sees next to the transcript that refers to them as `[image N]`."""
+"""Images, PDFs and audio an LLM judge gets next to the transcript, which refers to them by markers such as `[image N]`."""
import re
import base64
@@ -15,15 +15,23 @@ from posthog.models.team import Team
from posthog.storage.object_storage import ObjectStorageError
from products.ai_observability.backend.facade.blobs import BlobPointer
-from products.ai_observability.backend.llm.images import IMAGE_MIME_TYPES, MAX_IMAGE_BYTES, ImagePart, image_transport
+from products.ai_observability.backend.llm.attachments import (
+ AUDIO_FORMATS,
+ IMAGE_MIME_TYPES,
+ MAX_IMAGE_BYTES,
+ MAX_PDF_BYTES,
+ PDF_MIME_TYPE,
+ AttachmentPart,
+ attachment_transport,
+)
from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, MediaMarker
logger = structlog.get_logger(__name__)
-JUDGE_IMAGES_FLAG = "llm-analytics-judge-images"
+JUDGE_ATTACHMENTS_FLAG = "llm-analytics-judge-images"
-# Each image costs tokens on the customer's key, and above 20 images Anthropic tightens its per-image dimension limit.
-MAX_IMAGES = 20
+# Each attachment costs tokens on the customer's key, and above 20 images Anthropic tightens its per-image dimension limit.
+MAX_ATTACHMENTS = 20
# Bytes sent inline in one judge request. Links don't count: the provider downloads them.
MAX_INLINE_BYTES = 15_000_000
# Covers the judge call and its retries. Sign right before the call: a link signed with temporary
@@ -38,44 +46,54 @@ _NOT_ATTACHED_LABELS: dict[Outcome, str] = {
}
-_NUMBERED_MARKER_RE = re.compile(r"\[image \d+\]")
+_NUMBERED_MARKER_RE = re.compile(r"\[(image|file|audio) \d+\]")
@frozen
-class ImagePrompt:
- """The judge transcript with `[image N]` markers, and the images those markers refer to."""
+class AttachmentPrompt:
+ """The judge transcript with attachment markers, and the attachments they refer to."""
text: str
- images: "JudgeImages"
+ attachments: "JudgeAttachments"
def unnumbered_text(self) -> str:
- """The transcript for a judge call that sends no images, so no marker points at a missing image."""
- return _NUMBERED_MARKER_RE.sub("[image]", self.text)
+ """The transcript for a judge call that sends no attachments, so no marker points at a missing one."""
+ return _NUMBERED_MARKER_RE.sub(r"[\1]", self.text)
@frozen
class _Attachment:
outcome: Outcome
- part: ImagePart | None = None
+ part: AttachmentPart | None = None
inline_bytes: int = 0
@frozen
-class _Image:
+class _Media:
+ kind: Literal["image", "file", "audio"]
number: int
mime_type: str
# An offloaded image has a pointer, a small one stays inline as base64.
pointer: BlobPointer | None
base64_data: str | None
+ @property
+ def marker(self) -> str:
+ return f"[{self.kind} {self.number}]"
-class JudgeImages:
- """Numbers the images found while formatting a transcript, then attaches them to the judge request."""
+ @property
+ def label(self) -> str:
+ return f"{self.kind.capitalize()} {self.number}"
+
+
+class JudgeAttachments:
+ """Numbers the attachments found while formatting a transcript, then attaches them to the judge request."""
def __init__(self, team_id: int) -> None:
self._team_id = team_id
- self._images: list[_Image] = []
- self._numbers: dict[str, int] = {}
+ self._media: list[_Media] = []
+ self._media_by_key: dict[str, _Media] = {}
+ self._counts: Counter[str] = Counter()
# After one storage failure the rest would fail too, each after its own timeout and retry.
self._storage_failed = False
@@ -84,65 +102,81 @@ class JudgeImages:
mime_type = _normalize_mime_type(media.mime_type or (pointer.mime_type if pointer else None))
# Links from the customer's app keep a plain marker: we never fetch them.
key = pointer.sha256 if pointer else media.base64_data
- if media.modality != "image" or mime_type not in IMAGE_MIME_TYPES or key is None:
+ kind = _kind(media.modality, mime_type)
+ if kind is None or mime_type is None or key is None:
return media.marker()
- if (number := self._numbers.get(key)) is None:
- number = self._numbers[key] = len(self._images) + 1
- self._images.append(
- _Image(number=number, mime_type=mime_type, pointer=pointer, base64_data=media.base64_data)
+ if (image := self._media_by_key.get(key)) is None:
+ self._counts[kind] += 1
+ image = self._media_by_key[key] = _Media(
+ kind=kind,
+ number=self._counts[kind],
+ mime_type=mime_type,
+ pointer=pointer,
+ base64_data=media.base64_data,
)
- return f"[image {number}]"
+ self._media.append(image)
+ return image.marker
def content(self, user_prompt: str, provider: str, model: str) -> str | list[Any]:
- """The user message content: each referenced image under its label, then the transcript."""
- # Line sampling can drop a marker from a long transcript, and then its image only costs tokens.
- images = [image for image in self._images if f"[image {image.number}]" in user_prompt]
+ """The user message content: each referenced attachment under its label, then the transcript."""
+ # Line sampling can drop a marker from a long transcript, and then its attachment only costs tokens.
+ images = [image for image in self._media if image.marker in user_prompt]
if not images:
return user_prompt
+ # Attachments this judge can't read take no slot, so they can't push out ones it can.
+ readable = [image for image in images if attachment_transport(provider, model, image.mime_type) is not None]
# Keep the images referenced last: the newest input and the graded output, even when it repeats an old image.
- ranked = sorted(images, key=lambda image: user_prompt.rfind(f"[image {image.number}]"))
- attachments: dict[int, _Attachment] = {}
+ ranked = sorted(readable, key=lambda image: user_prompt.rfind(image.marker))
+ attachments: dict[str, _Attachment] = {}
inline_bytes = 0
- for image in reversed(ranked[-MAX_IMAGES:]):
+ for image in reversed(ranked[-MAX_ATTACHMENTS:]):
attachment = self._attach(image, provider, model, MAX_INLINE_BYTES - inline_bytes)
inline_bytes += attachment.inline_bytes
- attachments[image.number] = attachment
+ attachments[image.marker] = attachment
- parts: list[Any] = [{"type": "text", "text": "The transcript refers to these images as [image N]."}]
+ parts: list[Any] = [
+ {"type": "text", "text": "The transcript refers to these attachments as [image N], [file N] or [audio N]."}
+ ]
outcomes: Counter[str] = Counter()
for image in images:
- if image.number not in attachments:
+ if image.marker not in attachments:
continue
- attachment = attachments[image.number]
- outcomes[attachment.outcome] += 1
+ attachment = attachments[image.marker]
+ outcomes[f"{image.kind}_{attachment.outcome}"] += 1
if attachment.part is None:
label = _NOT_ATTACHED_LABELS[attachment.outcome]
- parts.append({"type": "text", "text": f"Image {image.number}: {label}."})
+ parts.append({"type": "text", "text": f"{image.label}: {label}."})
else:
- parts.extend([{"type": "text", "text": f"Image {image.number}:"}, attachment.part])
- # One line, not one per image: it sits outside the transcript's text budget.
- if over_limit := len(images) - len(attachments):
+ parts.extend([{"type": "text", "text": f"{image.label}:"}, attachment.part])
+ # One line each, not one per attachment: they sit outside the transcript's text budget.
+ if over_limit := len(readable) - len(attachments):
outcomes["over_limit"] = over_limit
- parts.append({"type": "text", "text": f"Images not attached, over the judge's image limit: {over_limit}."})
+ parts.append({"type": "text", "text": f"Not attached, over the judge's attachment limit: {over_limit}."})
+ if unreadable := len(images) - len(readable):
+ outcomes["unreadable"] = unreadable
+ parts.append(
+ {"type": "text", "text": f"Not attached, this judge can't read their file type: {unreadable}."}
+ )
parts.append({"type": "text", "text": user_prompt})
- logger.info("judge_images_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)
+ logger.info("judge_attachments_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)
return parts
- def _attach(self, image: _Image, provider: str, model: str, inline_budget: int) -> _Attachment:
- transport = image_transport(provider, model, image.mime_type)
+ def _attach(self, image: _Media, provider: str, model: str, inline_budget: int) -> _Attachment:
+ transport = attachment_transport(provider, model, image.mime_type)
if transport is None:
return _Attachment(outcome="unavailable")
- provider_limit = MAX_IMAGE_BYTES.get(provider)
+ provider_limit = MAX_PDF_BYTES if image.kind == "file" else MAX_IMAGE_BYTES.get(provider)
if image.pointer is None:
- if len(image.base64_data or "") * 3 // 4 > min(inline_budget, provider_limit or inline_budget):
+ encoded = image.base64_data or ""
+ if len(encoded) * 3 // 4 - encoded[-2:].count("=") > min(inline_budget, provider_limit or inline_budget):
return _Attachment(outcome="too_large")
try:
data = base64.b64decode(image.base64_data or "", validate=True)
except ValueError: # binascii.Error, or a non-ASCII string
return _Attachment(outcome="unavailable")
return _Attachment(
- outcome="inline", part=ImagePart(mime_type=image.mime_type, data=data), inline_bytes=len(data)
+ outcome="inline", part=AttachmentPart(mime_type=image.mime_type, data=data), inline_bytes=len(data)
)
if self._storage_failed:
return _Attachment(outcome="unavailable")
@@ -156,7 +190,7 @@ class JudgeImages:
return _Attachment(outcome="unavailable")
def _attach_blob(
- self, image: _Image, pointer: BlobPointer, transport: Literal["url", "inline"], max_bytes: int | None
+ self, image: _Media, pointer: BlobPointer, transport: Literal["url", "inline"], max_bytes: int | None
) -> _Attachment:
stored_size = 0
if max_bytes is not None:
@@ -171,20 +205,38 @@ class JudgeImages:
url = pointer.presigned_url(self._team_id, PRESIGNED_URL_SECONDS)
if url is None:
return _Attachment(outcome="unavailable")
- return _Attachment(outcome="link", part=ImagePart(mime_type=image.mime_type, url=url))
+ return _Attachment(outcome="link", part=AttachmentPart(mime_type=image.mime_type, url=url))
# ponytail: sequential reads, a thread pool if traces with many images get slow
data = pointer.read(self._team_id)
if data is None:
return _Attachment(outcome="unavailable")
- part = ImagePart(mime_type=image.mime_type, data=data)
+ part = AttachmentPart(mime_type=image.mime_type, data=data)
return _Attachment(outcome="inline", part=part, inline_bytes=stored_size)
+def _kind(modality: str, mime_type: str | None) -> Literal["image", "file", "audio"] | None:
+ if modality == "image" and mime_type in IMAGE_MIME_TYPES:
+ return "image"
+ if modality == "audio" and mime_type in AUDIO_FORMATS:
+ return "audio"
+ if mime_type == PDF_MIME_TYPE:
+ return "file"
+ return None
+
+
+_MIME_ALIASES = {
+ "image/jpg": "image/jpeg",
+ "audio/mp3": "audio/mpeg",
+ "audio/x-wav": "audio/wav",
+ "audio/wave": "audio/wav",
+}
+
+
def _normalize_mime_type(mime_type: str | None) -> str | None:
if mime_type is None:
return None
normalized = mime_type.strip().lower()
- return "image/jpeg" if normalized == "image/jpg" else normalized
+ return _MIME_ALIASES.get(normalized, normalized)
@lru_cache(maxsize=4096)
@@ -193,7 +245,7 @@ def _project_uuid(team_id: int) -> str | None:
return str(project_uuid) if project_uuid else None
-def judge_sees_images(evaluation: dict[str, Any]) -> bool:
+def judge_sees_attachments(evaluation: dict[str, Any]) -> bool:
if evaluation["evaluation_config"].get("include_media") is False:
return False
# Local evaluation keeps the check off the network. An inconclusive result means off.
@@ -204,7 +256,7 @@ def judge_sees_images(evaluation: dict[str, Any]) -> bool:
return False
return bool(
posthoganalytics.feature_enabled(
- JUDGE_IMAGES_FLAG,
+ JUDGE_ATTACHMENTS_FLAG,
project_uuid,
groups={"project": project_uuid},
group_properties={"project": {"id": str(evaluation["team_id"]), "uuid": project_uuid}},
@@ -216,15 +268,19 @@ def judge_sees_images(evaluation: dict[str, Any]) -> bool:
return False
-def build_image_prompt(
+def build_attachment_prompt(
evaluation: dict[str, Any], plain_prompt: str | None, render: Callable[[MediaMarker], str | None]
-) -> ImagePrompt | None:
- """The marked prompt and its images, or None when this judge run gets the plain prompt only.
+) -> AttachmentPrompt | None:
+ """The marked prompt and its attachments, or None when this judge run gets the plain prompt only.
`render` builds the same transcript as the plain prompt, with media blocks rendered by the marker it gets.
"""
- if not judge_sees_images(evaluation):
+ if not judge_sees_attachments(evaluation):
return None
- images = JudgeImages(evaluation["team_id"])
- marked = render(images.marker)
- return ImagePrompt(text=marked, images=images) if marked is not None and marked != plain_prompt else None
+ attachments = JudgeAttachments(evaluation["team_id"])
+ marked = render(attachments.marker)
+ return (
+ AttachmentPrompt(text=marked, attachments=attachments)
+ if marked is not None and marked != plain_prompt
+ else None
+ )
diff --git a/posthog/temporal/ai_observability/run_session_evaluation.py b/posthog/temporal/ai_observability/run_session_evaluation.py
index 8b683d743139..57c2600e2b87 100644
--- a/posthog/temporal/ai_observability/run_session_evaluation.py
+++ b/posthog/temporal/ai_observability/run_session_evaluation.py
@@ -42,7 +42,7 @@ from posthog.temporal.ai_observability.evaluation_payload import (
should_skip_for_payload,
)
from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult, build_skipped_evaluation_result
-from posthog.temporal.ai_observability.judge_images import build_image_prompt
+from posthog.temporal.ai_observability.judge_attachments import build_attachment_prompt
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages
from posthog.temporal.ai_observability.run_trace_evaluation import TRACE_EVENTS_LOOKBACK
from posthog.temporal.common.utils import close_db_connections
@@ -493,12 +493,12 @@ def execute_session_llm_judge_activity(inputs: ExecuteSessionEvaluationInputs) -
traces = outcome.traces
transcript = format_session_for_judge(traces)
- image_prompt = build_image_prompt(
+ attachment_prompt = build_attachment_prompt(
evaluation, transcript, lambda media_marker: format_session_for_judge(traces, media_marker)
)
- if transcript is None and image_prompt is not None:
+ if transcript is None and attachment_prompt is not None:
# Inline base64 can push the plain transcript over the limit when the marked one still fits.
- transcript = image_prompt.unnumbered_text()
+ transcript = attachment_prompt.unnumbered_text()
if transcript is None:
return build_session_skip_result(allows_na, "session_too_long_to_judge", output_type=output_type)
@@ -510,7 +510,7 @@ def execute_session_llm_judge_activity(inputs: ExecuteSessionEvaluationInputs) -
user_prompt=transcript,
allows_na=allows_na,
backfill=inputs.backfill_id is not None,
- image_prompt=image_prompt,
+ attachment_prompt=attachment_prompt,
)
diff --git a/posthog/temporal/ai_observability/run_trace_evaluation.py b/posthog/temporal/ai_observability/run_trace_evaluation.py
index 3e3b71439dee..b204541ad9d0 100644
--- a/posthog/temporal/ai_observability/run_trace_evaluation.py
+++ b/posthog/temporal/ai_observability/run_trace_evaluation.py
@@ -63,7 +63,7 @@ from posthog.temporal.ai_observability.evaluation_workflow_activities import (
emit_internal_telemetry_activity,
fetch_evaluation_activity,
)
-from posthog.temporal.ai_observability.judge_images import build_image_prompt
+from posthog.temporal.ai_observability.judge_attachments import build_attachment_prompt
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages
from posthog.temporal.ai_observability.metrics import increment_emit_event_outcome, increment_errors
from posthog.temporal.ai_observability.run_evaluation import (
@@ -708,7 +708,7 @@ def execute_trace_llm_judge_activity(inputs: ExecuteTraceEvaluationInputs) -> Ev
user_prompt=user_prompt,
allows_na=allows_na,
backfill=inputs.backfill_id is not None,
- image_prompt=build_image_prompt(
+ attachment_prompt=build_attachment_prompt(
evaluation, user_prompt, lambda media_marker: format_trace_for_judge(trace, media_marker)
),
)
diff --git a/posthog/temporal/ai_observability/test_judge_attachments.py b/posthog/temporal/ai_observability/test_judge_attachments.py
new file mode 100644
index 000000000000..d6d9b6d7d151
--- /dev/null
+++ b/posthog/temporal/ai_observability/test_judge_attachments.py
@@ -0,0 +1,319 @@
+import base64
+
+import pytest
+from unittest.mock import patch
+
+from posthog.storage.object_storage import ObjectStorageError
+from posthog.temporal.ai_observability.judge_attachments import MAX_ATTACHMENTS, JudgeAttachments
+
+from products.ai_observability.backend.llm.attachments import MAX_IMAGE_BYTES, MAX_PDF_BYTES, AttachmentPart
+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, parse_media_block
+
+OBJECT_STORAGE = "products.ai_observability.backend.ai_blobs.object_storage"
+
+
+def _pointer(digit: str, mime: str = "image%2Fjpeg") -> MediaBlock:
+ modality = "audio" if mime.startswith("audio") else "image"
+ return MediaBlock(modality=modality, url=f"phaiblob://v1/sha256/{digit * 64}?mime={mime}&size=63357")
+
+
+def _inline(digit: str, length: int = 4) -> MediaBlock:
+ return MediaBlock(modality="image", mime_type="image/png", base64_data=digit * length)
+
+
+class TestJudgeAttachmentsMarkers:
+ def test_numbers_each_kind_and_reuses_the_number_for_a_repeat(self):
+ images = JudgeAttachments(team_id=7)
+ inline = MediaBlock(modality="image", mime_type="image/png", base64_data="aGk=")
+
+ pdf = _pointer("b", mime="application%2Fpdf")
+
+ markers = [images.marker(media) for media in (_pointer("a"), pdf, inline, _pointer("a"))]
+
+ assert markers == ["[image 1]", "[file 1]", "[image 2]", "[image 1]"]
+
+ @pytest.mark.parametrize(
+ ("media", "marker"),
+ [
+ pytest.param(MediaBlock(modality="audio", base64_data="aGk="), "[audio]", id="not_an_image"),
+ pytest.param(_pointer("a", mime="image%2Fsvg%2Bxml"), "[image]", id="unsupported_format"),
+ pytest.param(
+ MediaBlock(modality="image", mime_type="image/png", url="https://example.com/a.png"),
+ "[image: https://example.com/a.png]",
+ id="customer_link",
+ ),
+ pytest.param(
+ parse_media_block({"file_data": {"mime_type": "image/png", "file_uri": "gs://bucket/a.png"}}),
+ "[image]",
+ id="gemini_cloud_storage_uri",
+ ),
+ ],
+ )
+ def test_images_it_cannot_attach_keep_an_unnumbered_marker(self, media: MediaBlock, marker: str):
+ assert JudgeAttachments(team_id=7).marker(media) == marker
+
+
+class TestJudgeAttachmentsContent:
+ def test_an_image_referenced_again_at_the_end_stays_attached(self):
+ images = JudgeAttachments(team_id=7)
+ markers = [images.marker(_inline(f"{i:02}")) for i in range(MAX_ATTACHMENTS + 1)]
+ # The output repeats the first input image after every newer one.
+ content = images.content(" ".join([*markers, markers[0]]), "openai", "gpt-5.5")
+
+ assert {"type": "text", "text": "Image 1:"} in content
+ assert {"type": "text", "text": "Image 2:"} not in content
+ assert {"type": "text", "text": "Not attached, over the judge's attachment limit: 1."} in content
+
+ def test_attaches_the_newest_images_when_over_the_limit(self):
+ images = JudgeAttachments(team_id=7)
+ prompt = " ".join(images.marker(_inline(f"{i:02}")) for i in range(MAX_ATTACHMENTS + 1))
+
+ content = images.content(prompt, "openai", "gpt-5.5")
+
+ assert content[1] == {"type": "text", "text": "Image 2:"}
+ assert content[-4:-1] == [
+ {"type": "text", "text": f"Image {MAX_ATTACHMENTS + 1}:"},
+ AttachmentPart(mime_type="image/png", data=base64.b64decode(f"{MAX_ATTACHMENTS:02}" * 4)),
+ {"type": "text", "text": "Not attached, over the judge's attachment limit: 1."},
+ ]
+
+ @pytest.mark.parametrize(
+ ("provider", "model", "mime", "expected_label", "expected_part"),
+ [
+ pytest.param(
+ "openai",
+ "gpt-5.5",
+ "image%2FJPG",
+ "Image 1:",
+ AttachmentPart(mime_type="image/jpeg", url="https://signed"),
+ id="link",
+ ),
+ pytest.param(
+ "together_ai",
+ "any",
+ "image%2FJPG",
+ "Image 1:",
+ AttachmentPart(mime_type="image/jpeg", data=b"hi"),
+ id="inline",
+ ),
+ pytest.param(
+ "gemini",
+ "gemini-2.0-flash",
+ "image%2FJPG",
+ "Image 1:",
+ AttachmentPart(mime_type="image/jpeg", data=b"hi"),
+ id="gemini_2_0",
+ ),
+ pytest.param(
+ "anthropic",
+ "claude-haiku-4-5",
+ "application%2Fpdf",
+ "File 1:",
+ AttachmentPart(mime_type="application/pdf", url="https://signed"),
+ id="pdf_link",
+ ),
+ pytest.param(
+ "openai",
+ "gpt-5.5",
+ "application%2Fpdf",
+ "File 1:",
+ AttachmentPart(mime_type="application/pdf", data=b"hi"),
+ id="pdf_inline_on_openai",
+ ),
+ pytest.param(
+ "gemini",
+ "gemini-3-flash",
+ "image%2Fgif",
+ "Not attached, this judge can't read their file type: 1.",
+ None,
+ id="format_rejected_by_provider",
+ ),
+ pytest.param(
+ "gemini",
+ "gemini-2.5-flash",
+ "audio%2Fwav",
+ "Audio 1:",
+ AttachmentPart(mime_type="audio/wav", data=b"hi"),
+ id="audio_inline_on_gemini",
+ ),
+ pytest.param(
+ "openai",
+ "gpt-4o-audio-preview",
+ "audio%2Fmp3",
+ "Audio 1:",
+ AttachmentPart(mime_type="audio/mpeg", data=b"hi"),
+ id="audio_on_an_openai_audio_model",
+ ),
+ pytest.param(
+ "openai",
+ "gpt-5.5",
+ "audio%2Fwav",
+ "Not attached, this judge can't read their file type: 1.",
+ None,
+ id="audio_on_an_openai_model_without_audio_input",
+ ),
+ pytest.param(
+ "openai",
+ "gpt-4o-audio-preview",
+ "image%2Fpng",
+ "Not attached, this judge can't read their file type: 1.",
+ None,
+ id="image_on_an_openai_audio_model",
+ ),
+ pytest.param(
+ "openrouter",
+ "any",
+ "application%2Fpdf",
+ "Not attached, this judge can't read their file type: 1.",
+ None,
+ id="pdf_unsupported_provider",
+ ),
+ ],
+ )
+ def test_attaches_each_attachment_under_its_label(
+ self, provider: str, model: str, mime: str, expected_label: str, expected_part: AttachmentPart | None
+ ):
+ images = JudgeAttachments(team_id=7)
+ prompt = f"Input: user: {images.marker(_pointer('a', mime=mime))} What does this chart show?"
+
+ with (
+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed") as presign,
+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 2}),
+ patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"hi") as read,
+ ):
+ content = images.content(prompt, provider, model)
+
+ assert content == [
+ {"type": "text", "text": "The transcript refers to these attachments as [image N], [file N] or [audio N]."},
+ {"type": "text", "text": expected_label},
+ *([expected_part] if expected_part else []),
+ {"type": "text", "text": prompt},
+ ]
+ if expected_part:
+ storage_call = presign if expected_part.url else read
+ assert storage_call.call_args.args[0] == f"aio/7/sha256/{'a' * 64}"
+
+ @pytest.mark.parametrize(
+ ("provider", "model", "media", "storage_error"),
+ [
+ pytest.param("openai", "gpt-5.5", _pointer("a"), None, id="blob_missing_for_a_link"),
+ pytest.param("together_ai", "any", _pointer("a"), None, id="blob_missing_for_inline"),
+ pytest.param(
+ "together_ai", "any", _pointer("a"), ObjectStorageError("head_object failed"), id="storage_error"
+ ),
+ pytest.param(
+ "gemini",
+ "gemini-3-flash",
+ MediaBlock(modality="image", mime_type="image/png", base64_data="aGk"),
+ None,
+ id="invalid_inline_base64",
+ ),
+ pytest.param(
+ "gemini",
+ "gemini-3-flash",
+ MediaBlock(modality="image", mime_type="image/png", base64_data="aGk\u00e9"),
+ None,
+ id="non_ascii_inline_base64",
+ ),
+ ],
+ )
+ def test_reports_an_image_it_could_not_attach(
+ self, provider: str, model: str, media: MediaBlock, storage_error: Exception | None
+ ):
+ images = JudgeAttachments(team_id=7)
+ prompt = f"user: {images.marker(media)}"
+
+ with (
+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value=None),
+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value=None, side_effect=storage_error),
+ ):
+ content = images.content(prompt, provider, model)
+
+ assert content[1] == {"type": "text", "text": "Image 1: could not be attached."}
+
+ def test_stops_calling_storage_after_a_storage_error(self):
+ images = JudgeAttachments(team_id=7)
+ prompt = f"user: {images.marker(_pointer('a'))} {images.marker(_pointer('b'))}"
+
+ with patch(f"{OBJECT_STORAGE}.head_object_strict", side_effect=ObjectStorageError("timed out")) as head:
+ content = images.content(prompt, "together_ai", "any")
+
+ assert head.call_count == 1
+ assert {"type": "text", "text": "Image 1: could not be attached."} in content
+ assert {"type": "text", "text": "Image 2: could not be attached."} in content
+
+ @pytest.mark.parametrize(
+ ("provider", "media", "too_large", "reads"),
+ [
+ pytest.param("openai", _pointer, set(), 0, id="links_have_no_byte_cap"),
+ pytest.param("anthropic", _pointer, {1, 2}, 0, id="provider_limit_on_links"),
+ # The pointers claim 63 KB, so this also checks that the budget uses the stored size.
+ pytest.param("together_ai", _pointer, {1}, 1, id="stored_size_for_inline_providers"),
+ pytest.param(
+ "openai", lambda digit: _inline(digit, length=13_400_000), {1}, 0, id="inline_images_on_link_providers"
+ ),
+ ],
+ )
+ def test_caps_image_bytes(self, provider: str, media, too_large: set[int], reads: int):
+ images = JudgeAttachments(team_id=7)
+ prompt = f"user: {images.marker(media('a'))} {images.marker(media('b'))}"
+
+ with (
+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed"),
+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 10_000_000}),
+ patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"x" * 10_000_000) as read,
+ ):
+ content = images.content(prompt, provider, "any")
+
+ assert {
+ number
+ for number in (1, 2)
+ if {"type": "text", "text": f"Image {number}: not attached, too large."} in content
+ } == too_large
+ # An image over the budget is never downloaded.
+ assert read.call_count == reads
+
+ def test_caps_pdf_bytes_on_links(self):
+ images = JudgeAttachments(team_id=7)
+ prompt = f"user: {images.marker(_pointer('a', mime='application%2Fpdf'))}"
+
+ with (
+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed") as presign,
+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": MAX_PDF_BYTES + 1}),
+ ):
+ # Gemini has no image byte limit, so only the PDF cap can reject this one.
+ content = images.content(prompt, "gemini", "gemini-3-flash")
+
+ assert {"type": "text", "text": "File 1: not attached, too large."} in content
+ presign.assert_not_called()
+
+ def test_attachments_the_judge_cannot_read_take_no_slot(self):
+ images = JudgeAttachments(team_id=7)
+ image = images.marker(_inline("aa"))
+ pdfs = " ".join(
+ images.marker(MediaBlock(modality="file", url=f"phaiblob://v1/sha256/{i:064x}?mime=application%2Fpdf"))
+ for i in range(MAX_ATTACHMENTS)
+ )
+
+ content = images.content(f"user: {image} {pdfs}", "openrouter", "openai/gpt-5-mini")
+
+ assert content[1:4] == [
+ {"type": "text", "text": "Image 1:"},
+ AttachmentPart(mime_type="image/png", data=base64.b64decode("aa" * 4)),
+ {"type": "text", "text": f"Not attached, this judge can't read their file type: {MAX_ATTACHMENTS}."},
+ ]
+
+ def test_an_inline_image_exactly_at_the_provider_limit_is_attached(self):
+ images = JudgeAttachments(team_id=7)
+ encoded = base64.b64encode(b"x" * MAX_IMAGE_BYTES["anthropic"]).decode()
+ prompt = f"user: {images.marker(MediaBlock(modality='image', mime_type='image/png', base64_data=encoded))}"
+
+ content = images.content(prompt, "anthropic", "claude-haiku-4-5")
+
+ assert content[1] == {"type": "text", "text": "Image 1:"}
+
+ def test_skips_images_whose_marker_was_sampled_out_of_the_prompt(self):
+ images = JudgeAttachments(team_id=7)
+ images.marker(_pointer("a"))
+
+ assert images.content("user: hello", "openai", "gpt-5.5") == "user: hello"
diff --git a/posthog/temporal/ai_observability/test_judge_images.py b/posthog/temporal/ai_observability/test_judge_images.py
deleted file mode 100644
index 3d755b001e4f..000000000000
--- a/posthog/temporal/ai_observability/test_judge_images.py
+++ /dev/null
@@ -1,192 +0,0 @@
-import base64
-
-import pytest
-from unittest.mock import patch
-
-from posthog.storage.object_storage import ObjectStorageError
-from posthog.temporal.ai_observability.judge_images import MAX_IMAGES, JudgeImages
-
-from products.ai_observability.backend.llm.images import ImagePart
-from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, parse_media_block
-
-OBJECT_STORAGE = "products.ai_observability.backend.ai_blobs.object_storage"
-
-
-def _pointer(digit: str, mime: str = "image%2Fjpeg") -> MediaBlock:
- return MediaBlock(modality="image", url=f"phaiblob://v1/sha256/{digit * 64}?mime={mime}&size=63357")
-
-
-def _inline(digit: str, length: int = 4) -> MediaBlock:
- return MediaBlock(modality="image", mime_type="image/png", base64_data=digit * length)
-
-
-class TestJudgeImagesMarkers:
- def test_numbers_images_and_reuses_the_number_for_a_repeat(self):
- images = JudgeImages(team_id=7)
- inline = MediaBlock(modality="image", mime_type="image/png", base64_data="aGk=")
-
- markers = [images.marker(media) for media in (_pointer("a"), inline, _pointer("a"))]
-
- assert markers == ["[image 1]", "[image 2]", "[image 1]"]
-
- @pytest.mark.parametrize(
- ("media", "marker"),
- [
- pytest.param(MediaBlock(modality="audio", base64_data="aGk="), "[audio]", id="not_an_image"),
- pytest.param(_pointer("a", mime="image%2Fsvg%2Bxml"), "[image]", id="unsupported_format"),
- pytest.param(
- MediaBlock(modality="image", mime_type="image/png", url="https://example.com/a.png"),
- "[image: https://example.com/a.png]",
- id="customer_link",
- ),
- pytest.param(
- parse_media_block({"file_data": {"mime_type": "image/png", "file_uri": "gs://bucket/a.png"}}),
- "[image]",
- id="gemini_cloud_storage_uri",
- ),
- ],
- )
- def test_images_it_cannot_attach_keep_an_unnumbered_marker(self, media: MediaBlock, marker: str):
- assert JudgeImages(team_id=7).marker(media) == marker
-
-
-class TestJudgeImagesContent:
- def test_an_image_referenced_again_at_the_end_stays_attached(self):
- images = JudgeImages(team_id=7)
- markers = [images.marker(_inline(f"{i:02}")) for i in range(MAX_IMAGES + 1)]
- # The output repeats the first input image after every newer one.
- content = images.content(" ".join([*markers, markers[0]]), "openai", "gpt-5.5")
-
- assert {"type": "text", "text": "Image 1:"} in content
- assert {"type": "text", "text": "Image 2:"} not in content
- assert {"type": "text", "text": "Images not attached, over the judge's image limit: 1."} in content
-
- def test_attaches_the_newest_images_when_over_the_limit(self):
- images = JudgeImages(team_id=7)
- prompt = " ".join(images.marker(_inline(f"{i:02}")) for i in range(MAX_IMAGES + 1))
-
- content = images.content(prompt, "openai", "gpt-5.5")
-
- assert content[1] == {"type": "text", "text": "Image 2:"}
- assert content[-4:-1] == [
- {"type": "text", "text": f"Image {MAX_IMAGES + 1}:"},
- ImagePart(mime_type="image/png", data=base64.b64decode(f"{MAX_IMAGES:02}" * 4)),
- {"type": "text", "text": "Images not attached, over the judge's image limit: 1."},
- ]
-
- @pytest.mark.parametrize(
- ("provider", "model", "expected_image"),
- [
- pytest.param("openai", "gpt-5.5", ImagePart(mime_type="image/jpeg", url="https://signed"), id="link"),
- pytest.param("together_ai", "any", ImagePart(mime_type="image/jpeg", data=b"hi"), id="inline"),
- pytest.param("gemini", "gemini-2.0-flash", ImagePart(mime_type="image/jpeg", data=b"hi"), id="gemini_2_0"),
- ],
- )
- def test_attaches_each_image_under_its_label(self, provider: str, model: str, expected_image: ImagePart):
- images = JudgeImages(team_id=7)
- prompt = f"Input: user: {images.marker(_pointer('a', mime='image%2FJPG'))} What does this chart show?"
-
- with (
- patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed") as presign,
- patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 2}),
- patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"hi") as read,
- ):
- content = images.content(prompt, provider, model)
-
- assert content == [
- {"type": "text", "text": "The transcript refers to these images as [image N]."},
- {"type": "text", "text": "Image 1:"},
- expected_image,
- {"type": "text", "text": prompt},
- ]
- storage_call = presign if expected_image.url else read
- assert storage_call.call_args.args[0] == f"aio/7/sha256/{'a' * 64}"
-
- @pytest.mark.parametrize(
- ("provider", "model", "media", "storage_error"),
- [
- pytest.param("openai", "gpt-5.5", _pointer("a"), None, id="blob_missing_for_a_link"),
- pytest.param("together_ai", "any", _pointer("a"), None, id="blob_missing_for_inline"),
- pytest.param(
- "together_ai", "any", _pointer("a"), ObjectStorageError("head_object failed"), id="storage_error"
- ),
- pytest.param(
- "gemini", "gemini-3-flash", _pointer("a", mime="image%2Fgif"), None, id="format_rejected_by_provider"
- ),
- pytest.param(
- "gemini",
- "gemini-3-flash",
- MediaBlock(modality="image", mime_type="image/png", base64_data="aGk"),
- None,
- id="invalid_inline_base64",
- ),
- pytest.param(
- "gemini",
- "gemini-3-flash",
- MediaBlock(modality="image", mime_type="image/png", base64_data="aGk\u00e9"),
- None,
- id="non_ascii_inline_base64",
- ),
- ],
- )
- def test_reports_an_image_it_could_not_attach(
- self, provider: str, model: str, media: MediaBlock, storage_error: Exception | None
- ):
- images = JudgeImages(team_id=7)
- prompt = f"user: {images.marker(media)}"
-
- with (
- patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value=None),
- patch(f"{OBJECT_STORAGE}.head_object_strict", return_value=None, side_effect=storage_error),
- ):
- content = images.content(prompt, provider, model)
-
- assert content[1] == {"type": "text", "text": "Image 1: could not be attached."}
-
- def test_stops_calling_storage_after_a_storage_error(self):
- images = JudgeImages(team_id=7)
- prompt = f"user: {images.marker(_pointer('a'))} {images.marker(_pointer('b'))}"
-
- with patch(f"{OBJECT_STORAGE}.head_object_strict", side_effect=ObjectStorageError("timed out")) as head:
- content = images.content(prompt, "together_ai", "any")
-
- assert head.call_count == 1
- assert {"type": "text", "text": "Image 1: could not be attached."} in content
- assert {"type": "text", "text": "Image 2: could not be attached."} in content
-
- @pytest.mark.parametrize(
- ("provider", "media", "too_large", "reads"),
- [
- pytest.param("openai", _pointer, set(), 0, id="links_have_no_byte_cap"),
- pytest.param("anthropic", _pointer, {1, 2}, 0, id="provider_limit_on_links"),
- # The pointers claim 63 KB, so this also checks that the budget uses the stored size.
- pytest.param("together_ai", _pointer, {1}, 1, id="stored_size_for_inline_providers"),
- pytest.param(
- "openai", lambda digit: _inline(digit, length=13_400_000), {1}, 0, id="inline_images_on_link_providers"
- ),
- ],
- )
- def test_caps_image_bytes(self, provider: str, media, too_large: set[int], reads: int):
- images = JudgeImages(team_id=7)
- prompt = f"user: {images.marker(media('a'))} {images.marker(media('b'))}"
-
- with (
- patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed"),
- patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 10_000_000}),
- patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"x" * 10_000_000) as read,
- ):
- content = images.content(prompt, provider, "any")
-
- assert {
- number
- for number in (1, 2)
- if {"type": "text", "text": f"Image {number}: not attached, too large."} in content
- } == too_large
- # An image over the budget is never downloaded.
- assert read.call_count == reads
-
- def test_skips_images_whose_marker_was_sampled_out_of_the_prompt(self):
- images = JudgeImages(team_id=7)
- images.marker(_pointer("a"))
-
- assert images.content("user: hello", "openai", "gpt-5.5") == "user: hello"
diff --git a/posthog/temporal/ai_observability/test_message_utils.py b/posthog/temporal/ai_observability/test_message_utils.py
index bcdc89602c96..78de7915e0f1 100644
--- a/posthog/temporal/ai_observability/test_message_utils.py
+++ b/posthog/temporal/ai_observability/test_message_utils.py
@@ -4,7 +4,7 @@ import json
import pytest
-from posthog.temporal.ai_observability.judge_images import JudgeImages
+from posthog.temporal.ai_observability.judge_attachments import JudgeAttachments
from posthog.temporal.ai_observability.message_utils import (
extract_text_from_messages,
format_tool_definitions,
@@ -113,20 +113,89 @@ class TestExtractTextFromMessages:
pytest.param(
{"type": "image", "data": "aGk=", "mime_type": "image/png"}, "[image 1]", id="gemini_interactions"
),
- pytest.param({"inlineData": {"mimeType": "audio/wav", "data": "aGk="}}, "[audio]", id="gemini_audio"),
+ pytest.param({"inlineData": {"mimeType": "audio/wav", "data": "aGk="}}, "[audio 1]", id="gemini_audio"),
+ pytest.param(
+ {"type": "input_audio", "input_audio": {"data": "aGk=", "format": "wav"}},
+ "[audio 1]",
+ id="openai_audio_wav",
+ ),
+ pytest.param(
+ {
+ "type": "input_audio",
+ "input_audio": {"data": f"phaiblob://v1/sha256/{'b' * 64}?mime=audio%2Fmp3", "format": "mp3"},
+ },
+ "[audio 1]",
+ id="openai_audio_offloaded",
+ ),
pytest.param({"type": "input_audio", "input_audio": {"data": "aGk="}}, "[audio]", id="openai_audio"),
pytest.param({"type": "document", "source": {"type": "base64", "data": "aGk="}}, "[file]", id="file"),
+ pytest.param(
+ {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "aGk="}},
+ "[file 1]",
+ id="anthropic_pdf",
+ ),
+ pytest.param(
+ {"type": "file", "file": {"filename": "a.pdf", "file_data": "data:application/pdf;base64,aGk="}},
+ "[file 1]",
+ id="openai_chat_pdf",
+ ),
+ pytest.param(
+ {"type": "input_file", "filename": "a.pdf", "file_data": "data:application/pdf;base64,aGk="},
+ "[file 1]",
+ id="openai_responses_pdf",
+ ),
+ pytest.param(
+ {"type": "file", "file": {"file_id": "file-abc"}}, "[file]", id="openai_uploaded_file_without_bytes"
+ ),
+ pytest.param(
+ {"type": "file", "mediaType": "image/png", "file": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng"},
+ "[image 1]",
+ id="vercel_image",
+ ),
+ pytest.param(
+ {"type": "file", "mediaType": "application/pdf", "data": "data:application/pdf;base64,aGk="},
+ "[file 1]",
+ id="vercel_pdf",
+ ),
+ pytest.param(
+ {"type": "document", "mime_type": "application/pdf", "data": "aGk="}, "[file 1]", id="gemini_pdf"
+ ),
+ pytest.param(
+ {"type": "file", "mediaType": "application/pdf", "data": "data:application/octet-stream;base64,aGk="},
+ "[file 1]",
+ id="declared_pdf_beats_a_generic_data_uri",
+ ),
+ pytest.param(
+ {"type": "file", "mediaType": "APPLICATION/OCTET-STREAM", "data": "data:application/pdf;base64,aGk="},
+ "[file 1]",
+ id="data_uri_pdf_beats_a_generic_declared_type",
+ ),
+ pytest.param(
+ {"type": "file", "file": {"filename": "report.PDF", "file_data": "aGk="}},
+ "[file 1]",
+ id="pdf_known_only_by_its_filename",
+ ),
+ pytest.param(
+ {"type": "image", "uri": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng"},
+ "[image 1]",
+ id="otel_style_image_uri",
+ ),
+ pytest.param(
+ {"type": "file", "mediaType": "image/png", "file": "raw files not supported"},
+ "[image]",
+ id="vercel_sdk_placeholder",
+ ),
],
)
- def test_judge_images_replace_media_blocks_with_markers(self, block, marker):
+ def test_judge_attachments_replace_media_blocks_with_markers(self, block, marker):
messages = [{"role": "user", "content": [block, {"type": "text", "text": "What does this chart show?"}]}]
assert (
- extract_text_from_messages(messages, JudgeImages(team_id=7).marker)
+ extract_text_from_messages(messages, JudgeAttachments(team_id=7).marker)
== f"user: {marker} What does this chart show?"
)
- def test_judge_images_number_otel_parts_and_tool_result_images(self):
+ def test_judge_attachments_number_otel_parts_and_tool_result_images(self):
pointer = f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1"
image = {"type": "image_url", "image_url": {"url": pointer}}
messages = [
@@ -143,12 +212,12 @@ class TestExtractTextFromMessages:
]
assert (
- extract_text_from_messages(messages, JudgeImages(team_id=7).marker)
+ extract_text_from_messages(messages, JudgeAttachments(team_id=7).marker)
== "user: [image 1]\nuser: [image 1]\ntool[c1]: [image 1]"
)
assert extract_text_from_messages(messages).startswith("user: [image]\n")
- def test_media_blocks_keep_their_text_without_judge_images(self):
+ def test_media_blocks_keep_their_text_without_judge_attachments(self):
block = {"type": "file", "file": {"filename": "report.pdf"}}
assert "report.pdf" in extract_text_from_messages([{"role": "user", "content": [block]}])
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 5ae5ee7572aa..5282732d3575 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -28,13 +28,14 @@ from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker
from posthog.api.capture import CaptureInternalError
from posthog.constants import AvailableFeature
from posthog.models import Organization, OrganizationMembership, Team, User
-from posthog.temporal.ai_observability.judge_images import ImagePrompt, JudgeImages
+from posthog.temporal.ai_observability.judge_attachments import AttachmentPrompt, JudgeAttachments
from posthog.temporal.ai_observability.sentiment.extraction import truncate_to_head_tail
from posthog.temporal.ai_observability.sentiment.schema import SentimentResult
from posthog.temporal.common.errors import NonReportableError
from posthog.temporal.common.posthog_client import is_expected_activity_failure
from products.access_control.backend.models.access_control import AccessControl
+from products.ai_observability.backend.llm.attachments import AttachmentPart
from products.ai_observability.backend.llm.errors import (
RESPONSE_LIMIT_MESSAGE,
AuthenticationError,
@@ -51,7 +52,6 @@ from products.ai_observability.backend.llm.errors import (
StructuredOutputParseError,
UnsupportedModelError,
)
-from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.models.evaluation_config import EvaluationConfig
from products.ai_observability.backend.models.evaluation_directories import EvaluationDirectory
from products.ai_observability.backend.models.evaluations import Evaluation
@@ -1169,9 +1169,9 @@ def _adapter_rejection(cause: Exception | None) -> ProviderRequestRejectedError:
def test_judge_falls_back_to_the_plain_prompt_only_when_an_attached_image_is_rejected(
error: Exception, presigned_url: str | None, retries: bool
) -> None:
- images = JudgeImages(team_id=1)
+ images = JudgeAttachments(team_id=1)
pointer = MediaBlock(modality="image", url=f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1")
- image_prompt = ImagePrompt(text=f"user: {images.marker(pointer)} Polite?", images=images)
+ attachment_prompt = AttachmentPrompt(text=f"user: {images.marker(pointer)} Polite?", attachments=images)
with (
patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
@@ -1198,7 +1198,7 @@ def test_judge_falls_back_to_the_plain_prompt_only_when_an_attached_image_is_rej
system_prompt="",
user_prompt="user: Polite?",
allows_na=False,
- image_prompt=image_prompt,
+ attachment_prompt=attachment_prompt,
)
if retries:
@@ -1624,7 +1624,7 @@ class TestRunEvaluationWorkflow:
with (
patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client") as mock_client_class,
patch(
- "posthog.temporal.ai_observability.judge_images.posthoganalytics.feature_enabled",
+ "posthog.temporal.ai_observability.judge_attachments.posthoganalytics.feature_enabled",
return_value=flag_enabled,
) as feature_enabled,
patch(
@@ -1644,7 +1644,7 @@ class TestRunEvaluationWorkflow:
# Team ids repeat across regions, so the flag is checked against the project uuid.
assert feature_enabled.call_args.kwargs["groups"] == {"project": str(team.uuid)}
if attaches_image:
- assert ImagePart(mime_type="image/jpeg", url="https://signed") in content
+ assert AttachmentPart(mime_type="image/jpeg", url="https://signed") in content
assert "user: [image 1] What does this chart show?" in content[-1]["text"]
assert "assistant: [image 2] Sales rose in March." in content[-1]["text"]
else:
@@ -4738,17 +4738,17 @@ class TestJudgePromptAssembly:
with (
patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as judge,
patch(
- "posthog.temporal.ai_observability.judge_images.posthoganalytics.feature_enabled",
+ "posthog.temporal.ai_observability.judge_attachments.posthoganalytics.feature_enabled",
return_value=True,
),
- patch("posthog.temporal.ai_observability.judge_images._project_uuid", return_value="uuid"),
+ patch("posthog.temporal.ai_observability.judge_attachments._project_uuid", return_value="uuid"),
):
execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))
- image_prompt = judge.call_args.kwargs["image_prompt"]
- assert image_prompt is not None
+ attachment_prompt = judge.call_args.kwargs["attachment_prompt"]
+ assert attachment_prompt is not None
# Decision judges and the text-only retry use this prompt, and they send no images.
- assert judge.call_args.kwargs["user_prompt"] == image_prompt.unnumbered_text()
+ assert judge.call_args.kwargs["user_prompt"] == attachment_prompt.unnumbered_text()
assert "[image 1]" not in judge.call_args.kwargs["user_prompt"]
@pytest.mark.parametrize(
diff --git a/posthog/temporal/ai_observability/test_run_session_evaluation.py b/posthog/temporal/ai_observability/test_run_session_evaluation.py
index 70a4b7cb3608..0ab649083086 100644
--- a/posthog/temporal/ai_observability/test_run_session_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_session_evaluation.py
@@ -20,7 +20,7 @@ from posthog.temporal.ai_observability.evaluation_llm_judge import (
get_output_type_config,
)
from posthog.temporal.ai_observability.evaluation_payload import payload_budget_bytes
-from posthog.temporal.ai_observability.judge_images import JudgeImages
+from posthog.temporal.ai_observability.judge_attachments import JudgeAttachments
from posthog.temporal.ai_observability.run_session_evaluation import (
_SESSION_EVENT_COUNT_SQL,
AI_EVENTS_RETENTION_DAYS,
@@ -46,6 +46,7 @@ from posthog.temporal.ai_observability.run_trace_evaluation import (
)
from products.ai_observability.backend.hog import compile_ai_observability_hog
+from products.ai_observability.backend.llm.attachments import AttachmentPart
from products.cdp.backend.validation import compile_hog
FROZEN_NOW = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC)
@@ -148,7 +149,7 @@ class TestFormatSessionForJudge:
for trace in traces:
trace.events[0].properties["$ai_input"] = [{"role": "user", "content": [image]}]
- rendered = format_session_for_judge(traces, JudgeImages(team_id=1).marker)
+ rendered = format_session_for_judge(traces, JudgeAttachments(team_id=1).marker)
assert rendered is not None
assert rendered.count("[image 1]") == 2
@@ -676,6 +677,39 @@ class TestExecuteSessionActivities:
# The whole point of the truncation-as-skip choice: never grade a partial transcript.
mock_call_llm_judge.assert_not_called()
+ def test_judge_gets_the_session_images(self) -> None:
+ trace = _trace("trace-1", cost=0, latency=0)
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}}
+ trace.events[0].properties["$ai_input"] = [{"role": "user", "content": [image]}]
+ with (
+ patch(
+ "posthog.temporal.ai_observability.run_session_evaluation.fetch_session_for_evaluation",
+ return_value=SessionFetchOutcome(traces=[trace], skip_reason=None, event_count=1),
+ ),
+ patch("posthog.temporal.ai_observability.judge_attachments.judge_sees_attachments", return_value=True),
+ patch("posthog.temporal.ai_observability.run_session_evaluation.call_llm_judge") as mock_call_llm_judge,
+ ):
+ execute_session_llm_judge_activity(
+ ExecuteSessionEvaluationInputs(
+ evaluation={
+ "team_id": 1,
+ "evaluation_type": "llm_judge",
+ "evaluation_config": {"prompt": "Did the user accomplish their goal?"},
+ "output_type": "boolean",
+ "output_config": {},
+ },
+ team_id=1,
+ session_id="s-1",
+ window_start=FROZEN_NOW.isoformat(),
+ )
+ )
+
+ attachment_prompt = mock_call_llm_judge.call_args.kwargs["attachment_prompt"]
+ assert "[image 1]" in attachment_prompt.text
+ assert attachment_prompt.attachments.content(attachment_prompt.text, "openai", "gpt-5.5")[2] == AttachmentPart(
+ mime_type="image/png", data=b"hi"
+ )
+
def test_judge_skips_without_judging_when_the_rendered_transcript_would_overflow(self):
# The fetch itself succeeds (session_truncated wasn't raised there), but the session has
# enough traces that format_session_for_judge's char budget can't fit all of them. This is
diff --git a/posthog/temporal/ai_observability/test_run_trace_evaluation.py b/posthog/temporal/ai_observability/test_run_trace_evaluation.py
index 3b94246a8d4e..4267993810d6 100644
--- a/posthog/temporal/ai_observability/test_run_trace_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_trace_evaluation.py
@@ -23,7 +23,7 @@ from posthog.hogql_queries.ai.trace_query_runner import TraceQueryRunner
from posthog.models import Organization, PropertyDefinition, Team, User
from posthog.models.ai_events.test_util import bulk_create_ai_events
from posthog.models.event.util import bulk_create_events
-from posthog.temporal.ai_observability.judge_images import JudgeImages
+from posthog.temporal.ai_observability.judge_attachments import JudgeAttachments
from posthog.temporal.ai_observability.run_session_evaluation import fetch_session_for_evaluation
from posthog.temporal.ai_observability.trace_summarization.fetch_and_format import _fetch_and_format_trace
@@ -31,6 +31,7 @@ from products.access_control.backend.facade.contracts import PropertyAccessLevel
from products.access_control.backend.models.access_control import AccessControl
from products.access_control.backend.models.property_access_control import PropertyAccessControl
from products.access_control.backend.property_access_control import restriction_cache_scope
+from products.ai_observability.backend.llm.attachments import AttachmentPart
from products.ai_observability.backend.models.evaluation_config import EvaluationConfig
from products.ai_observability.backend.models.evaluations import Evaluation
from products.ai_observability.backend.models.provider_keys import LLMProviderKey
@@ -194,7 +195,7 @@ class TestFormatTraceForJudge:
)
plain = format_trace_for_judge(trace)
- marked = format_trace_for_judge(trace, JudgeImages(team_id=1).marker)
+ marked = format_trace_for_judge(trace, JudgeAttachments(team_id=1).marker)
assert "[image 1]" not in plain
# The message and the tool result repeat one image, and the OTel part adds a second.
@@ -919,6 +920,37 @@ class TestExecuteTraceLLMJudgeActivity:
assert result["verdict"] is True
assert result["reasoning"] == "Resolved both questions"
+ @pytest.mark.django_db(transaction=True)
+ def test_sends_the_trace_images_to_a_judge_with_images(
+ self, setup_data: dict[str, Any], active_key_config: None
+ ) -> None:
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}}
+ trace = create_trace([], inputState=[{"role": "user", "content": [image, {"type": "text", "text": "Chart?"}]}])
+ with (
+ patch(
+ "posthog.temporal.ai_observability.run_trace_evaluation.fetch_trace_for_evaluation",
+ return_value=TraceFetchOutcome(trace=trace, skip_reason=None, event_count=1),
+ ),
+ patch("posthog.temporal.ai_observability.judge_attachments.judge_sees_attachments", return_value=True),
+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client") as mock_client_class,
+ ):
+ mock_client = mock_client_class.return_value
+ mock_client.complete.return_value.parsed = BooleanEvalResult(verdict=True, reasoning="ok")
+ mock_client.complete.return_value.usage = MagicMock(input_tokens=1, output_tokens=1, total_tokens=2)
+
+ execute_trace_llm_judge_activity(
+ ExecuteTraceEvaluationInputs(
+ evaluation=evaluation_dict(setup_data),
+ team_id=setup_data["team"].id,
+ trace_id="trace-123",
+ window_start=FROZEN_NOW.isoformat(),
+ )
+ )
+
+ content = mock_client.complete.call_args[0][0].messages[0]["content"]
+ assert AttachmentPart(mime_type="image/png", data=b"hi") in content
+ assert "[image 1]" in content[-1]["text"]
+
@pytest.mark.parametrize(
"trace_state",
[
diff --git a/products/ai_observability/backend/llm/attachments.py b/products/ai_observability/backend/llm/attachments.py
new file mode 100644
index 000000000000..6a43a21a2794
--- /dev/null
+++ b/products/ai_observability/backend/llm/attachments.py
@@ -0,0 +1,85 @@
+import base64
+from collections.abc import Callable
+from dataclasses import field
+from typing import Any, Literal
+
+from posthog.dataclasses import frozen
+
+IMAGE_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/webp", "image/gif"})
+PDF_MIME_TYPE = "application/pdf"
+# The formats OpenAI's chat API takes as input audio, which Gemini reads too.
+AUDIO_FORMATS = {"audio/wav": "wav", "audio/mpeg": "mp3"}
+# Providers whose image blocks take a link that they download themselves.
+_URL_PROVIDERS = frozenset({"openai", "azure_openai", "anthropic", "openrouter", "gemini"})
+# Providers that read PDFs natively. OpenRouter is left out: it parses PDFs for other models with a paid OCR engine.
+_PDF_PROVIDERS = frozenset({"openai", "anthropic", "gemini"})
+# Raw bytes per image. Anthropic allows 10 MB base64-encoded, about 7.5 MB raw, so keep a margin under it.
+MAX_IMAGE_BYTES = {"anthropic": 7_000_000}
+# Raw bytes per PDF for every provider. Each page costs about as many tokens as an image, and links have no other cap.
+MAX_PDF_BYTES = 10_000_000
+
+
+@frozen
+class AttachmentPart:
+ """An image, PDF or audio clip in a message's content list. Each adapter converts it to its provider's block.
+
+ Set `url` for a link the provider downloads, or `data` for inline bytes.
+ """
+
+ mime_type: str
+ url: str | None = None
+ data: bytes | None = field(default=None, repr=False)
+
+ def base64(self) -> str:
+ return base64.b64encode(self.data or b"").decode()
+
+ def data_uri(self) -> str:
+ return f"data:{self.mime_type};base64,{self.base64()}"
+
+ @property
+ def is_pdf(self) -> bool:
+ return self.mime_type == PDF_MIME_TYPE
+
+ @property
+ def audio_format(self) -> str | None:
+ return AUDIO_FORMATS.get(self.mime_type)
+
+
+def attachment_transport(provider: str, model: str, mime_type: str) -> Literal["url", "inline"] | None:
+ """How a provider accepts an attachment of this type: a link, inline base64, or not at all."""
+ openai_audio_model = provider == "openai" and "audio" in model
+ if mime_type in AUDIO_FORMATS:
+ # OpenAI takes audio only as inline base64, and only on its audio models, so audio always goes inline.
+ return "inline" if provider == "gemini" or openai_audio_model else None
+ if openai_audio_model:
+ # OpenAI's audio models reject images and PDFs, and the rejection would drop the audio too.
+ return None
+ is_pdf = mime_type == PDF_MIME_TYPE
+ if not is_pdf and mime_type not in IMAGE_MIME_TYPES:
+ return None
+ if is_pdf and provider not in _PDF_PROVIDERS:
+ return None
+ if provider == "gemini":
+ if mime_type == "image/gif":
+ return None
+ # The Gemini 2.0 family can't fetch external links.
+ return "inline" if model.startswith("gemini-2.0") else "url"
+ if is_pdf and provider == "openai":
+ # The chat API takes a PDF only as inline file data.
+ return "inline"
+ return "url" if provider in _URL_PROVIDERS else "inline"
+
+
+def convert_attachment_parts(
+ messages: list[dict[str, Any]], convert: Callable[[AttachmentPart], Any]
+) -> list[dict[str, Any]]:
+ """Replace each AttachmentPart in the messages' content lists with the provider block that `convert` returns."""
+ return [
+ {
+ **message,
+ "content": [convert(part) if isinstance(part, AttachmentPart) else part for part in message["content"]],
+ }
+ if isinstance(message.get("content"), list)
+ else message
+ for message in messages
+ ]
diff --git a/products/ai_observability/backend/llm/images.py b/products/ai_observability/backend/llm/images.py
deleted file mode 100644
index 9d54236ca84d..000000000000
--- a/products/ai_observability/backend/llm/images.py
+++ /dev/null
@@ -1,52 +0,0 @@
-import base64
-from collections.abc import Callable
-from dataclasses import field
-from typing import Any, Literal
-
-from posthog.dataclasses import frozen
-
-IMAGE_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/webp", "image/gif"})
-# Providers whose image blocks take a link that they download themselves.
-_URL_PROVIDERS = frozenset({"openai", "azure_openai", "anthropic", "openrouter", "gemini"})
-# Raw bytes per image. Anthropic allows 10 MB base64-encoded, about 7.5 MB raw, so keep a margin under it.
-MAX_IMAGE_BYTES = {"anthropic": 7_000_000}
-
-
-@frozen
-class ImagePart:
- """An image in a message's content list. Each adapter converts it to its provider's image block.
-
- Set `url` for a link the provider downloads, or `data` for inline bytes.
- """
-
- mime_type: str
- url: str | None = None
- data: bytes | None = field(default=None, repr=False)
-
- def base64(self) -> str:
- return base64.b64encode(self.data or b"").decode()
-
- def data_uri(self) -> str:
- return f"data:{self.mime_type};base64,{self.base64()}"
-
-
-def image_transport(provider: str, model: str, mime_type: str) -> Literal["url", "inline"] | None:
- """How a provider accepts an image of this type: a link, inline base64, or not at all."""
- if mime_type not in IMAGE_MIME_TYPES:
- return None
- if provider == "gemini":
- if mime_type == "image/gif":
- return None
- # The Gemini 2.0 family can't fetch external links.
- return "inline" if model.startswith("gemini-2.0") else "url"
- return "url" if provider in _URL_PROVIDERS else "inline"
-
-
-def convert_image_parts(messages: list[dict[str, Any]], convert: Callable[[ImagePart], Any]) -> list[dict[str, Any]]:
- """Replace each ImagePart in the messages' content lists with the provider block that `convert` returns."""
- return [
- {**message, "content": [convert(part) if isinstance(part, ImagePart) else part for part in message["content"]]}
- if isinstance(message.get("content"), list)
- else message
- for message in messages
- ]
diff --git a/products/ai_observability/backend/llm/providers/anthropic.py b/products/ai_observability/backend/llm/providers/anthropic.py
index 3cd96efa68a5..115b302767e8 100644
--- a/products/ai_observability/backend/llm/providers/anthropic.py
+++ b/products/ai_observability/backend/llm/providers/anthropic.py
@@ -17,6 +17,7 @@ from posthoganalytics.ai.anthropic.anthropic import WrappedMessages
from posthoganalytics.client import Client as PostHogClient
from pydantic import BaseModel
+from products.ai_observability.backend.llm.attachments import AttachmentPart, convert_attachment_parts
from products.ai_observability.backend.llm.errors import (
AuthenticationError,
ContextWindowExceededError,
@@ -34,7 +35,6 @@ from products.ai_observability.backend.llm.errors import (
is_output_limit_error_message,
stream_error_chunk,
)
-from products.ai_observability.backend.llm.images import ImagePart, convert_image_parts
from products.ai_observability.backend.llm.providers.azure_openai import is_allowed_azure_endpoint
from products.ai_observability.backend.llm.types import (
AnalyticsContext,
@@ -230,7 +230,7 @@ class AnthropicAdapter:
effective_api_key = api_key or self._get_default_api_key()
client = self._create_client(effective_api_key, analytics)
- messages: Any = convert_image_parts(request.messages, _anthropic_image_block)
+ messages: Any = convert_attachment_parts(request.messages, _anthropic_attachment_block)
system_prompt = request.system or ""
use_structured = request.response_format and issubclass(request.response_format, BaseModel)
@@ -360,7 +360,7 @@ class AnthropicAdapter:
tools = self._convert_tools(request.tools) if request.tools else None
system = request.system or ""
- messages = convert_image_parts(request.messages, _anthropic_image_block)
+ messages = convert_attachment_parts(request.messages, _anthropic_attachment_block)
# Handle cache control for supported models
system_prompt: list[TextBlockParam] = []
@@ -641,7 +641,8 @@ class AnthropicAdapter:
return prepared_messages
-def _anthropic_image_block(part: ImagePart) -> dict[str, Any]:
+def _anthropic_attachment_block(part: AttachmentPart) -> dict[str, Any]:
+ block_type = "document" if part.is_pdf else "image"
if part.url is not None:
- return {"type": "image", "source": {"type": "url", "url": part.url}}
- return {"type": "image", "source": {"type": "base64", "media_type": part.mime_type, "data": part.base64()}}
+ return {"type": block_type, "source": {"type": "url", "url": part.url}}
+ return {"type": block_type, "source": {"type": "base64", "media_type": part.mime_type, "data": part.base64()}}
diff --git a/products/ai_observability/backend/llm/providers/openai.py b/products/ai_observability/backend/llm/providers/openai.py
index b69b662732c0..90e6316ae8b8 100644
--- a/products/ai_observability/backend/llm/providers/openai.py
+++ b/products/ai_observability/backend/llm/providers/openai.py
@@ -20,6 +20,7 @@ from posthoganalytics.ai.openai import (
)
from pydantic import BaseModel, ValidationError
+from products.ai_observability.backend.llm.attachments import AttachmentPart, convert_attachment_parts
from products.ai_observability.backend.llm.errors import (
AuthenticationError,
ContentFilteredError,
@@ -36,7 +37,6 @@ from products.ai_observability.backend.llm.errors import (
is_output_limit_error_message,
stream_error_chunk,
)
-from products.ai_observability.backend.llm.images import ImagePart, convert_image_parts
from products.ai_observability.backend.llm.types import (
AnalyticsContext,
CompletionRequest,
@@ -311,7 +311,7 @@ You must respond with valid JSON that matches this schema:
Return ONLY the JSON object, no other text or markdown formatting."""
enhanced_messages = [{"role": "system", "content": enhanced_system}]
- enhanced_messages.extend(convert_image_parts(request.messages, _openai_image_block))
+ enhanced_messages.extend(convert_attachment_parts(request.messages, _openai_attachment_block))
create_response = client.chat.completions.create(
model=request.model,
@@ -520,7 +520,7 @@ Return ONLY the JSON object, no other text or markdown formatting."""
messages = []
if request.system:
messages.append({"role": "system", "content": request.system})
- messages.extend(convert_image_parts(request.messages, _openai_image_block))
+ messages.extend(convert_attachment_parts(request.messages, _openai_attachment_block))
return messages
def _build_analytics_kwargs(self, analytics: AnalyticsContext, client) -> dict:
@@ -573,5 +573,10 @@ Return ONLY the JSON object, no other text or markdown formatting."""
return result
-def _openai_image_block(part: ImagePart) -> dict[str, Any]:
+def _openai_attachment_block(part: AttachmentPart) -> dict[str, Any]:
+ if part.audio_format is not None:
+ return {"type": "input_audio", "input_audio": {"data": part.base64(), "format": part.audio_format}}
+ if part.is_pdf:
+ # The chat API takes a PDF only as inline data, and requires a file name with it.
+ return {"type": "file", "file": {"filename": "attachment.pdf", "file_data": part.data_uri()}}
return {"type": "image_url", "image_url": {"url": part.url or part.data_uri()}}
diff --git a/products/ai_observability/backend/llm/providers/test/test_anthropic.py b/products/ai_observability/backend/llm/providers/test/test_anthropic.py
index 56d0a632a23c..3a5fd718195f 100644
--- a/products/ai_observability/backend/llm/providers/test/test_anthropic.py
+++ b/products/ai_observability/backend/llm/providers/test/test_anthropic.py
@@ -5,6 +5,7 @@ import httpx
import anthropic
from parameterized import parameterized
+from products.ai_observability.backend.llm.attachments import AttachmentPart
from products.ai_observability.backend.llm.errors import (
ContextWindowExceededError,
ModelNotFoundError,
@@ -12,7 +13,6 @@ from products.ai_observability.backend.llm.errors import (
OutputTokenLimitError,
ProviderConfigurationError,
)
-from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.llm.providers.anthropic import (
AnthropicAdapter,
AnthropicConfig,
@@ -173,22 +173,30 @@ class TestAnthropicStreamOutputSchema:
assert "output_config" not in self._stream_kwargs(None)
-class TestAnthropicImageParts:
+class TestAnthropicAttachmentParts:
@parameterized.expand(
[
(
"link",
- ImagePart(mime_type="image/png", url="https://example.com/a.png"),
+ AttachmentPart(mime_type="image/png", url="https://example.com/a.png"),
{"type": "url", "url": "https://example.com/a.png"},
),
(
"inline",
- ImagePart(mime_type="image/png", data=b"hi"),
+ AttachmentPart(mime_type="image/png", data=b"hi"),
{"type": "base64", "media_type": "image/png", "data": "aGk="},
),
+ (
+ "pdf_link",
+ AttachmentPart(mime_type="application/pdf", url="https://example.com/a.pdf"),
+ {"type": "url", "url": "https://example.com/a.pdf"},
+ "document",
+ ),
]
)
- def test_image_parts_are_sent_as_image_blocks(self, _name: str, image: ImagePart, expected_source: dict):
+ def test_attachment_parts_are_sent_as_blocks(
+ self, _name: str, image: AttachmentPart, expected_source: dict, expected_type: str = "image"
+ ):
request = CompletionRequest(
model="claude-haiku-4-5",
messages=[{"role": "user", "content": [image, {"type": "text", "text": "hi"}]}],
@@ -202,7 +210,7 @@ class TestAnthropicImageParts:
AnthropicAdapter().complete(request, api_key="sk-ant-test", analytics=AnalyticsContext(capture=False))
assert mock_client.messages.create.call_args.kwargs["messages"][0]["content"] == [
- {"type": "image", "source": expected_source},
+ {"type": expected_type, "source": expected_source},
{"type": "text", "text": "hi"},
]
diff --git a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
index 93cce2da4829..7280577a605a 100644
--- a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
+++ b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
@@ -12,6 +12,7 @@ from posthoganalytics.ai.openai import (
)
from pydantic import BaseModel, ValidationError, model_validator
+from products.ai_observability.backend.llm.attachments import AttachmentPart
from products.ai_observability.backend.llm.errors import (
ContentFilteredError,
ContextWindowExceededError,
@@ -19,7 +20,6 @@ from products.ai_observability.backend.llm.errors import (
QuotaExceededError,
StructuredOutputParseError,
)
-from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter, OpenAIConfig
from products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema
@@ -178,23 +178,40 @@ class TestOpenAIStreamOutputSchema:
assert "response_format" not in self._stream_kwargs(None)
-class TestOpenAIImageParts:
+class TestOpenAIAttachmentParts:
@pytest.mark.parametrize(
- ("image", "expected_url"),
+ ("part", "expected_block"),
[
pytest.param(
- ImagePart(mime_type="image/png", url="https://example.com/a.png"),
- "https://example.com/a.png",
+ AttachmentPart(mime_type="image/png", url="https://example.com/a.png"),
+ {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},
id="link",
),
- pytest.param(ImagePart(mime_type="image/png", data=b"hi"), "data:image/png;base64,aGk=", id="inline"),
+ pytest.param(
+ AttachmentPart(mime_type="image/png", data=b"hi"),
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}},
+ id="inline",
+ ),
+ pytest.param(
+ AttachmentPart(mime_type="application/pdf", data=b"hi"),
+ {
+ "type": "file",
+ "file": {"filename": "attachment.pdf", "file_data": "data:application/pdf;base64,aGk="},
+ },
+ id="pdf",
+ ),
+ pytest.param(
+ AttachmentPart(mime_type="audio/mpeg", data=b"hi"),
+ {"type": "input_audio", "input_audio": {"data": "aGk=", "format": "mp3"}},
+ id="audio",
+ ),
],
)
@pytest.mark.parametrize("json_fallback", [False, True], ids=["plain", "json_fallback"])
- def test_image_parts_are_sent_as_image_url_blocks(self, image: ImagePart, expected_url: str, json_fallback: bool):
+ def test_attachment_parts_are_sent_as_blocks(self, part: AttachmentPart, expected_block: dict, json_fallback: bool):
request = CompletionRequest(
model="gpt-4.1",
- messages=[{"role": "user", "content": [image, {"type": "text", "text": "hi"}]}],
+ messages=[{"role": "user", "content": [part, {"type": "text", "text": "hi"}]}],
provider="openai",
response_format=_Verdict if json_fallback else None,
)
@@ -209,7 +226,7 @@ class TestOpenAIImageParts:
sent = mock_client.chat.completions.create.call_args.kwargs["messages"]
assert next(message for message in sent if message["role"] == "user")["content"] == [
- {"type": "image_url", "image_url": {"url": expected_url}},
+ expected_block,
{"type": "text", "text": "hi"},
]
diff --git a/products/ai_observability/backend/providers/formatters/gemini_formatter.py b/products/ai_observability/backend/providers/formatters/gemini_formatter.py
index 209e5f27aaa8..7d1f857281ca 100644
--- a/products/ai_observability/backend/providers/formatters/gemini_formatter.py
+++ b/products/ai_observability/backend/providers/formatters/gemini_formatter.py
@@ -4,7 +4,7 @@ from typing import Any, cast
from google.genai.types import Blob, Content, ContentListUnion, FileData, FunctionCall, FunctionResponse, Part
-from products.ai_observability.backend.llm.images import ImagePart
+from products.ai_observability.backend.llm.attachments import AttachmentPart
from products.ai_observability.backend.providers.formatters.anthropic_typeguards import (
is_base64_image_param,
is_image_block_param,
@@ -85,7 +85,7 @@ def _tool_response_payload(content: Any, *, is_error: bool = False) -> dict[str,
return value if isinstance(value, dict) else {"result": value}
-def _gemini_image_part(image: ImagePart) -> Part:
+def _gemini_attachment_part(image: AttachmentPart) -> Part:
if image.url is not None:
return Part(file_data=FileData(file_uri=image.url, mime_type=image.mime_type))
return Part(inline_data=Blob(data=image.data or b"", mime_type=image.mime_type))
@@ -103,8 +103,8 @@ def convert_anthropic_messages_to_gemini(messages: list[dict[str, Any]]) -> Cont
parts.append(Part(text=message["content"]))
elif isinstance(message["content"], list):
for block in message["content"]:
- if isinstance(block, ImagePart):
- parts.append(_gemini_image_part(block))
+ if isinstance(block, AttachmentPart):
+ parts.append(_gemini_attachment_part(block))
elif is_text_block_param(block):
parts.append(Part(text=block["text"]))
elif is_image_block_param(block):
diff --git a/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py b/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py
index 676458ec055f..c63ad13fb4e6 100644
--- a/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py
+++ b/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py
@@ -5,7 +5,7 @@ import pytest
from google.genai.types import Blob, Content, FileData, FunctionCall, FunctionResponse, Part
from parameterized import parameterized
-from products.ai_observability.backend.llm.images import ImagePart
+from products.ai_observability.backend.llm.attachments import AttachmentPart
from products.ai_observability.backend.providers.formatters.gemini_formatter import (
MessageConversionError,
convert_anthropic_messages_to_gemini,
@@ -97,17 +97,17 @@ class TestConvertAnthropicMessagesToGemini:
[
(
"link",
- ImagePart(mime_type="image/png", url="https://example.com/a.png"),
+ AttachmentPart(mime_type="image/png", url="https://example.com/a.png"),
Part(file_data=FileData(file_uri="https://example.com/a.png", mime_type="image/png")),
),
(
"inline",
- ImagePart(mime_type="image/png", data=b"hi"),
+ AttachmentPart(mime_type="image/png", data=b"hi"),
Part(inline_data=Blob(data=b"hi", mime_type="image/png")),
),
]
)
- def test_image_part_becomes_gemini_part(self, _name: str, image: ImagePart, expected: Part):
+ def test_image_part_becomes_gemini_part(self, _name: str, image: AttachmentPart, expected: Part):
contents = _convert([{"role": "user", "content": [image]}])
assert _part(contents[0], 0) == expected
diff --git a/products/ai_observability/backend/text_repr/formatters/media_blocks.py b/products/ai_observability/backend/text_repr/formatters/media_blocks.py
index 210d9eb50ab0..6605338ca21e 100644
--- a/products/ai_observability/backend/text_repr/formatters/media_blocks.py
+++ b/products/ai_observability/backend/text_repr/formatters/media_blocks.py
@@ -1,5 +1,6 @@
import re
from collections.abc import Callable
+from dataclasses import replace
from posthog.dataclasses import frozen
@@ -8,6 +9,9 @@ _AUDIO_TYPES = frozenset({"audio", "input_audio"})
_FILE_TYPES = frozenset({"document", "file", "input_file"})
_GEMINI_DATA_KEYS = ("inline_data", "inlineData", "file_data", "fileData")
_OTEL_TYPES = frozenset({"blob", "uri"})
+_GENERIC_MIME_TYPE = "application/octet-stream"
+_MIME_KEYS = ("mediaType", "mimeType", "mime_type", "media_type")
+_FILE_SOURCE_KEYS = ("file_data", "file_url", "data", "uri", "url")
_LINK_SCHEMES = ("https://", "http://", "phaiblob://")
# Without full capture the SDKs replace the bytes with a placeholder such as `[base64 image redacted]`.
# Same pattern as the frontend's `isRedactedMediaSentinel`.
@@ -62,9 +66,9 @@ def parse_media_block(block: dict) -> MediaBlock | None:
if block_type in _IMAGE_TYPES:
return _image_block(block)
if block_type in _AUDIO_TYPES:
- return MediaBlock(modality="audio")
+ return _audio_block(block)
if block_type in _FILE_TYPES:
- return MediaBlock(modality="file")
+ return _file_block(block)
return None
@@ -97,7 +101,47 @@ def _image_block(block: dict) -> MediaBlock:
)
# The Gemini Interactions API puts `data` and `mime_type` on the block itself.
mime_type = block.get("mime_type") or block.get("mimeType")
- return _with_source("image", mime_type if isinstance(mime_type, str) else None, image_url or block.get("data"))
+ return _with_source(
+ "image", mime_type if isinstance(mime_type, str) else None, image_url or block.get("data") or block.get("uri")
+ )
+
+
+def _audio_block(block: dict) -> MediaBlock:
+ # OpenAI chat names the format instead of a MIME type: `{"input_audio": {"data": ..., "format": "wav"}}`.
+ audio = block.get("input_audio")
+ if isinstance(audio, dict):
+ audio_format = audio.get("format")
+ mime_type = f"audio/{audio_format}" if isinstance(audio_format, str) and audio_format else None
+ return _with_source("audio", mime_type, audio.get("data"))
+ source = block.get("source")
+ fields = source if isinstance(source, dict) else block
+ mime_type = next((fields[key] for key in _MIME_KEYS if isinstance(fields.get(key), str)), None)
+ value = next((fields[key] for key in _FILE_SOURCE_KEYS if fields.get(key)), None)
+ return _with_source("audio", mime_type, value)
+
+
+def _file_block(block: dict) -> MediaBlock:
+ source = block.get("source")
+ if isinstance(source, dict):
+ # Anthropic document.
+ mime_type, value = source.get("media_type"), source.get("data") or source.get("url")
+ filename = None
+ else:
+ file = block.get("file")
+ # OpenAI chat nests the file, and the Responses API, Gemini and the Vercel AI SDK put it on the block.
+ # Vercel records `{"type": "file", "mediaType": ..., "file": <data>}`.
+ fields = file if isinstance(file, dict) else block
+ mime_type = next((block[key] for key in _MIME_KEYS if isinstance(block.get(key), str)), None)
+ value = next((fields[key] for key in _FILE_SOURCE_KEYS if fields.get(key)), None)
+ if value is None and isinstance(file, str):
+ value = file
+ filename = fields.get("filename")
+ media = _with_source("file", mime_type if isinstance(mime_type, str) else None, value)
+ if media.mime_type is None and isinstance(filename, str) and filename.lower().endswith(".pdf"):
+ media = replace(media, mime_type="application/pdf")
+ # The Vercel AI SDK sends images, audio and video as file parts too.
+ modality = media.mime_type.split("/")[0] if media.mime_type else ""
+ return replace(media, modality=modality) if modality in ("image", "audio", "video") else media
def _with_source(modality: str, mime_type: str | None, value: object) -> MediaBlock:
@@ -107,9 +151,14 @@ def _with_source(modality: str, mime_type: str | None, value: object) -> MediaBl
# Unwrap one envelope only. Its payload is bytes or a wrapped pointer, never another data URI,
# and unwrapping nested prefixes would copy the payload once per level.
header, value = value[len("data:") :].split(";base64,", 1)
- mime_type = header or mime_type
+ # A declared type beats a generic one in the data URI, as in the trace view.
+ if header and (mime_type is None or mime_type.strip().lower() == _GENERIC_MIME_TYPE):
+ mime_type = header
if _REDACTED_RE.match(value) or value.startswith("data:"):
return MediaBlock(modality=modality, mime_type=mime_type)
if value.startswith(_LINK_SCHEMES):
return MediaBlock(modality=modality, mime_type=mime_type, url=value)
+ # Base64 never holds a space, so a value with one is an SDK placeholder such as `raw files not supported`.
+ if " " in value:
+ return MediaBlock(modality=modality, mime_type=mime_type)
return MediaBlock(modality=modality, mime_type=mime_type, base64_data=value)
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx
index 3dee7c4c5f35..4d9018869803 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx
@@ -41,7 +41,9 @@ export function EvaluationModelPanel(): JSX.Element | null {
data-attr="llma-evaluation-include-attachments"
/>
<p className="text-muted text-sm">
- The judge sees the images in what it evaluates. Other file types are not supported yet.
+ The judge sees the images, PDFs and audio in what it evaluates. PDFs need an OpenAI, Anthropic
+ or Gemini judge, and audio a Gemini judge or an OpenAI audio model. Other file types are not
+ supported yet.
</p>
</LemonField.Pure>
)}