Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): let llm judges see images (#124879)

Bernat Torresb30fcfa45fcf
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 12b24733e73b..9ce5903e0806 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -42,6 +42,9 @@ If the assembled fallback session still exceeds 500,000 characters, the evaluati
Generation evaluations extract message text without the per-message character cutoff.
They reserve space for the complete output, then sample input and tool definitions into the remaining space within the 150,000-character limit.
If the output and section labels alone exceed that limit, the run is skipped with reason `generation_output_too_long` without calling the judge.
+An exception applies when the judge sees images (the `llm-analytics-judge-images` flag is on and the evaluation does not turn attachments off).
+Then each image becomes an `[image N]` marker instead of its base64 or link, so an output with large inline images can fit and the run continues.
+The image labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 images, plus one summary line for the rest.
Generation evaluation and LLM tagger user prompts repair UTF-16 surrogate sequences after sampling.
Lone surrogates become replacement characters (`�`), while valid pairs become their Unicode character.
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx
index cdbbcc37ab83..e5f8b02c96e6 100644
--- a/frontend/src/lib/constants.tsx
+++ b/frontend/src/lib/constants.tsx
@@ -359,8 +359,8 @@ export const FEATURE_FLAGS = {
INBOX_MCP_USED: 'inbox-mcp-used', // owner: #team-self-driving, hides the MCP banner after confirmed Inbox tool use
INBOX_MODEL_SORT: 'inbox-model-sort', // owner: #team-self-driving, gates the ranking model sort options and the score tag on inbox cards
INBOX_REPORT_FEEDBACK_BANG: 'inbox-report-feedback-bang', // owner: #team-self-driving, gates the party hog that pops on a report thumbs-up in the inbox and on the Today home page
- INBOX_SOURCE_FILTER: 'inbox-source-filter', // owner: #team-self-driving, gates the Source filter on the inbox report list
INBOX_SLACK_NOTIFICATIONS: 'inbox-slack-notifications', // owner: #team-self-driving, gates the Slack notifications config card in the inbox
+ INBOX_SOURCE_FILTER: 'inbox-source-filter', // owner: #team-self-driving, gates the Source filter on the inbox report list
INBOX_TIME_WINDOW: 'inbox-time-window', // owner: #team-self-driving, gates the created-in window filter on the inbox report list
INSIGHT_ALERTS_EMPTY_STATE_EXPERIMENT: 'insight-alerts-empty-state',
INSIGHT_DATA_CHECKS: 'insight-data-checks', // owner: @sampennington #team-product-analytics, checks the events behind an insight and shows the result on the insight page
@@ -374,6 +374,7 @@ export const FEATURE_FLAGS = {
LLM_ANALYTICS_COMMUNITY_SKILLS: 'llm-analytics-community-skills', // owner: #team-ai-observability
LLM_ANALYTICS_DATASETS: 'llm-analytics-datasets', // owner: #team-ai-observability #team-posthog-ai
LLM_ANALYTICS_EVAL_SETTLING_STRATEGY: 'llm-analytics-eval-settling-strategy', // owner: #team-ai-observability
+ LLM_ANALYTICS_JUDGE_IMAGES: 'llm-analytics-judge-images', // owner: #team-ai-observability
LLM_ANALYTICS_TAGS: 'llm-analytics-tags', // owner: #team-ai-observability
LLM_ANALYTICS_TRACE_NAVIGATION: 'llm-analytics-trace-navigation', // owner: #team-ai-observability
LLM_OBSERVABILITY_TRACE_SEARCH: 'llm-observability-trace-search', // owner: #team-ai-observability
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 312383fed0c0..2b474a65e0b4 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -1,6 +1,6 @@
import json
import math
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from datetime import timedelta
from typing import Annotated, Any, Literal
@@ -13,6 +13,7 @@ from temporalio.common import RetryPolicy
from temporalio.exceptions import ApplicationError
from posthog.dataclasses import frozen
+from posthog.exceptions_capture import capture_exception
from posthog.llm.system_one import (
MAX_SCORE_LEVELS,
ChoiceAnswer,
@@ -40,6 +41,7 @@ from posthog.temporal.ai_observability.evaluation_types import (
build_generation_not_found_result,
build_skipped_evaluation_result,
)
+from posthog.temporal.ai_observability.judge_images import ImagePrompt, build_image_prompt
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages, format_tool_definitions
from posthog.temporal.ai_observability.metrics import (
increment_errors,
@@ -78,6 +80,7 @@ from products.ai_observability.backend.llm.errors import (
UnsupportedModelError,
provider_error_detail,
)
+from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.llm.providers.openai import OPENAI_DECISIONS_BASE_URL
from products.ai_observability.backend.llm.providers.openai_decisions import OpenAIDecisionAdapter
from products.ai_observability.backend.llm.providers.openrouter import OPENROUTER_DECISIONS_BASE_URL
@@ -93,6 +96,7 @@ from products.ai_observability.backend.text_repr.formatters import (
sanitize_surrogates,
truncate_by_line_sampling,
)
+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaMarker
logger = structlog.get_logger(__name__)
@@ -442,6 +446,25 @@ def _build_unparsable_response_skip_result(
return result
+# Statuses an attached image can cause. Auth, permission and missing-model errors fail the same way without images.
+_IMAGE_REJECTION_STATUSES = frozenset({400, 413, 415, 422})
+
+
+class JudgeImagesRejectedError(Exception):
+ """Reported without the provider's message, which can quote a live presigned link."""
+
+
+def _image_rejection_status(error: Exception) -> int | None:
+ """The status of a provider rejection an image can cause, raw or wrapped by an adapter."""
+ for candidate in (error, error.__cause__):
+ if (
+ isinstance(candidate, Exception)
+ and (status := _rejected_request_status(candidate)) in _IMAGE_REJECTION_STATUSES
+ ):
+ return status
+ return None
+
+
def _rejected_request_status(error: Exception) -> int | None:
"""The 4xx status of a provider rejection that no retry can fix, or None.
@@ -558,17 +581,19 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActi
return _build_errored_trace_result(allows_na, output_type=output_type)
io = extract_event_io(event_type, properties)
- tools_raw = extract_event_tools(properties)
-
- input_data = extract_text_from_messages(io.input_raw)
- output_data = extract_text_from_messages(io.output_raw)
- tools_data = format_tool_definitions(tools_raw)
-
+ tools_data = format_tool_definitions(extract_event_tools(properties))
system_prompt = build_system_prompt(prompt, allows_na, output_type=output_type, output_config=output_config)
- output_section = f"\n\nOutput: {output_data}"
- context_budget = EVENT_TEXT_MAX_CHARS - len("Input: ") - len(output_section)
- if context_budget < 0:
+ user_prompt = _build_user_prompt(io.input_raw, io.output_raw, tools_data)
+ image_prompt = build_image_prompt(
+ evaluation,
+ user_prompt,
+ lambda media_marker: _build_user_prompt(io.input_raw, io.output_raw, tools_data, media_marker=media_marker),
+ )
+ if user_prompt is None and image_prompt is not None:
+ # Inline base64 can push the plain text over the limit when the marked text still fits.
+ user_prompt = image_prompt.unnumbered_text()
+ if user_prompt is None:
return build_skipped_evaluation_result(
output_type=output_type,
allows_na=allows_na,
@@ -576,22 +601,34 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActi
skip_reason="generation_output_too_long",
)
- sections = [input_data]
- if tools_data:
- sections.append(f"Tools available:\n{tools_data}")
- # Keep the full answer being graded outside the sampled context.
- context = truncate_by_line_sampling("\n\n".join(sections), context_budget)
- user_prompt = sanitize_surrogates(f"Input: {context}{output_section}")
-
return call_llm_judge(
evaluation=evaluation,
system_prompt=system_prompt,
user_prompt=user_prompt,
allows_na=allows_na,
backfill=inputs.backfill_id is not None,
+ image_prompt=image_prompt,
)
+def _build_user_prompt(
+ input_raw: Any, output_raw: Any, tools_data: str, *, media_marker: MediaMarker | None = None
+) -> str | None:
+ """The judge's user prompt, or None when the output alone exceeds the judge's text limit."""
+ # Render the input first, so its images get the lower numbers in reading order.
+ input_text = extract_text_from_messages(input_raw, media_marker)
+ output_section = f"\n\nOutput: {extract_text_from_messages(output_raw, media_marker)}"
+ context_budget = EVENT_TEXT_MAX_CHARS - len("Input: ") - len(output_section)
+ if context_budget < 0:
+ return None
+ sections = [input_text]
+ if tools_data:
+ sections.append(f"Tools available:\n{tools_data}")
+ # Keep the full answer being graded outside the sampled context.
+ context = truncate_by_line_sampling("\n\n".join(sections), context_budget)
+ return sanitize_surrogates(f"Input: {context}{output_section}")
+
+
def _decision_numeric_score(minimum: float, maximum: float, index: float) -> float:
last_index = MAX_SCORE_LEVELS - 1
if index == 0:
@@ -613,6 +650,7 @@ def call_llm_judge(
user_prompt: str,
allows_na: bool,
backfill: bool = False,
+ image_prompt: ImagePrompt | None = None,
) -> EvaluationActivityResult:
"""Resolve the judge model/key for `evaluation` and run a single judge completion.
@@ -821,16 +859,40 @@ def call_llm_judge(
),
)
else:
- response = client.complete(
- CompletionRequest(
- model=model,
- system=system_prompt,
- messages=[{"role": "user", "content": user_prompt}],
- provider=provider,
- response_format=response_format,
- reasoning_level=evaluation["evaluation_config"].get("reasoning_level"),
- )
+ content = image_prompt.images.content(image_prompt.text, provider, model) if image_prompt else user_prompt
+ sent_images = isinstance(content, list) and any(isinstance(part, ImagePart) for part in content)
+ request = CompletionRequest(
+ model=model,
+ system=system_prompt,
+ messages=[{"role": "user", "content": content}],
+ provider=provider,
+ response_format=response_format,
+ reasoning_level=evaluation["evaluation_config"].get("reasoning_level"),
)
+ status: int | None = None
+ try:
+ response = client.complete(request)
+ except Exception as e:
+ status = _image_rejection_status(e) if sent_images else None
+ if status is None:
+ raise
+ # Handle the rejection outside the except: the provider's error can quote a presigned link, and
+ # an active exception reaches the logs through capture_exception and chains onto a retry failure.
+ if status is not None:
+ # A model without image input, or an image link the provider could not download.
+ # Grade the plain transcript rather than skip the run.
+ context = {
+ "evaluation_id": evaluation["id"],
+ "team_id": evaluation["team_id"],
+ "provider": provider,
+ "model": model,
+ "status": status,
+ }
+ logger.warning("judge_images_rejected", **context)
+ if not is_byok:
+ # On a PostHog key this can mean broken presigned links, so keep it visible.
+ capture_exception(JudgeImagesRejectedError(f"Provider rejected judge images ({status})"), context)
+ response = client.complete(replace(request, messages=[{"role": "user", "content": user_prompt}]))
except (DecisionEndpointBlockedError, ProviderConfigurationError) as e:
increment_user_errors("endpoint_blocked", provider=provider)
return terminal_user_error_result(
diff --git a/posthog/temporal/ai_observability/judge_images.py b/posthog/temporal/ai_observability/judge_images.py
new file mode 100644
index 000000000000..121420fed86b
--- /dev/null
+++ b/posthog/temporal/ai_observability/judge_images.py
@@ -0,0 +1,230 @@
+"""Images an LLM judge sees next to the transcript that refers to them as `[image N]`."""
+
+import re
+import base64
+from collections import Counter
+from collections.abc import Callable
+from functools import lru_cache
+from typing import Any, Literal
+
+import structlog
+import posthoganalytics
+
+from posthog.dataclasses import frozen
+from posthog.models.team import Team
+from posthog.storage.object_storage import ObjectStorageError
+
+from products.ai_observability.backend.facade.blobs import BlobPointer
+from products.ai_observability.backend.llm.images import IMAGE_MIME_TYPES, MAX_IMAGE_BYTES, ImagePart, image_transport
+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, MediaMarker
+
+logger = structlog.get_logger(__name__)
+
+JUDGE_IMAGES_FLAG = "llm-analytics-judge-images"
+
+# Each image costs tokens on the customer's key, and above 20 images Anthropic tightens its per-image dimension limit.
+MAX_IMAGES = 20
+# Bytes sent inline in one judge request. Links don't count: the provider downloads them.
+MAX_INLINE_BYTES = 15_000_000
+# Covers the judge call and its retries. Sign right before the call: a link signed with temporary
+# AWS credentials stops working when the credentials expire.
+PRESIGNED_URL_SECONDS = 15 * 60
+
+Outcome = Literal["link", "inline", "too_large", "unavailable"]
+
+_NOT_ATTACHED_LABELS: dict[Outcome, str] = {
+ "too_large": "not attached, too large",
+ "unavailable": "could not be attached",
+}
+
+
+_NUMBERED_MARKER_RE = re.compile(r"\[image \d+\]")
+
+
+@frozen
+class ImagePrompt:
+ """The judge transcript with `[image N]` markers, and the images those markers refer to."""
+
+ text: str
+ images: "JudgeImages"
+
+ def unnumbered_text(self) -> str:
+ """The transcript for a judge call that sends no images, so no marker points at a missing image."""
+ return _NUMBERED_MARKER_RE.sub("[image]", self.text)
+
+
+@frozen
+class _Attachment:
+ outcome: Outcome
+ part: ImagePart | None = None
+ inline_bytes: int = 0
+
+
+@frozen
+class _Image:
+ number: int
+ mime_type: str
+ # An offloaded image has a pointer, a small one stays inline as base64.
+ pointer: BlobPointer | None
+ base64_data: str | None
+
+
+class JudgeImages:
+ """Numbers the images found while formatting a transcript, then attaches them to the judge request."""
+
+ def __init__(self, team_id: int) -> None:
+ self._team_id = team_id
+ self._images: list[_Image] = []
+ self._numbers: dict[str, int] = {}
+ # After one storage failure the rest would fail too, each after its own timeout and retry.
+ self._storage_failed = False
+
+ def marker(self, media: MediaBlock) -> str:
+ pointer = BlobPointer.parse(media.url) if media.url else None
+ mime_type = _normalize_mime_type(media.mime_type or (pointer.mime_type if pointer else None))
+ # Links from the customer's app keep a plain marker: we never fetch them.
+ key = pointer.sha256 if pointer else media.base64_data
+ if media.modality != "image" or mime_type not in IMAGE_MIME_TYPES or key is None:
+ return media.marker()
+ if (number := self._numbers.get(key)) is None:
+ number = self._numbers[key] = len(self._images) + 1
+ self._images.append(
+ _Image(number=number, mime_type=mime_type, pointer=pointer, base64_data=media.base64_data)
+ )
+ return f"[image {number}]"
+
+ def content(self, user_prompt: str, provider: str, model: str) -> str | list[Any]:
+ """The user message content: each referenced image under its label, then the transcript."""
+ # Line sampling can drop a marker from a long transcript, and then its image only costs tokens.
+ images = [image for image in self._images if f"[image {image.number}]" in user_prompt]
+ if not images:
+ return user_prompt
+
+ # Keep the images referenced last: the newest input and the graded output, even when it repeats an old image.
+ ranked = sorted(images, key=lambda image: user_prompt.rfind(f"[image {image.number}]"))
+ attachments: dict[int, _Attachment] = {}
+ inline_bytes = 0
+ for image in reversed(ranked[-MAX_IMAGES:]):
+ attachment = self._attach(image, provider, model, MAX_INLINE_BYTES - inline_bytes)
+ inline_bytes += attachment.inline_bytes
+ attachments[image.number] = attachment
+
+ parts: list[Any] = [{"type": "text", "text": "The transcript refers to these images as [image N]."}]
+ outcomes: Counter[str] = Counter()
+ for image in images:
+ if image.number not in attachments:
+ continue
+ attachment = attachments[image.number]
+ outcomes[attachment.outcome] += 1
+ if attachment.part is None:
+ label = _NOT_ATTACHED_LABELS[attachment.outcome]
+ parts.append({"type": "text", "text": f"Image {image.number}: {label}."})
+ else:
+ parts.extend([{"type": "text", "text": f"Image {image.number}:"}, attachment.part])
+ # One line, not one per image: it sits outside the transcript's text budget.
+ if over_limit := len(images) - len(attachments):
+ outcomes["over_limit"] = over_limit
+ parts.append({"type": "text", "text": f"Images not attached, over the judge's image limit: {over_limit}."})
+ parts.append({"type": "text", "text": user_prompt})
+ logger.info("judge_images_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)
+ return parts
+
+ def _attach(self, image: _Image, provider: str, model: str, inline_budget: int) -> _Attachment:
+ transport = image_transport(provider, model, image.mime_type)
+ if transport is None:
+ return _Attachment(outcome="unavailable")
+ provider_limit = MAX_IMAGE_BYTES.get(provider)
+ if image.pointer is None:
+ if len(image.base64_data or "") * 3 // 4 > min(inline_budget, provider_limit or inline_budget):
+ return _Attachment(outcome="too_large")
+ try:
+ data = base64.b64decode(image.base64_data or "", validate=True)
+ except ValueError: # binascii.Error, or a non-ASCII string
+ return _Attachment(outcome="unavailable")
+ return _Attachment(
+ outcome="inline", part=ImagePart(mime_type=image.mime_type, data=data), inline_bytes=len(data)
+ )
+ if self._storage_failed:
+ return _Attachment(outcome="unavailable")
+ limits = [
+ limit for limit in (inline_budget if transport == "inline" else None, provider_limit) if limit is not None
+ ]
+ try:
+ return self._attach_blob(image, image.pointer, transport, min(limits, default=None))
+ except ObjectStorageError:
+ self._storage_failed = True
+ return _Attachment(outcome="unavailable")
+
+ def _attach_blob(
+ self, image: _Image, pointer: BlobPointer, transport: Literal["url", "inline"], max_bytes: int | None
+ ) -> _Attachment:
+ stored_size = 0
+ if max_bytes is not None:
+ # Check the stored size first, so an oversized blob is never downloaded or sent.
+ size = pointer.stored_size(self._team_id)
+ if size is None:
+ return _Attachment(outcome="unavailable")
+ if size > max_bytes:
+ return _Attachment(outcome="too_large")
+ stored_size = size
+ if transport == "url":
+ url = pointer.presigned_url(self._team_id, PRESIGNED_URL_SECONDS)
+ if url is None:
+ return _Attachment(outcome="unavailable")
+ return _Attachment(outcome="link", part=ImagePart(mime_type=image.mime_type, url=url))
+ # ponytail: sequential reads, a thread pool if traces with many images get slow
+ data = pointer.read(self._team_id)
+ if data is None:
+ return _Attachment(outcome="unavailable")
+ part = ImagePart(mime_type=image.mime_type, data=data)
+ return _Attachment(outcome="inline", part=part, inline_bytes=stored_size)
+
+
+def _normalize_mime_type(mime_type: str | None) -> str | None:
+ if mime_type is None:
+ return None
+ normalized = mime_type.strip().lower()
+ return "image/jpeg" if normalized == "image/jpg" else normalized
+
+
+@lru_cache(maxsize=4096)
+def _project_uuid(team_id: int) -> str | None:
+ project_uuid = Team.objects.filter(pk=team_id).values_list("uuid", flat=True).first()
+ return str(project_uuid) if project_uuid else None
+
+
+def judge_sees_images(evaluation: dict[str, Any]) -> bool:
+ if evaluation["evaluation_config"].get("include_media") is False:
+ return False
+ # Local evaluation keeps the check off the network. An inconclusive result means off.
+ try:
+ # Team ids repeat across regions and self-hosted instances, so the flag targets the project uuid.
+ project_uuid = _project_uuid(evaluation["team_id"])
+ if project_uuid is None:
+ return False
+ return bool(
+ posthoganalytics.feature_enabled(
+ JUDGE_IMAGES_FLAG,
+ project_uuid,
+ groups={"project": project_uuid},
+ group_properties={"project": {"id": str(evaluation["team_id"]), "uuid": project_uuid}},
+ only_evaluate_locally=True,
+ send_feature_flag_events=False,
+ )
+ )
+ except Exception:
+ return False
+
+
+def build_image_prompt(
+ evaluation: dict[str, Any], plain_prompt: str | None, render: Callable[[MediaMarker], str | None]
+) -> ImagePrompt | None:
+ """The marked prompt and its images, or None when this judge run gets the plain prompt only.
+
+ `render` builds the same transcript as the plain prompt, with media blocks rendered by the marker it gets.
+ """
+ if not judge_sees_images(evaluation):
+ return None
+ images = JudgeImages(evaluation["team_id"])
+ marked = render(images.marker)
+ return ImagePrompt(text=marked, images=images) if marked is not None and marked != plain_prompt else None
diff --git a/posthog/temporal/ai_observability/message_utils.py b/posthog/temporal/ai_observability/message_utils.py
index 9f8c7b799117..50f922b992f3 100644
--- a/posthog/temporal/ai_observability/message_utils.py
+++ b/posthog/temporal/ai_observability/message_utils.py
@@ -3,10 +3,12 @@
import json
from typing import Any, Union
-from products.ai_observability.backend.text_repr.formatters.otel_parts import flatten_parts_message
+from products.ai_observability.backend.text_repr.formatters.constants import RESPONSES_TOOL_OUTPUT_TYPES
+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaMarker, parse_media_block
+from products.ai_observability.backend.text_repr.formatters.otel_parts import MEDIA_PART_KEY, flatten_parts_message
-def extract_text_from_messages(messages: Union[str, list, dict, None]) -> str:
+def extract_text_from_messages(messages: Union[str, list, dict, None], media_marker: MediaMarker | None = None) -> str:
"""
Extract readable text from LLM message structures.
@@ -22,6 +24,8 @@ def extract_text_from_messages(messages: Union[str, list, dict, None]) -> str:
Returns formatted string like:
"user: Hello\\nassistant: Hi there"
+
+ With `media_marker`, media blocks render as the markers it returns, such as `[image 1]`.
"""
if not messages:
return ""
@@ -36,7 +40,7 @@ def extract_text_from_messages(messages: Union[str, list, dict, None]) -> str:
for msg in messages:
if isinstance(msg, dict):
for flattened in flatten_parts_message(msg):
- rendered_msg = _render_message(flattened)
+ rendered_msg = _render_message(flattened, media_marker)
if rendered_msg is not None:
formatted_parts.append(rendered_msg)
elif isinstance(msg, str):
@@ -47,7 +51,7 @@ def extract_text_from_messages(messages: Union[str, list, dict, None]) -> str:
# Handle single dict message — render it the same way as a one-element
# list so that role prefixes and tool_call_id correlation are surfaced
# consistently regardless of the wrapper shape.
- rendered_messages = [_render_message(flattened) for flattened in flatten_parts_message(messages)]
+ rendered_messages = [_render_message(flattened, media_marker) for flattened in flatten_parts_message(messages)]
return "\n".join(rendered for rendered in rendered_messages if rendered is not None)
@@ -75,15 +79,25 @@ def _only_tool_results(msg: dict) -> bool:
)
-def _render_message(msg: dict) -> str | None:
+def _render_message(msg: dict, media_marker: MediaMarker | None) -> str | None:
"""Render a single message dict into a `role: body` line.
Returns None when the message has neither a role nor any body to render
(so the caller can skip it).
"""
+ item_type = msg.get("type")
+ if media_marker is not None and isinstance(item_type, str) and item_type in RESPONSES_TOOL_OUTPUT_TYPES:
+ # A Responses API tool output has no role or content, so render it as a tool message to reach its images.
+ msg = {"role": "tool", "tool_call_id": msg.get("call_id"), "content": msg.get("output")}
role = msg.get("role", "")
content = msg.get("content", "")
- text = _extract_content_text(content)
+ part = msg.get(MEDIA_PART_KEY)
+ media = parse_media_block(part) if media_marker is not None and isinstance(part, dict) else None
+ text = (
+ media_marker(media)
+ if media_marker is not None and media is not None
+ else _extract_content_text(content, media_marker)
+ )
tool_calls_text = _format_tool_calls(msg.get("tool_calls"))
rendered = " ".join(part for part in (text, tool_calls_text) if part)
@@ -112,13 +126,14 @@ def _render_message(msg: dict) -> str | None:
return None
-def _extract_content_text(content: Union[str, list, dict, None]) -> str:
+def _extract_content_text(content: Union[str, list, dict, None], media_marker: MediaMarker | None = None) -> str:
"""Extract text from message content, handling nested structures.
Handles multiple provider formats:
- Anthropic: [{"type": "text", "text": "..."}]
- OpenAI Responses API: [{"text": "...", "annotations": [...]}]
- Generic: [{"content": "..."}]
+ - With `media_marker`, image, audio and file blocks, as markers such as `[image 1]`
- Plain strings
Always falls back to str() rather than returning empty — an LLM judge
@@ -139,7 +154,14 @@ def _extract_content_text(content: Union[str, list, dict, None]) -> str:
if "text" in block:
text_parts.append(str(block["text"]))
elif "content" in block:
- text_parts.append(str(block["content"]))
+ # Tool results can hold image blocks, which only a judge with images unpacks.
+ nested = block["content"]
+ if media_marker is not None and isinstance(nested, list):
+ text_parts.append(_extract_content_text(nested, media_marker))
+ else:
+ text_parts.append(str(nested))
+ elif media_marker is not None and (media := parse_media_block(block)) is not None:
+ text_parts.append(media_marker(media))
else:
# Unknown block shape — stringify rather than silently drop
text_parts.append(str(block))
@@ -147,6 +169,9 @@ def _extract_content_text(content: Union[str, list, dict, None]) -> str:
text_parts.append(block)
return " ".join(text_parts)
+ if media_marker is not None and isinstance(content, dict) and (media := parse_media_block(content)) is not None:
+ return media_marker(media)
+
# Fallback: convert to string
return str(content)
diff --git a/posthog/temporal/ai_observability/run_session_evaluation.py b/posthog/temporal/ai_observability/run_session_evaluation.py
index 565cb4784ab8..8b683d743139 100644
--- a/posthog/temporal/ai_observability/run_session_evaluation.py
+++ b/posthog/temporal/ai_observability/run_session_evaluation.py
@@ -42,6 +42,7 @@ from posthog.temporal.ai_observability.evaluation_payload import (
should_skip_for_payload,
)
from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult, build_skipped_evaluation_result
+from posthog.temporal.ai_observability.judge_images import build_image_prompt
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages
from posthog.temporal.ai_observability.run_trace_evaluation import TRACE_EVENTS_LOOKBACK
from posthog.temporal.common.utils import close_db_connections
@@ -56,6 +57,7 @@ from products.ai_observability.backend.text_repr.formatters import (
format_trace_text_repr,
llm_trace_to_formatter_format,
)
+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaMarker
# Sessions run ~3.5x a trace at p90 (222 events against a trace's typical handful) and the shared
# formatter uniformly samples lines rather than truncating, so the trace's 150k budget would hand
@@ -358,7 +360,7 @@ user's conversation, in order — according to this criteria:
{config.instructions}"""
-def format_session_for_judge(traces: list[LLMTrace]) -> str | None:
+def format_session_for_judge(traces: list[LLMTrace], media_marker: MediaMarker | None = None) -> str | None:
"""Render a session as the canonical text representation, one section per trace.
Preserve generation outputs across the full session before truncating them, including
@@ -386,6 +388,9 @@ def format_session_for_judge(traces: list[LLMTrace]) -> str | None:
"include_line_numbers": True,
"max_length": None if preserve_generation_output else per_trace_budget,
}
+ if media_marker is not None:
+ # One marker for the whole session, so an image repeated across traces is attached once.
+ options["media_marker"] = media_marker
sections: list[str] = []
rendered_length = 0
for index, trace in enumerate(traces, start=1):
@@ -486,7 +491,14 @@ def execute_session_llm_judge_activity(inputs: ExecuteSessionEvaluationInputs) -
if outcome.skip_reason or outcome.traces is None:
return build_session_skip_result(allows_na, outcome.skip_reason or "session_not_found", output_type=output_type)
- transcript = format_session_for_judge(outcome.traces)
+ traces = outcome.traces
+ transcript = format_session_for_judge(traces)
+ image_prompt = build_image_prompt(
+ evaluation, transcript, lambda media_marker: format_session_for_judge(traces, media_marker)
+ )
+ if transcript is None and image_prompt is not None:
+ # Inline base64 can push the plain transcript over the limit when the marked one still fits.
+ transcript = image_prompt.unnumbered_text()
if transcript is None:
return build_session_skip_result(allows_na, "session_too_long_to_judge", output_type=output_type)
@@ -498,6 +510,7 @@ def execute_session_llm_judge_activity(inputs: ExecuteSessionEvaluationInputs) -
user_prompt=transcript,
allows_na=allows_na,
backfill=inputs.backfill_id is not None,
+ image_prompt=image_prompt,
)
diff --git a/posthog/temporal/ai_observability/run_trace_evaluation.py b/posthog/temporal/ai_observability/run_trace_evaluation.py
index fe92da4f6bf8..3e3b71439dee 100644
--- a/posthog/temporal/ai_observability/run_trace_evaluation.py
+++ b/posthog/temporal/ai_observability/run_trace_evaluation.py
@@ -63,6 +63,7 @@ from posthog.temporal.ai_observability.evaluation_workflow_activities import (
emit_internal_telemetry_activity,
fetch_evaluation_activity,
)
+from posthog.temporal.ai_observability.judge_images import build_image_prompt
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages
from posthog.temporal.ai_observability.metrics import increment_emit_event_outcome, increment_errors
from posthog.temporal.ai_observability.run_evaluation import (
@@ -92,6 +93,7 @@ from products.ai_observability.backend.text_repr.formatters import (
format_trace_text_repr,
llm_trace_to_formatter_format,
)
+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaMarker
from products.ai_observability.backend.text_repr.formatters.message_formatter import has_message_content
if TYPE_CHECKING:
@@ -549,7 +551,7 @@ def build_trace_system_prompt(
{config.instructions}"""
-def format_trace_for_judge(trace: LLMTrace) -> str:
+def format_trace_for_judge(trace: LLMTrace, media_marker: MediaMarker | None = None) -> str:
"""Serialize a trace into the canonical text representation for the LLM judge.
Preserve complete generation outputs before truncating them so the judge does not grade
@@ -565,6 +567,8 @@ def format_trace_for_judge(trace: LLMTrace) -> str:
"max_length": None,
"max_render_length": JUDGE_TRACE_MAX_CHARS,
}
+ if media_marker is not None:
+ options["media_marker"] = media_marker
for truncated in (False, True):
options["truncated"] = truncated
try:
@@ -694,14 +698,19 @@ def execute_trace_llm_judge_activity(inputs: ExecuteTraceEvaluationInputs) -> Ev
if not _has_judge_transcript(outcome.trace):
return _build_trace_skip_result(allows_na, "trace_not_found", output_type=output_type)
+ trace = outcome.trace
+ user_prompt = format_trace_for_judge(trace)
return call_llm_judge(
evaluation=evaluation,
system_prompt=build_trace_system_prompt(
prompt, allows_na, output_type=output_type, output_config=output_config
),
- user_prompt=format_trace_for_judge(outcome.trace),
+ user_prompt=user_prompt,
allows_na=allows_na,
backfill=inputs.backfill_id is not None,
+ image_prompt=build_image_prompt(
+ evaluation, user_prompt, lambda media_marker: format_trace_for_judge(trace, media_marker)
+ ),
)
diff --git a/posthog/temporal/ai_observability/test_judge_images.py b/posthog/temporal/ai_observability/test_judge_images.py
new file mode 100644
index 000000000000..3d755b001e4f
--- /dev/null
+++ b/posthog/temporal/ai_observability/test_judge_images.py
@@ -0,0 +1,192 @@
+import base64
+
+import pytest
+from unittest.mock import patch
+
+from posthog.storage.object_storage import ObjectStorageError
+from posthog.temporal.ai_observability.judge_images import MAX_IMAGES, JudgeImages
+
+from products.ai_observability.backend.llm.images import ImagePart
+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, parse_media_block
+
+OBJECT_STORAGE = "products.ai_observability.backend.ai_blobs.object_storage"
+
+
+def _pointer(digit: str, mime: str = "image%2Fjpeg") -> MediaBlock:
+ return MediaBlock(modality="image", url=f"phaiblob://v1/sha256/{digit * 64}?mime={mime}&size=63357")
+
+
+def _inline(digit: str, length: int = 4) -> MediaBlock:
+ return MediaBlock(modality="image", mime_type="image/png", base64_data=digit * length)
+
+
+class TestJudgeImagesMarkers:
+ def test_numbers_images_and_reuses_the_number_for_a_repeat(self):
+ images = JudgeImages(team_id=7)
+ inline = MediaBlock(modality="image", mime_type="image/png", base64_data="aGk=")
+
+ markers = [images.marker(media) for media in (_pointer("a"), inline, _pointer("a"))]
+
+ assert markers == ["[image 1]", "[image 2]", "[image 1]"]
+
+ @pytest.mark.parametrize(
+ ("media", "marker"),
+ [
+ pytest.param(MediaBlock(modality="audio", base64_data="aGk="), "[audio]", id="not_an_image"),
+ pytest.param(_pointer("a", mime="image%2Fsvg%2Bxml"), "[image]", id="unsupported_format"),
+ pytest.param(
+ MediaBlock(modality="image", mime_type="image/png", url="https://example.com/a.png"),
+ "[image: https://example.com/a.png]",
+ id="customer_link",
+ ),
+ pytest.param(
+ parse_media_block({"file_data": {"mime_type": "image/png", "file_uri": "gs://bucket/a.png"}}),
+ "[image]",
+ id="gemini_cloud_storage_uri",
+ ),
+ ],
+ )
+ def test_images_it_cannot_attach_keep_an_unnumbered_marker(self, media: MediaBlock, marker: str):
+ assert JudgeImages(team_id=7).marker(media) == marker
+
+
+class TestJudgeImagesContent:
+ def test_an_image_referenced_again_at_the_end_stays_attached(self):
+ images = JudgeImages(team_id=7)
+ markers = [images.marker(_inline(f"{i:02}")) for i in range(MAX_IMAGES + 1)]
+ # The output repeats the first input image after every newer one.
+ content = images.content(" ".join([*markers, markers[0]]), "openai", "gpt-5.5")
+
+ assert {"type": "text", "text": "Image 1:"} in content
+ assert {"type": "text", "text": "Image 2:"} not in content
+ assert {"type": "text", "text": "Images not attached, over the judge's image limit: 1."} in content
+
+ def test_attaches_the_newest_images_when_over_the_limit(self):
+ images = JudgeImages(team_id=7)
+ prompt = " ".join(images.marker(_inline(f"{i:02}")) for i in range(MAX_IMAGES + 1))
+
+ content = images.content(prompt, "openai", "gpt-5.5")
+
+ assert content[1] == {"type": "text", "text": "Image 2:"}
+ assert content[-4:-1] == [
+ {"type": "text", "text": f"Image {MAX_IMAGES + 1}:"},
+ ImagePart(mime_type="image/png", data=base64.b64decode(f"{MAX_IMAGES:02}" * 4)),
+ {"type": "text", "text": "Images not attached, over the judge's image limit: 1."},
+ ]
+
+ @pytest.mark.parametrize(
+ ("provider", "model", "expected_image"),
+ [
+ pytest.param("openai", "gpt-5.5", ImagePart(mime_type="image/jpeg", url="https://signed"), id="link"),
+ pytest.param("together_ai", "any", ImagePart(mime_type="image/jpeg", data=b"hi"), id="inline"),
+ pytest.param("gemini", "gemini-2.0-flash", ImagePart(mime_type="image/jpeg", data=b"hi"), id="gemini_2_0"),
+ ],
+ )
+ def test_attaches_each_image_under_its_label(self, provider: str, model: str, expected_image: ImagePart):
+ images = JudgeImages(team_id=7)
+ prompt = f"Input: user: {images.marker(_pointer('a', mime='image%2FJPG'))} What does this chart show?"
+
+ with (
+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed") as presign,
+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 2}),
+ patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"hi") as read,
+ ):
+ content = images.content(prompt, provider, model)
+
+ assert content == [
+ {"type": "text", "text": "The transcript refers to these images as [image N]."},
+ {"type": "text", "text": "Image 1:"},
+ expected_image,
+ {"type": "text", "text": prompt},
+ ]
+ storage_call = presign if expected_image.url else read
+ assert storage_call.call_args.args[0] == f"aio/7/sha256/{'a' * 64}"
+
+ @pytest.mark.parametrize(
+ ("provider", "model", "media", "storage_error"),
+ [
+ pytest.param("openai", "gpt-5.5", _pointer("a"), None, id="blob_missing_for_a_link"),
+ pytest.param("together_ai", "any", _pointer("a"), None, id="blob_missing_for_inline"),
+ pytest.param(
+ "together_ai", "any", _pointer("a"), ObjectStorageError("head_object failed"), id="storage_error"
+ ),
+ pytest.param(
+ "gemini", "gemini-3-flash", _pointer("a", mime="image%2Fgif"), None, id="format_rejected_by_provider"
+ ),
+ pytest.param(
+ "gemini",
+ "gemini-3-flash",
+ MediaBlock(modality="image", mime_type="image/png", base64_data="aGk"),
+ None,
+ id="invalid_inline_base64",
+ ),
+ pytest.param(
+ "gemini",
+ "gemini-3-flash",
+ MediaBlock(modality="image", mime_type="image/png", base64_data="aGk\u00e9"),
+ None,
+ id="non_ascii_inline_base64",
+ ),
+ ],
+ )
+ def test_reports_an_image_it_could_not_attach(
+ self, provider: str, model: str, media: MediaBlock, storage_error: Exception | None
+ ):
+ images = JudgeImages(team_id=7)
+ prompt = f"user: {images.marker(media)}"
+
+ with (
+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value=None),
+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value=None, side_effect=storage_error),
+ ):
+ content = images.content(prompt, provider, model)
+
+ assert content[1] == {"type": "text", "text": "Image 1: could not be attached."}
+
+ def test_stops_calling_storage_after_a_storage_error(self):
+ images = JudgeImages(team_id=7)
+ prompt = f"user: {images.marker(_pointer('a'))} {images.marker(_pointer('b'))}"
+
+ with patch(f"{OBJECT_STORAGE}.head_object_strict", side_effect=ObjectStorageError("timed out")) as head:
+ content = images.content(prompt, "together_ai", "any")
+
+ assert head.call_count == 1
+ assert {"type": "text", "text": "Image 1: could not be attached."} in content
+ assert {"type": "text", "text": "Image 2: could not be attached."} in content
+
+ @pytest.mark.parametrize(
+ ("provider", "media", "too_large", "reads"),
+ [
+ pytest.param("openai", _pointer, set(), 0, id="links_have_no_byte_cap"),
+ pytest.param("anthropic", _pointer, {1, 2}, 0, id="provider_limit_on_links"),
+ # The pointers claim 63 KB, so this also checks that the budget uses the stored size.
+ pytest.param("together_ai", _pointer, {1}, 1, id="stored_size_for_inline_providers"),
+ pytest.param(
+ "openai", lambda digit: _inline(digit, length=13_400_000), {1}, 0, id="inline_images_on_link_providers"
+ ),
+ ],
+ )
+ def test_caps_image_bytes(self, provider: str, media, too_large: set[int], reads: int):
+ images = JudgeImages(team_id=7)
+ prompt = f"user: {images.marker(media('a'))} {images.marker(media('b'))}"
+
+ with (
+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed"),
+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 10_000_000}),
+ patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"x" * 10_000_000) as read,
+ ):
+ content = images.content(prompt, provider, "any")
+
+ assert {
+ number
+ for number in (1, 2)
+ if {"type": "text", "text": f"Image {number}: not attached, too large."} in content
+ } == too_large
+ # An image over the budget is never downloaded.
+ assert read.call_count == reads
+
+ def test_skips_images_whose_marker_was_sampled_out_of_the_prompt(self):
+ images = JudgeImages(team_id=7)
+ images.marker(_pointer("a"))
+
+ assert images.content("user: hello", "openai", "gpt-5.5") == "user: hello"
diff --git a/posthog/temporal/ai_observability/test_message_utils.py b/posthog/temporal/ai_observability/test_message_utils.py
index 548c36ccf6e1..bcdc89602c96 100644
--- a/posthog/temporal/ai_observability/test_message_utils.py
+++ b/posthog/temporal/ai_observability/test_message_utils.py
@@ -4,6 +4,7 @@ import json
import pytest
+from posthog.temporal.ai_observability.judge_images import JudgeImages
from posthog.temporal.ai_observability.message_utils import (
extract_text_from_messages,
format_tool_definitions,
@@ -62,6 +63,96 @@ class TestExtractTextFromMessages:
assert expected_substring in result
assert result != ""
+ @pytest.mark.parametrize(
+ ("block", "marker"),
+ [
+ pytest.param(
+ {"type": "image_url", "image_url": {"url": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fjpeg"}},
+ "[image 1]",
+ id="openai_offloaded_pointer",
+ ),
+ pytest.param(
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}},
+ "[image 1]",
+ id="openai_inline_data_uri",
+ ),
+ pytest.param(
+ {"type": "input_image", "image_url": "https://example.com/cat.jpg"},
+ "[image: https://example.com/cat.jpg]",
+ id="openai_responses_customer_link",
+ ),
+ pytest.param(
+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}},
+ "[image 1]",
+ id="anthropic_base64",
+ ),
+ pytest.param(
+ {
+ "type": "image",
+ "source": {"type": "base64", "media_type": "image/png", "data": "[base64 image redacted]"},
+ },
+ "[image]",
+ id="anthropic_redacted",
+ ),
+ pytest.param(
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,[base64 image redacted]"}},
+ "[image]",
+ id="openai_redacted_data_uri",
+ ),
+ pytest.param(
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64," * 3 + "aGk="}},
+ "[image]",
+ id="nested_data_uri",
+ ),
+ pytest.param({"inline_data": {"mime_type": "image/png", "data": "aGk="}}, "[image 1]", id="gemini_inline"),
+ pytest.param(
+ {"type": "image", "inline_data": {"mime_type": "image/png", "data": "aGk="}},
+ "[image 1]",
+ id="gemini_posthog_sdk",
+ ),
+ pytest.param(
+ {"type": "image", "data": "aGk=", "mime_type": "image/png"}, "[image 1]", id="gemini_interactions"
+ ),
+ pytest.param({"inlineData": {"mimeType": "audio/wav", "data": "aGk="}}, "[audio]", id="gemini_audio"),
+ pytest.param({"type": "input_audio", "input_audio": {"data": "aGk="}}, "[audio]", id="openai_audio"),
+ pytest.param({"type": "document", "source": {"type": "base64", "data": "aGk="}}, "[file]", id="file"),
+ ],
+ )
+ def test_judge_images_replace_media_blocks_with_markers(self, block, marker):
+ messages = [{"role": "user", "content": [block, {"type": "text", "text": "What does this chart show?"}]}]
+
+ assert (
+ extract_text_from_messages(messages, JudgeImages(team_id=7).marker)
+ == f"user: {marker} What does this chart show?"
+ )
+
+ def test_judge_images_number_otel_parts_and_tool_result_images(self):
+ pointer = f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1"
+ image = {"type": "image_url", "image_url": {"url": pointer}}
+ messages = [
+ {
+ "role": "user",
+ "parts": [{"type": "blob", "modality": "image", "mime_type": "image/png", "content": pointer}],
+ },
+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [image]}]},
+ {
+ "type": "function_call_output",
+ "call_id": "c1",
+ "output": [{"type": "input_image", "image_url": pointer}],
+ },
+ ]
+
+ assert (
+ extract_text_from_messages(messages, JudgeImages(team_id=7).marker)
+ == "user: [image 1]\nuser: [image 1]\ntool[c1]: [image 1]"
+ )
+ assert extract_text_from_messages(messages).startswith("user: [image]\n")
+
+ def test_media_blocks_keep_their_text_without_judge_images(self):
+ block = {"type": "file", "file": {"filename": "report.pdf"}}
+
+ assert "report.pdf" in extract_text_from_messages([{"role": "user", "content": [block]}])
+
def test_mixed_content_blocks(self):
messages = [
{
@@ -499,6 +590,16 @@ class TestExtractTextFromMessages:
),
pytest.param({"type": "file", "file_id": "file_123"}, "[file: file_123]", id="file_reference"),
pytest.param({"type": "blob", "content": "aGVsbG8="}, "[media]", id="blob_without_modality"),
+ pytest.param(
+ {"type": "blob", "mime_type": "audio/wav", "content": "aGVsbG8="},
+ "[audio]",
+ id="modality_from_mime_type",
+ ),
+ pytest.param(
+ {"type": "uri", "modality": "image", "mime_type": "image/png", "uri": "gs://bucket/a.png"},
+ "[image]",
+ id="uri_nobody_else_can_fetch",
+ ),
],
)
def test_otel_parts_media_render_markers(self, part, marker):
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 3366a68f9844..5ae5ee7572aa 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -1,3 +1,4 @@
+import sys
import json
import uuid
import dataclasses
@@ -27,6 +28,7 @@ from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker
from posthog.api.capture import CaptureInternalError
from posthog.constants import AvailableFeature
from posthog.models import Organization, OrganizationMembership, Team, User
+from posthog.temporal.ai_observability.judge_images import ImagePrompt, JudgeImages
from posthog.temporal.ai_observability.sentiment.extraction import truncate_to_head_tail
from posthog.temporal.ai_observability.sentiment.schema import SentimentResult
from posthog.temporal.common.errors import NonReportableError
@@ -34,6 +36,7 @@ from posthog.temporal.common.posthog_client import is_expected_activity_failure
from products.access_control.backend.models.access_control import AccessControl
from products.ai_observability.backend.llm.errors import (
+ RESPONSE_LIMIT_MESSAGE,
AuthenticationError,
ClaudeDeploymentOnAzureOpenAIError,
ContentFilteredError,
@@ -42,16 +45,19 @@ from products.ai_observability.backend.llm.errors import (
ModelPermissionError,
OutputTokenLimitError,
ProviderConnectionError,
+ ProviderRequestRejectedError,
QuotaExceededError,
RateLimitError,
StructuredOutputParseError,
UnsupportedModelError,
)
+from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.models.evaluation_config import EvaluationConfig
from products.ai_observability.backend.models.evaluation_directories import EvaluationDirectory
from products.ai_observability.backend.models.evaluations import Evaluation
from products.ai_observability.backend.models.provider_keys import LLMProviderKey
from products.ai_observability.backend.text_repr.formatters import EVENT_TEXT_MAX_CHARS
+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock
from . import run_evaluation as run_evaluation_module
from .evaluation_errors import (
@@ -1137,6 +1143,73 @@ def _openai_status_error(status: int, message: str) -> openai.APIStatusError:
return openai.APIStatusError(f"Error code: {status}", response=response, body={"error": {"message": message}})
+def _adapter_rejection(cause: Exception | None) -> ProviderRequestRejectedError:
+ error = ProviderRequestRejectedError(RESPONSE_LIMIT_MESSAGE if cause is None else "The provider rejected it.")
+ error.__cause__ = cause
+ return error
+
+
+ ("error", "presigned_url", "retries"),
+ [
+ pytest.param(
+ _adapter_rejection(_openai_status_error(400, "Image input is not supported.")),
+ "https://signed",
+ True,
+ id="adapter_wrapped_400",
+ ),
+ pytest.param(
+ _openai_status_error(400, "Error while downloading https://signed."), "https://signed", True, id="raw_400"
+ ),
+ pytest.param(_adapter_rejection(None), "https://signed", False, id="unreadable_response"),
+ pytest.param(_openai_status_error(401, "Invalid API key."), "https://signed", False, id="auth_error"),
+ pytest.param(_openai_status_error(400, "Bad request."), None, False, id="no_image_attached"),
+ ],
+)
+def test_judge_falls_back_to_the_plain_prompt_only_when_an_attached_image_is_rejected(
+ error: Exception, presigned_url: str | None, retries: bool
+) -> None:
+ images = JudgeImages(team_id=1)
+ pointer = MediaBlock(modality="image", url=f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1")
+ image_prompt = ImagePrompt(text=f"user: {images.marker(pointer)} Polite?", images=images)
+
+ with (
+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client") as client_class,
+ patch(
+ "products.ai_observability.backend.ai_blobs.object_storage.get_presigned_url", return_value=presigned_url
+ ),
+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.capture_exception") as capture,
+ ):
+ # Only a PostHog key reports the fallback. On BYOK a rejection ends as a skip instead of raising.
+ spec.return_value.resolve.return_value = MagicMock(
+ provider="openai", model="gpt-5.5", provider_key=MagicMock(), is_byok=not retries
+ )
+ # The provider error can quote the link, so nothing may log or chain it while the fallback runs.
+ active_errors: list[BaseException | None] = []
+ capture.side_effect = lambda *_: active_errors.append(sys.exc_info()[1])
+ client = client_class.return_value
+ client.complete.side_effect = [
+ error,
+ MagicMock(parsed=BooleanEvalResult(verdict=True, reasoning="ok"), usage=None),
+ ]
+ result = call_llm_judge(
+ evaluation={"id": "test-evaluation", "team_id": 1, "evaluation_config": {"prompt": "Polite?"}},
+ system_prompt="",
+ user_prompt="user: Polite?",
+ allows_na=False,
+ image_prompt=image_prompt,
+ )
+
+ if retries:
+ assert result["verdict"] is True
+ assert client.complete.call_args_list[1].args[0].messages == [{"role": "user", "content": "user: Polite?"}]
+ assert "https://signed" not in str(capture.call_args)
+ assert active_errors == [None]
+ else:
+ assert client.complete.call_count == 1
+
+
@pytest.mark.parametrize(
"provider, raised_exception, expected_status, expected_detail",
[
@@ -1499,6 +1572,86 @@ class TestRunEvaluationWorkflow:
assert result["reasoning"] == "The answer is correct"
mock_client.complete.assert_called_once()
+ @pytest.mark.parametrize(
+ ("flag_enabled", "evaluation_config", "attaches_image"),
+ [
+ pytest.param(True, {"prompt": "Is the chart described correctly?"}, True, id="flag_on"),
+ pytest.param(False, {"prompt": "Is the chart described correctly?"}, False, id="flag_off"),
+ pytest.param(
+ True, {"prompt": "Is the chart described correctly?", "include_media": False}, False, id="eval_opts_out"
+ ),
+ ],
+ )
+ @pytest.mark.django_db(transaction=True)
+ def test_execute_llm_judge_activity_attaches_images_when_enabled(
+ self, flag_enabled, evaluation_config, attaches_image, setup_data, active_key_config
+ ):
+ team = setup_data["team"]
+ evaluation = {
+ "id": str(setup_data["evaluation"].id),
+ "name": "Test Evaluation",
+ "evaluation_type": "llm_judge",
+ "evaluation_config": evaluation_config,
+ "output_type": "boolean",
+ "output_config": {},
+ "team_id": team.id,
+ }
+ pointer = f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fjpeg&size=63357"
+ event_data = create_mock_event_data(
+ team.id,
+ properties={
+ "$ai_input": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image_url", "image_url": {"url": pointer}},
+ {"type": "text", "text": "What does this chart show?"},
+ ],
+ }
+ ],
+ "$ai_output_choices": [
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "image_url", "image_url": {"url": pointer.replace("a" * 64, "b" * 64)}},
+ {"type": "text", "text": "Sales rose in March."},
+ ],
+ }
+ ],
+ },
+ )
+
+ with (
+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client") as mock_client_class,
+ patch(
+ "posthog.temporal.ai_observability.judge_images.posthoganalytics.feature_enabled",
+ return_value=flag_enabled,
+ ) as feature_enabled,
+ patch(
+ "products.ai_observability.backend.ai_blobs.object_storage.get_presigned_url",
+ return_value="https://signed",
+ ),
+ ):
+ mock_client = mock_client_class.return_value
+ mock_client.complete.return_value = MagicMock(
+ parsed=BooleanEvalResult(verdict=True, reasoning="ok"),
+ usage=MagicMock(input_tokens=1, output_tokens=1, total_tokens=1),
+ )
+ execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))
+
+ content = mock_client.complete.call_args.args[0].messages[0]["content"]
+ if "include_media" not in evaluation_config:
+ # Team ids repeat across regions, so the flag is checked against the project uuid.
+ assert feature_enabled.call_args.kwargs["groups"] == {"project": str(team.uuid)}
+ if attaches_image:
+ assert ImagePart(mime_type="image/jpeg", url="https://signed") in content
+ assert "user: [image 1] What does this chart show?" in content[-1]["text"]
+ assert "assistant: [image 2] Sales rose in March." in content[-1]["text"]
+ else:
+ assert isinstance(content, str)
+ assert "[image" not in content
+ assert "What does this chart show?" in content
+
@pytest.mark.parametrize(
"oversized_input",
[
@@ -4514,6 +4667,7 @@ class TestJudgePromptAssembly:
"evaluation_type": "llm_judge",
"evaluation_config": {"prompt": "Is this correct?"},
"output_type": "boolean",
+ "team_id": 1,
}
event_data = create_mock_event_data(
1,
@@ -4539,6 +4693,7 @@ class TestJudgePromptAssembly:
"evaluation_type": "llm_judge",
"evaluation_config": {"prompt": "Is this correct?"},
"output_type": output_type,
+ "team_id": 1,
"output_config": {
"allows_na": allows_na,
**({"options": [{"key": "correct", "label": "Correct"}]} if output_type == "categorical" else {}),
@@ -4563,6 +4718,39 @@ class TestJudgePromptAssembly:
skip_reason="generation_output_too_long",
)
+ def test_inline_images_that_overflow_the_plain_prompt_are_still_judged(self) -> None:
+ evaluation = {
+ "evaluation_type": "llm_judge",
+ "evaluation_config": {"prompt": "Is this correct?"},
+ "output_type": "boolean",
+ "team_id": 1,
+ "output_config": {},
+ }
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64," + "A" * 20_000}}
+ event_data = create_mock_event_data(
+ 1,
+ properties={
+ "$ai_input": "Draw eight charts.",
+ "$ai_output_choices": [{"role": "assistant", "content": [image] * 8}],
+ },
+ )
+
+ with (
+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as judge,
+ patch(
+ "posthog.temporal.ai_observability.judge_images.posthoganalytics.feature_enabled",
+ return_value=True,
+ ),
+ patch("posthog.temporal.ai_observability.judge_images._project_uuid", return_value="uuid"),
+ ):
+ execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))
+
+ image_prompt = judge.call_args.kwargs["image_prompt"]
+ assert image_prompt is not None
+ # Decision judges and the text-only retry use this prompt, and they send no images.
+ assert judge.call_args.kwargs["user_prompt"] == image_prompt.unnumbered_text()
+ assert "[image 1]" not in judge.call_args.kwargs["user_prompt"]
+
@pytest.mark.parametrize(
"text,expected_text",
[
diff --git a/posthog/temporal/ai_observability/test_run_session_evaluation.py b/posthog/temporal/ai_observability/test_run_session_evaluation.py
index e32bd6ec8d0b..70a4b7cb3608 100644
--- a/posthog/temporal/ai_observability/test_run_session_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_session_evaluation.py
@@ -20,6 +20,7 @@ from posthog.temporal.ai_observability.evaluation_llm_judge import (
get_output_type_config,
)
from posthog.temporal.ai_observability.evaluation_payload import payload_budget_bytes
+from posthog.temporal.ai_observability.judge_images import JudgeImages
from posthog.temporal.ai_observability.run_session_evaluation import (
_SESSION_EVENT_COUNT_SQL,
AI_EVENTS_RETENTION_DAYS,
@@ -141,6 +142,18 @@ class TestFormatSessionForJudge:
traces = [_trace(f"t{i}", cost=0, latency=0, event_count=100) for i in range(260)]
assert format_session_for_judge(traces) is None
+ def test_numbers_an_image_once_across_the_session(self):
+ image = {"type": "image_url", "image_url": {"url": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1"}}
+ traces = [_trace(f"t{i}", cost=0, latency=0) for i in range(2)]
+ for trace in traces:
+ trace.events[0].properties["$ai_input"] = [{"role": "user", "content": [image]}]
+
+ rendered = format_session_for_judge(traces, JudgeImages(team_id=1).marker)
+
+ assert rendered is not None
+ assert rendered.count("[image 1]") == 2
+ assert "[image 2]" not in rendered
+
def test_many_small_traces_are_judged_rather_than_skipped_on_count_alone(self):
"""The overflow check must measure the rendered transcript, not `per_trace_budget * n`.
The product form made this a hard cliff at 251 traces however little they contained, so a
diff --git a/posthog/temporal/ai_observability/test_run_trace_evaluation.py b/posthog/temporal/ai_observability/test_run_trace_evaluation.py
index a78b2f6f9b12..3b94246a8d4e 100644
--- a/posthog/temporal/ai_observability/test_run_trace_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_trace_evaluation.py
@@ -23,6 +23,7 @@ from posthog.hogql_queries.ai.trace_query_runner import TraceQueryRunner
from posthog.models import Organization, PropertyDefinition, Team, User
from posthog.models.ai_events.test_util import bulk_create_ai_events
from posthog.models.event.util import bulk_create_events
+from posthog.temporal.ai_observability.judge_images import JudgeImages
from posthog.temporal.ai_observability.run_session_evaluation import fetch_session_for_evaluation
from posthog.temporal.ai_observability.trace_summarization.fetch_and_format import _fetch_and_format_trace
@@ -160,6 +161,47 @@ class TestFormatTraceForJudge:
assert "gpt-4o" in transcript
assert "What is 2+2?" in transcript
+ def test_numbers_images_only_for_a_judge_with_images(self):
+ image = {"type": "image_url", "image_url": {"url": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1"}}
+ otel_blob = {"type": "blob", "modality": "image", "mime_type": "image/png", "content": "aGk="}
+ trace = create_trace(
+ [
+ create_trace_event(
+ "$ai_generation",
+ **{
+ "$ai_input": [
+ {"role": "user", "content": [image, {"type": "text", "text": "Describe the chart."}]},
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [image]}],
+ },
+ {"role": "user", "parts": [otel_blob]},
+ {
+ "type": "function_call_output",
+ "call_id": "c1",
+ "output": [
+ {
+ "type": "input_image",
+ "image_url": f"phaiblob://v1/sha256/{'c' * 64}?mime=image%2Fpng&size=1",
+ }
+ ],
+ },
+ ],
+ "$ai_output": "A rising line.",
+ },
+ )
+ ]
+ )
+
+ plain = format_trace_for_judge(trace)
+ marked = format_trace_for_judge(trace, JudgeImages(team_id=1).marker)
+
+ assert "[image 1]" not in plain
+ # The message and the tool result repeat one image, and the OTel part adds a second.
+ assert marked.count("[image 1]") == 2
+ assert "[image 2]" in marked
+ assert "[image 3]" in marked
+
def test_includes_tool_catalog(self):
tools = [{"type": "function", "function": {"name": "search_docs", "description": "Search the docs"}}]
trace = create_trace(
diff --git a/products/ai_observability/backend/ai_blobs.py b/products/ai_observability/backend/ai_blobs.py
new file mode 100644
index 000000000000..80b2e4d0308c
--- /dev/null
+++ b/products/ai_observability/backend/ai_blobs.py
@@ -0,0 +1,46 @@
+"""`phaiblob://` pointers that ingestion writes in place of offloaded AI media. Twin of `frontend/aiBlob.ts`."""
+
+import re
+from urllib.parse import parse_qs
+
+from django.conf import settings
+
+from posthog.dataclasses import frozen
+from posthog.storage import object_storage
+
+_POINTER_RE = re.compile(r"^phaiblob://v1/sha256/([0-9a-f]{64})(?:\?(.*))?$")
+
+
+def blob_key(team_id: int, sha256: str) -> str:
+ return f"{settings.AI_BLOB_S3_PREFIX}{team_id}/sha256/{sha256}"
+
+
+@frozen
+class BlobPointer:
+ sha256: str
+ mime_type: str | None = None
+
+ @classmethod
+ def parse(cls, value: str) -> "BlobPointer | None":
+ match = _POINTER_RE.match(value)
+ if match is None:
+ return None
+ # The pointer's `size=` comes from event data, so readers ask storage for the real size instead.
+ mime_type = parse_qs(match.group(2) or "").get("mime", [None])[0]
+ return cls(sha256=match.group(1), mime_type=mime_type)
+
+ # The team id always comes from the caller's own scope, never from event data,
+ # so a pointer copied from another team's event can't reach that team's blob.
+ def read(self, team_id: int) -> bytes | None:
+ return object_storage.read_bytes(
+ blob_key(team_id, self.sha256), bucket=settings.AI_BLOB_S3_BUCKET, missing_ok=True
+ )
+
+ def stored_size(self, team_id: int) -> int | None:
+ head = object_storage.head_object_strict(blob_key(team_id, self.sha256), bucket=settings.AI_BLOB_S3_BUCKET)
+ return int(head["ContentLength"]) if head else None
+
+ def presigned_url(self, team_id: int, expiration_seconds: int) -> str | None:
+ return object_storage.get_presigned_url(
+ blob_key(team_id, self.sha256), expiration=expiration_seconds, bucket=settings.AI_BLOB_S3_BUCKET
+ )
diff --git a/products/ai_observability/backend/api/ai_blob.py b/products/ai_observability/backend/api/ai_blob.py
index 22ab3c8da8a5..64977523acb8 100644
--- a/products/ai_observability/backend/api/ai_blob.py
+++ b/products/ai_observability/backend/api/ai_blob.py
@@ -9,6 +9,8 @@ from rest_framework.request import Request
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.storage import object_storage
+from products.ai_observability.backend.ai_blobs import blob_key
+
SCRIPTABLE_IMAGE_MIMES = {"image/svg+xml"}
@@ -27,8 +29,9 @@ class AIBlobViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):
@extend_schema(exclude=True)
@action(detail=False, methods=["GET"], url_path=r"v1/sha256/(?P<hash>[0-9a-f]{64})")
def serve(self, request: Request, hash: str, **kwargs) -> HttpResponse:
- key = f"{settings.AI_BLOB_S3_PREFIX}{self.team_id}/sha256/{hash}"
- result = object_storage.read_object(key, bucket=settings.AI_BLOB_S3_BUCKET, missing_ok=True)
+ result = object_storage.read_object(
+ blob_key(self.team_id, hash), bucket=settings.AI_BLOB_S3_BUCKET, missing_ok=True
+ )
if result is None:
return HttpResponse(status=404)
body, content_type = result
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index 7798809a85bf..a8f971a1d99f 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -126,6 +126,10 @@ logger = structlog.get_logger(__name__)
"nullable": True,
"description": "Judge method. Omit to keep the saved method; null restores automatic routing.",
},
+ "include_media": {
+ "type": "boolean",
+ "description": "Set to false to judge text only.",
+ },
"reasoning_level": {
"type": "string",
"enum": ["minimal", "low", "medium", "high"],
@@ -602,10 +606,10 @@ class EvaluationSerializer(UserAccessControlSerializerMixin, serializers.ModelSe
if not isinstance(config, dict):
raise serializers.ValidationError({"evaluation_config": "Must be an object."})
if self.partial and self.instance is not None and evaluation_type == EvaluationType.LLM_JUDGE.value:
- # A prompt-only PATCH must not reset the saved judge settings.
+ # A prompt-only PATCH must not reset the saved judge settings or image opt-out.
saved = {
key: self.instance.evaluation_config[key]
- for key in ("judge_method", "reasoning_level")
+ for key in ("judge_method", "reasoning_level", "include_media")
if key not in config and key in self.instance.evaluation_config
}
data["evaluation_config"] = {**saved, **config}
diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.py
index 8d155c62fc1b..d4a88f9e76e6 100644
--- a/products/ai_observability/backend/api/test/test_evaluations.py
+++ b/products/ai_observability/backend/api/test/test_evaluations.py
@@ -137,17 +137,19 @@ class TestModelConfigurationSerializer(SimpleTestCase):
@parameterized.expand(
[
- ("llm", {}, {"judge_method": "llm"}),
- ("decision", {}, {"judge_method": "decision"}),
- ("llm", {"judge_method": None}, {}),
- ("decision", {"judge_method": "llm"}, {"judge_method": "llm"}),
- (None, {}, {}),
+ ({"judge_method": "llm"}, {}, {"judge_method": "llm"}),
+ ({"judge_method": "decision"}, {}, {"judge_method": "decision"}),
+ ({"judge_method": "llm"}, {"judge_method": None}, {}),
+ ({"judge_method": "decision"}, {"judge_method": "llm"}, {"judge_method": "llm"}),
+ ({}, {}, {}),
+ ({"include_media": False}, {}, {"include_media": False}),
+ ({"include_media": False}, {"include_media": True}, {"include_media": True}),
]
)
- def test_prompt_patch_preserves_method_unless_explicitly_changed(
- self, method: str | None, patch_config: dict[str, object], expected_method: dict[str, str]
+ def test_prompt_patch_preserves_saved_choices_unless_explicitly_changed(
+ self, saved: dict[str, object], patch_config: dict[str, object], expected_method: dict[str, object]
) -> None:
- stored_config = {"prompt": "Check the response.", **({"judge_method": method} if method else {})}
+ stored_config = {"prompt": "Check the response.", **saved}
evaluation = Evaluation(
evaluation_type="llm_judge",
evaluation_config=stored_config.copy(),
diff --git a/products/ai_observability/backend/facade/blobs.py b/products/ai_observability/backend/facade/blobs.py
new file mode 100644
index 000000000000..d46305f3be64
--- /dev/null
+++ b/products/ai_observability/backend/facade/blobs.py
@@ -0,0 +1,9 @@
+"""Facade re-export for offloaded AI blobs.
+
+Isolated from ``facade/api.py``, which imports the clustering workflows, so the eval workers can read blobs without an
+import cycle.
+"""
+
+from products.ai_observability.backend.ai_blobs import BlobPointer
+
+__all__ = ["BlobPointer"]
diff --git a/products/ai_observability/backend/llm/images.py b/products/ai_observability/backend/llm/images.py
new file mode 100644
index 000000000000..9d54236ca84d
--- /dev/null
+++ b/products/ai_observability/backend/llm/images.py
@@ -0,0 +1,52 @@
+import base64
+from collections.abc import Callable
+from dataclasses import field
+from typing import Any, Literal
+
+from posthog.dataclasses import frozen
+
+IMAGE_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/webp", "image/gif"})
+# Providers whose image blocks take a link that they download themselves.
+_URL_PROVIDERS = frozenset({"openai", "azure_openai", "anthropic", "openrouter", "gemini"})
+# Raw bytes per image. Anthropic allows 10 MB base64-encoded, about 7.5 MB raw, so keep a margin under it.
+MAX_IMAGE_BYTES = {"anthropic": 7_000_000}
+
+
+@frozen
+class ImagePart:
+ """An image in a message's content list. Each adapter converts it to its provider's image block.
+
+ Set `url` for a link the provider downloads, or `data` for inline bytes.
+ """
+
+ mime_type: str
+ url: str | None = None
+ data: bytes | None = field(default=None, repr=False)
+
+ def base64(self) -> str:
+ return base64.b64encode(self.data or b"").decode()
+
+ def data_uri(self) -> str:
+ return f"data:{self.mime_type};base64,{self.base64()}"
+
+
+def image_transport(provider: str, model: str, mime_type: str) -> Literal["url", "inline"] | None:
+ """How a provider accepts an image of this type: a link, inline base64, or not at all."""
+ if mime_type not in IMAGE_MIME_TYPES:
+ return None
+ if provider == "gemini":
+ if mime_type == "image/gif":
+ return None
+ # The Gemini 2.0 family can't fetch external links.
+ return "inline" if model.startswith("gemini-2.0") else "url"
+ return "url" if provider in _URL_PROVIDERS else "inline"
+
+
+def convert_image_parts(messages: list[dict[str, Any]], convert: Callable[[ImagePart], Any]) -> list[dict[str, Any]]:
+ """Replace each ImagePart in the messages' content lists with the provider block that `convert` returns."""
+ return [
+ {**message, "content": [convert(part) if isinstance(part, ImagePart) else part for part in message["content"]]}
+ if isinstance(message.get("content"), list)
+ else message
+ for message in messages
+ ]
diff --git a/products/ai_observability/backend/llm/providers/anthropic.py b/products/ai_observability/backend/llm/providers/anthropic.py
index 306dbb9b0f97..3cd96efa68a5 100644
--- a/products/ai_observability/backend/llm/providers/anthropic.py
+++ b/products/ai_observability/backend/llm/providers/anthropic.py
@@ -34,6 +34,7 @@ from products.ai_observability.backend.llm.errors import (
is_output_limit_error_message,
stream_error_chunk,
)
+from products.ai_observability.backend.llm.images import ImagePart, convert_image_parts
from products.ai_observability.backend.llm.providers.azure_openai import is_allowed_azure_endpoint
from products.ai_observability.backend.llm.types import (
AnalyticsContext,
@@ -229,7 +230,7 @@ class AnthropicAdapter:
effective_api_key = api_key or self._get_default_api_key()
client = self._create_client(effective_api_key, analytics)
- messages: Any = request.messages
+ messages: Any = convert_image_parts(request.messages, _anthropic_image_block)
system_prompt = request.system or ""
use_structured = request.response_format and issubclass(request.response_format, BaseModel)
@@ -359,7 +360,7 @@ class AnthropicAdapter:
tools = self._convert_tools(request.tools) if request.tools else None
system = request.system or ""
- messages = request.messages
+ messages = convert_image_parts(request.messages, _anthropic_image_block)
# Handle cache control for supported models
system_prompt: list[TextBlockParam] = []
@@ -638,3 +639,9 @@ class AnthropicAdapter:
prepared_messages.append(prepared_message)
return prepared_messages
+
+
+def _anthropic_image_block(part: ImagePart) -> dict[str, Any]:
+ if part.url is not None:
+ return {"type": "image", "source": {"type": "url", "url": part.url}}
+ return {"type": "image", "source": {"type": "base64", "media_type": part.mime_type, "data": part.base64()}}
diff --git a/products/ai_observability/backend/llm/providers/openai.py b/products/ai_observability/backend/llm/providers/openai.py
index 5be139fd6a08..b69b662732c0 100644
--- a/products/ai_observability/backend/llm/providers/openai.py
+++ b/products/ai_observability/backend/llm/providers/openai.py
@@ -36,6 +36,7 @@ from products.ai_observability.backend.llm.errors import (
is_output_limit_error_message,
stream_error_chunk,
)
+from products.ai_observability.backend.llm.images import ImagePart, convert_image_parts
from products.ai_observability.backend.llm.types import (
AnalyticsContext,
CompletionRequest,
@@ -310,7 +311,7 @@ You must respond with valid JSON that matches this schema:
Return ONLY the JSON object, no other text or markdown formatting."""
enhanced_messages = [{"role": "system", "content": enhanced_system}]
- enhanced_messages.extend(request.messages)
+ enhanced_messages.extend(convert_image_parts(request.messages, _openai_image_block))
create_response = client.chat.completions.create(
model=request.model,
@@ -519,7 +520,7 @@ Return ONLY the JSON object, no other text or markdown formatting."""
messages = []
if request.system:
messages.append({"role": "system", "content": request.system})
- messages.extend(request.messages)
+ messages.extend(convert_image_parts(request.messages, _openai_image_block))
return messages
def _build_analytics_kwargs(self, analytics: AnalyticsContext, client) -> dict:
@@ -570,3 +571,7 @@ Return ONLY the JSON object, no other text or markdown formatting."""
result = handler.convert_to(ToolFormat.OPENAI)
assert result is not None, "tools must be non-empty when calling _convert_tools"
return result
+
+
+def _openai_image_block(part: ImagePart) -> dict[str, Any]:
+ return {"type": "image_url", "image_url": {"url": part.url or part.data_uri()}}
diff --git a/products/ai_observability/backend/llm/providers/test/test_anthropic.py b/products/ai_observability/backend/llm/providers/test/test_anthropic.py
index 656bb0399ad2..56d0a632a23c 100644
--- a/products/ai_observability/backend/llm/providers/test/test_anthropic.py
+++ b/products/ai_observability/backend/llm/providers/test/test_anthropic.py
@@ -12,6 +12,7 @@ from products.ai_observability.backend.llm.errors import (
OutputTokenLimitError,
ProviderConfigurationError,
)
+from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.llm.providers.anthropic import (
AnthropicAdapter,
AnthropicConfig,
@@ -172,6 +173,40 @@ class TestAnthropicStreamOutputSchema:
assert "output_config" not in self._stream_kwargs(None)
+class TestAnthropicImageParts:
+ @parameterized.expand(
+ [
+ (
+ "link",
+ ImagePart(mime_type="image/png", url="https://example.com/a.png"),
+ {"type": "url", "url": "https://example.com/a.png"},
+ ),
+ (
+ "inline",
+ ImagePart(mime_type="image/png", data=b"hi"),
+ {"type": "base64", "media_type": "image/png", "data": "aGk="},
+ ),
+ ]
+ )
+ def test_image_parts_are_sent_as_image_blocks(self, _name: str, image: ImagePart, expected_source: dict):
+ request = CompletionRequest(
+ model="claude-haiku-4-5",
+ messages=[{"role": "user", "content": [image, {"type": "text", "text": "hi"}]}],
+ provider="anthropic",
+ )
+ with patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.Anthropic") as mock_cls:
+ mock_client = MagicMock()
+ mock_cls.return_value = mock_client
+ mock_client.messages.create.return_value.content = []
+ mock_client.messages.create.return_value.usage = MagicMock(input_tokens=1, output_tokens=1)
+ AnthropicAdapter().complete(request, api_key="sk-ant-test", analytics=AnalyticsContext(capture=False))
+
+ assert mock_client.messages.create.call_args.kwargs["messages"][0]["content"] == [
+ {"type": "image", "source": expected_source},
+ {"type": "text", "text": "hi"},
+ ]
+
+
def _make_bad_request_error(message: str) -> anthropic.BadRequestError:
request = httpx.Request("POST", "https://api.anthropic.com/v1/messages")
response = httpx.Response(status_code=400, request=request, json={"error": {"message": message}})
diff --git a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
index aa92919c8818..93cce2da4829 100644
--- a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
+++ b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
@@ -19,6 +19,7 @@ from products.ai_observability.backend.llm.errors import (
QuotaExceededError,
StructuredOutputParseError,
)
+from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter, OpenAIConfig
from products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema
@@ -177,6 +178,42 @@ class TestOpenAIStreamOutputSchema:
assert "response_format" not in self._stream_kwargs(None)
+class TestOpenAIImageParts:
+ @pytest.mark.parametrize(
+ ("image", "expected_url"),
+ [
+ pytest.param(
+ ImagePart(mime_type="image/png", url="https://example.com/a.png"),
+ "https://example.com/a.png",
+ id="link",
+ ),
+ pytest.param(ImagePart(mime_type="image/png", data=b"hi"), "data:image/png;base64,aGk=", id="inline"),
+ ],
+ )
+ @pytest.mark.parametrize("json_fallback", [False, True], ids=["plain", "json_fallback"])
+ def test_image_parts_are_sent_as_image_url_blocks(self, image: ImagePart, expected_url: str, json_fallback: bool):
+ request = CompletionRequest(
+ model="gpt-4.1",
+ messages=[{"role": "user", "content": [image, {"type": "text", "text": "hi"}]}],
+ provider="openai",
+ response_format=_Verdict if json_fallback else None,
+ )
+ mock_client = MagicMock()
+ mock_client.beta.chat.completions.parse.side_effect = _make_bad_request_error(
+ "response_format is not supported"
+ )
+ mock_client.chat.completions.create.return_value.usage = None
+ mock_client.chat.completions.create.return_value.choices[0].message.content = '{"verdict": true}'
+ with patch("products.ai_observability.backend.llm.providers.openai.openai.OpenAI", return_value=mock_client):
+ OpenAIAdapter().complete(request, api_key="sk-test", analytics=AnalyticsContext(capture=False))
+
+ sent = mock_client.chat.completions.create.call_args.kwargs["messages"]
+ assert next(message for message in sent if message["role"] == "user")["content"] == [
+ {"type": "image_url", "image_url": {"url": expected_url}},
+ {"type": "text", "text": "hi"},
+ ]
+
+
class TestOpenAICompleteReasoningEffort:
@parameterized.expand(
[
diff --git a/products/ai_observability/backend/models/evaluation_configs.py b/products/ai_observability/backend/models/evaluation_configs.py
index 46f2bf5dca78..cdfe11cde92f 100644
--- a/products/ai_observability/backend/models/evaluation_configs.py
+++ b/products/ai_observability/backend/models/evaluation_configs.py
@@ -30,6 +30,7 @@ class LLMJudgeConfig(BaseModel):
judge_method: Literal["llm", "decision"] | None = Field(
default=None, description="Judge method. Omit to keep the saved method; null restores automatic routing."
)
+ include_media: bool | None = Field(default=None, description="Set to false to judge text only.")
# Matches llm.types.ReasoningLevel; importing the llm package here would load provider SDKs at startup.
reasoning_level: Literal["minimal", "low", "medium", "high"] | None = Field(
default=None,
diff --git a/products/ai_observability/backend/providers/formatters/gemini_formatter.py b/products/ai_observability/backend/providers/formatters/gemini_formatter.py
index 904f7b28ee16..209e5f27aaa8 100644
--- a/products/ai_observability/backend/providers/formatters/gemini_formatter.py
+++ b/products/ai_observability/backend/providers/formatters/gemini_formatter.py
@@ -2,8 +2,9 @@ import json
import base64
from typing import Any, cast
-from google.genai.types import Blob, Content, ContentListUnion, FunctionCall, FunctionResponse, Part
+from google.genai.types import Blob, Content, ContentListUnion, FileData, FunctionCall, FunctionResponse, Part
+from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.providers.formatters.anthropic_typeguards import (
is_base64_image_param,
is_image_block_param,
@@ -84,6 +85,12 @@ def _tool_response_payload(content: Any, *, is_error: bool = False) -> dict[str,
return value if isinstance(value, dict) else {"result": value}
+def _gemini_image_part(image: ImagePart) -> Part:
+ if image.url is not None:
+ return Part(file_data=FileData(file_uri=image.url, mime_type=image.mime_type))
+ return Part(inline_data=Blob(data=image.data or b"", mime_type=image.mime_type))
+
+
def convert_anthropic_messages_to_gemini(messages: list[dict[str, Any]]) -> ContentListUnion:
contents: list[Content] = [] # Sticking to Content, as we don't support other formats yet
# Gemini keys a function response by function name, but Anthropic-style tool_result
@@ -96,7 +103,9 @@ def convert_anthropic_messages_to_gemini(messages: list[dict[str, Any]]) -> Cont
parts.append(Part(text=message["content"]))
elif isinstance(message["content"], list):
for block in message["content"]:
- if is_text_block_param(block):
+ if isinstance(block, ImagePart):
+ parts.append(_gemini_image_part(block))
+ elif is_text_block_param(block):
parts.append(Part(text=block["text"]))
elif is_image_block_param(block):
if not is_base64_image_param(block["source"]):
diff --git a/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py b/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py
index bca4b7dd194d..676458ec055f 100644
--- a/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py
+++ b/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py
@@ -2,9 +2,10 @@ from typing import Any, cast
import pytest
-from google.genai.types import Content, FunctionCall, FunctionResponse, Part
+from google.genai.types import Blob, Content, FileData, FunctionCall, FunctionResponse, Part
from parameterized import parameterized
+from products.ai_observability.backend.llm.images import ImagePart
from products.ai_observability.backend.providers.formatters.gemini_formatter import (
MessageConversionError,
convert_anthropic_messages_to_gemini,
@@ -92,6 +93,25 @@ class TestConvertAnthropicMessagesToGemini:
assert function_response.name == "get_weather"
assert function_response.response == expected_response
+ @parameterized.expand(
+ [
+ (
+ "link",
+ ImagePart(mime_type="image/png", url="https://example.com/a.png"),
+ Part(file_data=FileData(file_uri="https://example.com/a.png", mime_type="image/png")),
+ ),
+ (
+ "inline",
+ ImagePart(mime_type="image/png", data=b"hi"),
+ Part(inline_data=Blob(data=b"hi", mime_type="image/png")),
+ ),
+ ]
+ )
+ def test_image_part_becomes_gemini_part(self, _name: str, image: ImagePart, expected: Part):
+ contents = _convert([{"role": "user", "content": [image]}])
+
+ assert _part(contents[0], 0) == expected
+
def test_first_function_call_in_a_turn_gets_the_skip_signature(self):
contents = _convert(
[
diff --git a/products/ai_observability/backend/text_repr/formatters/media_blocks.py b/products/ai_observability/backend/text_repr/formatters/media_blocks.py
new file mode 100644
index 000000000000..210d9eb50ab0
--- /dev/null
+++ b/products/ai_observability/backend/text_repr/formatters/media_blocks.py
@@ -0,0 +1,115 @@
+import re
+from collections.abc import Callable
+
+from posthog.dataclasses import frozen
+
+_IMAGE_TYPES = frozenset({"image", "image_url", "input_image"})
+_AUDIO_TYPES = frozenset({"audio", "input_audio"})
+_FILE_TYPES = frozenset({"document", "file", "input_file"})
+_GEMINI_DATA_KEYS = ("inline_data", "inlineData", "file_data", "fileData")
+_OTEL_TYPES = frozenset({"blob", "uri"})
+_LINK_SCHEMES = ("https://", "http://", "phaiblob://")
+# Without full capture the SDKs replace the bytes with a placeholder such as `[base64 image redacted]`.
+# Same pattern as the frontend's `isRedactedMediaSentinel`.
+_REDACTED_RE = re.compile(r"^\s*\[base64\b[^\]]*\bredacted\]\s*$", re.IGNORECASE)
+
+
+@frozen
+class MediaBlock:
+ """An image, audio, video or file content block, and where its bytes are.
+
+ `url` holds an https link or a `phaiblob://` pointer, `base64_data` holds inline bytes.
+ """
+
+ modality: str
+ mime_type: str | None = None
+ url: str | None = None
+ base64_data: str | None = None
+
+ def marker(self) -> str:
+ # A web link can tell the judge what the media shows. Base64 and pointers are token noise.
+ if self.url is not None and self.url.startswith(("https://", "http://")):
+ return f"[{self.modality}: {self.url}]"
+ return f"[{self.modality}]"
+
+
+# Turns a media block into the text a transcript shows in its place, for example `[image 1]`.
+MediaMarker = Callable[[MediaBlock], str]
+
+
+def parse_media_block(block: dict) -> MediaBlock | None:
+ """Read an OpenAI chat or Responses, Anthropic, Gemini, or OTel GenAI media block."""
+ # Before the `type` check: the PostHog Gemini SDK records `{"type": "image", "inline_data": {...}}`.
+ for key in _GEMINI_DATA_KEYS:
+ data = block.get(key)
+ if isinstance(data, dict):
+ mime_type = data.get("mime_type") or data.get("mimeType")
+ mime_type = mime_type if isinstance(mime_type, str) else None
+ modality = mime_type.split("/")[0] if mime_type else ""
+ if modality not in ("image", "audio", "video"):
+ modality = "file"
+ uri = data.get("file_uri") or data.get("fileUri")
+ if isinstance(uri, str):
+ # A file URI is never inline bytes, and a `gs://` URI is not a link anyone else can fetch.
+ return MediaBlock(
+ modality=modality, mime_type=mime_type, url=uri if uri.startswith(_LINK_SCHEMES) else None
+ )
+ return _with_source(modality, mime_type, data.get("data"))
+ block_type = block.get("type")
+ if isinstance(block_type, str):
+ if block_type in _OTEL_TYPES:
+ return _otel_part(block)
+ if block_type in _IMAGE_TYPES:
+ return _image_block(block)
+ if block_type in _AUDIO_TYPES:
+ return MediaBlock(modality="audio")
+ if block_type in _FILE_TYPES:
+ return MediaBlock(modality="file")
+ return None
+
+
+def _otel_part(part: dict) -> MediaBlock:
+ mime_type = part.get("mime_type")
+ mime_type = mime_type if isinstance(mime_type, str) else None
+ modality = part.get("modality")
+ if not isinstance(modality, str) or not modality:
+ modality = mime_type.split("/")[0] if mime_type else "media"
+ if part["type"] == "blob":
+ return _with_source(modality, mime_type, part.get("content"))
+ uri = part.get("uri")
+ # A `uri` part never holds inline bytes, and a `gs://` URI is not a link anyone else can fetch.
+ return MediaBlock(
+ modality=modality,
+ mime_type=mime_type,
+ url=uri if isinstance(uri, str) and uri.startswith(_LINK_SCHEMES) else None,
+ )
+
+
+def _image_block(block: dict) -> MediaBlock:
+ image_url = block.get("image_url")
+ if isinstance(image_url, dict):
+ image_url = image_url.get("url")
+ source = block.get("source")
+ if isinstance(source, dict):
+ media_type = source.get("media_type")
+ return _with_source(
+ "image", media_type if isinstance(media_type, str) else None, source.get("data") or source.get("url")
+ )
+ # The Gemini Interactions API puts `data` and `mime_type` on the block itself.
+ mime_type = block.get("mime_type") or block.get("mimeType")
+ return _with_source("image", mime_type if isinstance(mime_type, str) else None, image_url or block.get("data"))
+
+
+def _with_source(modality: str, mime_type: str | None, value: object) -> MediaBlock:
+ if not isinstance(value, str) or not value:
+ return MediaBlock(modality=modality, mime_type=mime_type)
+ if value.startswith("data:") and ";base64," in value:
+ # Unwrap one envelope only. Its payload is bytes or a wrapped pointer, never another data URI,
+ # and unwrapping nested prefixes would copy the payload once per level.
+ header, value = value[len("data:") :].split(";base64,", 1)
+ mime_type = header or mime_type
+ if _REDACTED_RE.match(value) or value.startswith("data:"):
+ return MediaBlock(modality=modality, mime_type=mime_type)
+ if value.startswith(_LINK_SCHEMES):
+ return MediaBlock(modality=modality, mime_type=mime_type, url=value)
+ return MediaBlock(modality=modality, mime_type=mime_type, base64_data=value)
diff --git a/products/ai_observability/backend/text_repr/formatters/message_formatter.py b/products/ai_observability/backend/text_repr/formatters/message_formatter.py
index 49f9e69cd75a..430fa59958c5 100644
--- a/products/ai_observability/backend/text_repr/formatters/message_formatter.py
+++ b/products/ai_observability/backend/text_repr/formatters/message_formatter.py
@@ -32,7 +32,8 @@ from .constants import (
SAMPLING_REDUCTION_FACTOR,
SPECIAL_BLOCK_TYPES,
)
-from .otel_parts import flatten_parts_message
+from .media_blocks import MediaMarker, parse_media_block
+from .otel_parts import MEDIA_PART_KEY, flatten_parts_message
from .tool_formatter import format_tools
@@ -47,6 +48,7 @@ class FormatterOptions(TypedDict, total=False):
include_line_numbers: bool # Prefix each line with line number (default: False)
max_length: int | None # Max output length; randomly drop lines if exceeded (default: None)
max_render_length: int # Abort oversized renders before sampling or assembling the full transcript
+ media_marker: MediaMarker # Render media blocks with this instead of dropping them (LLM judges with images)
class ToolCall(TypedDict, total=False):
@@ -344,7 +346,7 @@ def _string_type(msg: dict[str, Any]) -> str | None:
return msg_type if isinstance(msg_type, str) else None
-def safe_extract_text(content: Any) -> str:
+def safe_extract_text(content: Any, media_marker: MediaMarker | None = None) -> str:
"""
Safely extract text from various content formats.
Handles strings, dicts with 'text' key, arrays of content blocks, etc.
@@ -360,7 +362,9 @@ def safe_extract_text(content: Any) -> str:
if "text" in content:
return str(content["text"])
if "content" in content:
- return safe_extract_text(content["content"])
+ return safe_extract_text(content["content"], media_marker)
+ if media_marker is not None and (media := parse_media_block(content)) is not None:
+ return media_marker(media)
# Fallback to JSON representation
return json.dumps(content)
@@ -371,6 +375,13 @@ def safe_extract_text(content: Any) -> str:
item_type = _string_type(item)
# Try both "text" and "content" keys (tool_result uses "content")
text_value = item.get("text") or item.get("content")
+ if media_marker is not None:
+ if (media := parse_media_block(item)) is not None:
+ text_value = media_marker(media)
+ item_type = None
+ elif isinstance(text_value, list):
+ # Tool results can hold image blocks.
+ text_value = safe_extract_text(text_value, media_marker)
if text_value:
# Add spacing between all content blocks in the array
@@ -421,7 +432,7 @@ def _is_special_block(block: Any) -> bool:
return False
-def _format_special_block(block: dict) -> str | None:
+def _format_special_block(block: dict, media_marker: MediaMarker | None = None) -> str | None:
"""
Format special blocks (tool calls, tool_use, etc).
@@ -467,10 +478,10 @@ def _format_special_block(block: dict) -> str | None:
return f"[Tool result: {tool_name}]"
# Block not handled by this function
- return safe_extract_text(block)
+ return safe_extract_text(block, media_marker)
-def extract_text_content(content: Any) -> str:
+def extract_text_content(content: Any, media_marker: MediaMarker | None = None) -> str:
"""
Extract text content from various message content formats.
Uses safe extraction with fallback for unparseable content.
@@ -484,7 +495,7 @@ def extract_text_content(content: Any) -> str:
text_parts: list[str] = []
for block in content:
if isinstance(block, dict):
- formatted = _format_special_block(block)
+ formatted = _format_special_block(block, media_marker)
if formatted is not None and formatted:
text_parts.append(formatted)
# Handle non-dict items in list
@@ -495,10 +506,10 @@ def extract_text_content(content: Any) -> str:
return "\n\n".join(text_parts)
# Use safe extraction for non-special content (handles type labels for text/reasoning/etc)
- return safe_extract_text(content)
+ return safe_extract_text(content, media_marker)
-def extract_payload_text(payload: Any) -> str:
+def extract_payload_text(payload: Any, media_marker: MediaMarker | None = None) -> str:
"""Extract text from a tool payload, returning "" only when nothing was recorded.
A falsey scalar such as `0` or `False` is a real result, so it must not collapse to "" the
@@ -513,7 +524,7 @@ def extract_payload_text(payload: Any) -> str:
# `extract_text_content`'s repr fallback would render an empty container as a literal
# "[]" / "{}", which reads as recorded output.
return ""
- return extract_text_content(payload)
+ return extract_text_content(payload, media_marker)
def _is_responses_item(msg: dict[str, Any]) -> bool:
@@ -596,7 +607,8 @@ def _format_responses_item(msg: dict[str, Any], options: FormatterOptions | None
if item_type in RESPONSES_TOOL_OUTPUT_TYPES:
output = msg.get("output")
- text = extract_payload_text(msg.get("content") if output is None else output)
+ media_marker = options.get("media_marker") if options else None
+ text = extract_payload_text(msg.get("content") if output is None else output, media_marker)
if not text:
return [MISSING_TOOL_OUTPUT_NOTE]
lines, _ = truncate_content(text, options)
@@ -648,10 +660,14 @@ def _format_message_body(msg: dict[str, Any], options: FormatterOptions | None)
responses_lines = _format_responses_item(msg, options)
content = msg.get("content", "")
- if responses_lines is not None:
+ media_marker = options.get("media_marker") if options else None
+ media_part = msg.get(MEDIA_PART_KEY)
+ if media_marker is not None and isinstance(media_part, dict) and (media := parse_media_block(media_part)):
+ lines.append(media_marker(media))
+ elif responses_lines is not None:
lines.extend(responses_lines)
elif content:
- text_content = extract_text_content(content)
+ text_content = extract_text_content(content, media_marker)
if text_content:
content_lines, _ = truncate_content(text_content, options)
lines.extend(content_lines)
diff --git a/products/ai_observability/backend/text_repr/formatters/otel_parts.py b/products/ai_observability/backend/text_repr/formatters/otel_parts.py
index 388e1162fcdd..51c040208df6 100644
--- a/products/ai_observability/backend/text_repr/formatters/otel_parts.py
+++ b/products/ai_observability/backend/text_repr/formatters/otel_parts.py
@@ -1,5 +1,10 @@
import json
+from .media_blocks import parse_media_block
+
+# A media part's flattened message keeps the original part here, so a judge with images can attach it.
+MEDIA_PART_KEY = "media_part"
+
def flatten_parts_message(msg: dict) -> list[dict]:
"""Convert an OTel GenAI semconv parts-shaped message into flat chat message(s).
@@ -27,7 +32,7 @@ def flatten_parts_message(msg: dict) -> list[dict]:
- `blob`/`uri`/`file` parts become bracketed text markers (`[image]`,
`[image: <uri>]`, `[file: <id>]`). The trace view renders image blobs and uris
as actual images; the judge gets markers instead because base64 payloads are
- token noise to an LLM
+ token noise to an LLM. The original part stays under `MEDIA_PART_KEY`
- `compaction` parts become follow-up messages carrying the compacted summary,
or a `[conversation compacted]` marker when the summary is absent
@@ -76,7 +81,9 @@ def flatten_parts_message(msg: dict) -> list[dict]:
if isinstance(reasoning, str) and reasoning:
followups.append({"role": "thinking", "content": reasoning})
elif part_type in ("blob", "uri", "file"):
- followups.append({"role": msg["role"], "content": _media_part_marker(part_type, part)})
+ followups.append(
+ {"role": msg["role"], "content": _media_part_marker(part_type, part), MEDIA_PART_KEY: part}
+ )
elif part_type == "compaction":
summary = part.get("content")
if not (isinstance(summary, str) and summary):
@@ -101,13 +108,9 @@ def flatten_parts_message(msg: dict) -> list[dict]:
def _media_part_marker(part_type: str, part: dict) -> str:
- modality = part.get("modality")
- if not isinstance(modality, str) or not modality:
- modality = "media"
- if part_type == "blob":
- return f"[{modality}]"
- if part_type == "uri":
- uri = part.get("uri")
- return f"[{modality}: {uri}]" if isinstance(uri, str) and uri else f"[{modality}]"
+ if part_type in ("blob", "uri"):
+ # The same rule a judge with images falls back to, so both transcripts match where no image attaches.
+ media = parse_media_block(part)
+ return media.marker() if media else "[media]"
file_id = part.get("file_id")
return f"[file: {file_id}]" if isinstance(file_id, str) and file_id else "[file]"
diff --git a/products/ai_observability/backend/text_repr/formatters/span_formatter.py b/products/ai_observability/backend/text_repr/formatters/span_formatter.py
index 80d699f813b5..7e24b6f5e023 100644
--- a/products/ai_observability/backend/text_repr/formatters/span_formatter.py
+++ b/products/ai_observability/backend/text_repr/formatters/span_formatter.py
@@ -105,7 +105,7 @@ def _format_tool_result(state: dict, options: FormatterOptions | None) -> list[s
lines.append("")
# `content` is often a list of content blocks, which `str()` would render as a Python repr.
- text = extract_payload_text(content)
+ text = extract_payload_text(content, options.get("media_marker") if options else None)
if text:
content_lines, _ = truncate_content(text, options)
lines.extend(content_lines)
diff --git a/products/ai_observability/backend/text_repr/formatters/test/test_event_formatter.py b/products/ai_observability/backend/text_repr/formatters/test/test_event_formatter.py
index e24a9b19dacb..0f7aa32d3bf0 100644
--- a/products/ai_observability/backend/text_repr/formatters/test/test_event_formatter.py
+++ b/products/ai_observability/backend/text_repr/formatters/test/test_event_formatter.py
@@ -475,3 +475,10 @@ class TestSpanToolResultFormatting:
result = format_event_text_repr(self._span(state))
assert "3 issues found" in result
assert "'type': 'text'" not in result
+
+ def test_tool_result_images_reach_the_media_marker(self):
+ image = {"type": "input_image", "image_url": "data:image/png;base64,aGk="}
+ state = {"type": "tool", "name": "screenshot", "status": "success", "content": [image]}
+ result = format_event_text_repr(self._span(state), {"media_marker": lambda media: "[image 1]"})
+ assert "[image 1]" in result
+ assert "aGk=" not in result
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx
index 1645b679a376..f75f3d11e37b 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx
@@ -1,6 +1,8 @@
import type { Meta, StoryObj } from '@storybook/react'
import { BindLogic } from 'kea'
+import { FEATURE_FLAGS } from 'lib/constants'
+
import { useStorybookMocks } from '~/mocks/browser'
import { llmEvaluationLogic } from '../llmEvaluationLogic'
@@ -119,6 +121,10 @@ export const DecisionNarrow: Story = { args: { method: 'decision', narrow: true
export const NoDecisionModels: Story = { args: { method: 'decision', empty: true } }
export const NativeOpenAI: Story = { args: { method: 'decision', nativeOpenAI: true } }
export const NativeOpenAINarrow: Story = { args: { method: 'decision', nativeOpenAI: true, narrow: true } }
+export const WithAttachments: Story = {
+ args: { method: 'llm' },
+ parameters: { featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_JUDGE_IMAGES], testOptions: { snapshotBrowsers: [] } },
+}
export const ReasoningLevelNarrow: Story = {
args: { method: 'llm', nativeOpenAI: true, narrow: true, reasoningLevel: 'medium' },
}
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx
index 79801c6a2d87..55b6aa67405e 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx
@@ -1,8 +1,10 @@
import { useActions, useValues } from 'kea'
-import { LemonSegmentedButton } from '@posthog/lemon-ui'
+import { LemonSegmentedButton, LemonSwitch } from '@posthog/lemon-ui'
+import { FEATURE_FLAGS } from 'lib/constants'
import { LemonField } from 'lib/lemon-ui/LemonField'
+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { modelPickerLogic } from '../../modelPickerLogic'
import { booleanOutputConfigError, decisionModelHint } from '../constants'
@@ -13,8 +15,9 @@ import { EvaluationJudgeModelPicker } from './EvaluationJudgeModelPicker'
export function EvaluationModelPicker(): JSX.Element {
const { byokModelsLoading, providerKeysLoading } = useValues(modelPickerLogic)
- const { evaluation, usesDecisionModel, judgeMethod } = useValues(llmEvaluationLogic)
- const { setJudgeMethod } = useActions(llmEvaluationLogic)
+ const { evaluation, usesDecisionModel, judgeMethod, includeMedia } = useValues(llmEvaluationLogic)
+ const { setJudgeMethod, setIncludeMedia } = useActions(llmEvaluationLogic)
+ const { featureFlags } = useValues(featureFlagLogic)
const loading = byokModelsLoading || providerKeysLoading
@@ -61,6 +64,22 @@ export function EvaluationModelPicker(): JSX.Element {
)}
</div>
</LemonField.Pure>
+ {featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_JUDGE_IMAGES] &&
+ evaluation?.evaluation_type === 'llm_judge' &&
+ !usesDecisionModel && (
+ <LemonField.Pure label="Attachments">
+ <LemonSwitch
+ checked={includeMedia}
+ onChange={setIncludeMedia}
+ bordered
+ label="Send to the judge"
+ data-attr="llma-evaluation-include-attachments"
+ />
+ <p className="text-muted text-sm">
+ The judge sees the images in what it evaluates. Other file types are not supported yet.
+ </p>
+ </LemonField.Pure>
+ )}
{evaluation?.evaluation_type === 'llm_judge' &&
evaluation.output_type === 'boolean' &&
usesDecisionModel && (
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
index 5b58de8384bc..f3553e77ca71 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts
@@ -444,6 +444,20 @@ describe('llmEvaluationLogic', () => {
})
})
+ it('sends an explicit attachments choice so turning it back on is saved', async () => {
+ await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])
+
+ logic.actions.setIncludeMedia(false)
+ expect(logic.values.evaluation?.evaluation_config).toEqual(
+ expect.objectContaining({ include_media: false })
+ )
+ expect(logic.values.includeMedia).toBe(false)
+
+ logic.actions.setIncludeMedia(true)
+ expect(logic.values.evaluation?.evaluation_config).toEqual(expect.objectContaining({ include_media: true }))
+ expect(logic.values.includeMedia).toBe(true)
+ })
+
it('records the polarity on the boolean output config', async () => {
await expectLogic(logic, () => {
logic.actions.loadEvaluationSuccess({ ...mockEvaluation })
diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
index 502d0b0cc659..e798e7ee913e 100644
--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts
@@ -343,6 +343,7 @@ export interface llmEvaluationLogicValues {
hogTestMessage: string | null
hogTestResults: TestHogResultItemApi[] | null
hogTestResultsLoading: boolean
+ includeMedia: boolean
isForceRefresh: boolean
isNewEvaluation: boolean
isReportableEvaluation: boolean
@@ -536,6 +537,9 @@ export interface llmEvaluationLogicActions {
setHogTestMessage: (message: string | null) => {
message: string | null
}
+ setIncludeMedia: (includeMedia: boolean) => {
+ includeMedia: boolean
+ }
setJudgeMethod: (judgeMethod: JudgeMethod) => {
judgeMethod: JudgeMethod
}
@@ -607,6 +611,7 @@ export interface llmEvaluationLogicMeta {
selectedModelOption: (evaluation: EvaluationConfig | null, byokModels: ModelOption[]) => ModelOption | null
judgeMethod: (evaluation: EvaluationConfig | null, selectedModelOption: ModelOption | null) => JudgeMethod
usesDecisionModel: (judgeMethod: JudgeMethod) => boolean
+ includeMedia: (evaluation: EvaluationConfig | null) => boolean
reasoningLevel: (evaluation: EvaluationConfig | null) => ReasoningLevel | null
selectedModelSupportsReasoning: (selectedModelOption: ModelOption | null, usesDecisionModel: boolean) => boolean
selectedModelName: (
@@ -704,6 +709,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
setTriggerConditions: (conditions: EvaluationConditionSet[]) => ({ conditions }),
setModelConfiguration: (modelConfiguration: ModelConfiguration | null) => ({ modelConfiguration }),
setJudgeMethod: (judgeMethod: JudgeMethod) => ({ judgeMethod }),
+ setIncludeMedia: (includeMedia: boolean) => ({ includeMedia }),
setReasoningLevel: (reasoningLevel: ReasoningLevel | null) => ({ reasoningLevel }),
setEvaluationType: (evaluationType: EvaluationType) => ({ evaluationType }),
setEvaluationTarget: (target: EvaluationTarget) => ({ target }),
@@ -1042,6 +1048,11 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
: state.output_config,
}
: state,
+ // Always explicit: a PATCH without the field keeps the saved value.
+ setIncludeMedia: (state, { includeMedia }) =>
+ state && isLLMJudgeEvaluation(state)
+ ? { ...state, evaluation_config: { ...state.evaluation_config, include_media: includeMedia } }
+ : state,
setEvaluationType: (state, { evaluationType }) => {
if (!state) {
return null
@@ -1192,6 +1203,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
setTriggerConditions: () => true,
setModelConfiguration: () => true,
setJudgeMethod: () => true,
+ setIncludeMedia: () => true,
setReasoningLevel: () => true,
setEvaluationType: () => true,
setEvaluationTarget: () => true,
@@ -1776,6 +1788,11 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([
},
],
usesDecisionModel: [(s) => [s.judgeMethod], (judgeMethod: JudgeMethod): boolean => judgeMethod === 'decision'],
+ includeMedia: [
+ (s) => [s.evaluation],
+ (evaluation: EvaluationConfig | null): boolean =>
+ !isLLMJudgeEvaluation(evaluation) || evaluation.evaluation_config?.include_media !== false,
+ ],
reasoningLevel: [
(s) => [s.evaluation],
(evaluation: EvaluationConfig | null): ReasoningLevel | null =>
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index c352b9432d93..22e5876ab9f2 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -2363,6 +2363,8 @@ export type EvaluationApiEvaluationConfig =
* @nullable
*/
judge_method?: 'llm' | 'decision' | null
+ /** Set to false to judge text only. */
+ include_media?: boolean
/**
* How much the judge model reasons before it answers. Only Anthropic and OpenAI reasoning models with the 'llm' judge method support it. Omit to keep the saved level; null uses the model's default.
* @nullable
@@ -2697,6 +2699,8 @@ export type PatchedEvaluationApiEvaluationConfig =
* @nullable
*/
judge_method?: 'llm' | 'decision' | null
+ /** Set to false to judge text only. */
+ include_media?: boolean
/**
* How much the judge model reasons before it answers. Only Anthropic and OpenAI reasoning models with the 'llm' judge method support it. Omit to keep the saved level; null uses the model's default.
* @nullable
diff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.ts
index 91e12dbbdce9..477a9fa04acc 100644
--- a/products/ai_observability/frontend/generated/api.zod.ts
+++ b/products/ai_observability/frontend/generated/api.zod.ts
@@ -620,6 +620,7 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod
.enum(['llm', 'decision'])
.nullish()
.describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),
+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),
reasoning_level: zod
.enum(['minimal', 'low', 'medium', 'high'])
.nullish()
@@ -1021,6 +1022,7 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod
.enum(['llm', 'decision'])
.nullish()
.describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),
+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),
reasoning_level: zod
.enum(['minimal', 'low', 'medium', 'high'])
.nullish()
@@ -1325,6 +1327,7 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod
.enum(['llm', 'decision'])
.nullish()
.describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),
+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),
reasoning_level: zod
.enum(['minimal', 'low', 'medium', 'high'])
.nullish()
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 91fc3813119d..62de6013b5ed 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -9228,6 +9228,7 @@ export namespace Schemas {
judge_method?:
| (("llm" | "decision") | null)
| undefined;
+ include_media?: boolean | undefined;
reasoning_level?:
| (("minimal" | "low" | "medium" | "high") | null)
| undefined;
@@ -19483,6 +19484,7 @@ export namespace Schemas {
| {
prompt: string;
judge_method?: (("llm" | "decision") | null) | undefined;
+ include_media?: boolean | undefined;
reasoning_level?:
| (("minimal" | "low" | "medium" | "high") | null)
| undefined;
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 508a1bcacbba..11e7cdeec398 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -45316,6 +45316,8 @@ export namespace Schemas {
* @nullable
*/
judge_method?: 'llm' | 'decision' | null;
+ /** Set to false to judge text only. */
+ include_media?: boolean;
/**
* How much the judge model reasons before it answers. Only Anthropic and OpenAI reasoning models with the 'llm' judge method support it. Omit to keep the saved level; null uses the model's default.
* @nullable
@@ -87695,6 +87697,8 @@ export namespace Schemas {
* @nullable
*/
judge_method?: 'llm' | 'decision' | null;
+ /** Set to false to judge text only. */
+ include_media?: boolean;
/**
* How much the judge model reasons before it answers. Only Anthropic and OpenAI reasoning models with the 'llm' judge method support it. Omit to keep the saved level; null uses the model's default.
* @nullable
diff --git a/services/mcp/src/generated/ai_observability/api.ts b/services/mcp/src/generated/ai_observability/api.ts
index 7bdbe19643b3..4944eee9f83f 100644
--- a/services/mcp/src/generated/ai_observability/api.ts
+++ b/services/mcp/src/generated/ai_observability/api.ts
@@ -1435,6 +1435,7 @@ export const EvaluationsCreateBody = () => zod
.enum(['llm', 'decision'])
.nullish()
.describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),
+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),
reasoning_level: zod
.enum(['minimal', 'low', 'medium', 'high'])
.nullish()
@@ -1916,6 +1917,7 @@ export const EvaluationsPartialUpdateBody = () => zod
.enum(['llm', 'decision'])
.nullish()
.describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),
+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),
reasoning_level: zod
.enum(['minimal', 'low', 'medium', 'high'])
.nullish()
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
index 2211a7d9905a..7e7e386739ed 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json
@@ -63,6 +63,10 @@
"anyOf": [
{
"properties": {
+ "include_media": {
+ "description": "Set to false to judge text only.",
+ "type": "boolean"
+ },
"judge_method": {
"anyOf": [
{
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
index d4eba03e9095..e0d197215fc8 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json
@@ -62,6 +62,10 @@
"anyOf": [
{
"properties": {
+ "include_media": {
+ "description": "Set to false to judge text only.",
+ "type": "boolean"
+ },
"judge_method": {
"anyOf": [
{