:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): let llm judges see images (#124879)
Bernat Torresb30fcfa45fcf
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex 12b24733e73b..9ce5903e0806 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -42,6 +42,9 @@ If the assembled fallback session still exceeds 500,000 characters, the evaluatiGeneration evaluations extract message text without the per-message character cutoff.They reserve space for the complete output, then sample input and tool definitions into the remaining space within the 150,000-character limit.If the output and section labels alone exceed that limit, the run is skipped with reason `generation_output_too_long` without calling the judge.+An exception applies when the judge sees images (the `llm-analytics-judge-images` flag is on and the evaluation does not turn attachments off).+Then each image becomes an `[image N]` marker instead of its base64 or link, so an output with large inline images can fit and the run continues.+The image labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 images, plus one summary line for the rest.Generation evaluation and LLM tagger user prompts repair UTF-16 surrogate sequences after sampling.Lone surrogates become replacement characters (`�`), while valid pairs become their Unicode character.diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsxindex cdbbcc37ab83..e5f8b02c96e6 100644--- a/frontend/src/lib/constants.tsx+++ b/frontend/src/lib/constants.tsx@@ -359,8 +359,8 @@ export const FEATURE_FLAGS = {INBOX_MCP_USED: 'inbox-mcp-used', // owner: #team-self-driving, hides the MCP banner after confirmed Inbox tool useINBOX_MODEL_SORT: 'inbox-model-sort', // owner: #team-self-driving, gates the ranking model sort options and the score tag on inbox cardsINBOX_REPORT_FEEDBACK_BANG: 'inbox-report-feedback-bang', // owner: #team-self-driving, gates the party hog that pops on a report thumbs-up in the inbox and on the Today home page- INBOX_SOURCE_FILTER: 'inbox-source-filter', // owner: #team-self-driving, gates the Source filter on the inbox report listINBOX_SLACK_NOTIFICATIONS: 'inbox-slack-notifications', // owner: #team-self-driving, gates the Slack notifications config card in the inbox+ INBOX_SOURCE_FILTER: 'inbox-source-filter', // owner: #team-self-driving, gates the Source filter on the inbox report listINBOX_TIME_WINDOW: 'inbox-time-window', // owner: #team-self-driving, gates the created-in window filter on the inbox report listINSIGHT_ALERTS_EMPTY_STATE_EXPERIMENT: 'insight-alerts-empty-state',INSIGHT_DATA_CHECKS: 'insight-data-checks', // owner: @sampennington #team-product-analytics, checks the events behind an insight and shows the result on the insight page@@ -374,6 +374,7 @@ export const FEATURE_FLAGS = {LLM_ANALYTICS_COMMUNITY_SKILLS: 'llm-analytics-community-skills', // owner: #team-ai-observabilityLLM_ANALYTICS_DATASETS: 'llm-analytics-datasets', // owner: #team-ai-observability #team-posthog-aiLLM_ANALYTICS_EVAL_SETTLING_STRATEGY: 'llm-analytics-eval-settling-strategy', // owner: #team-ai-observability+ LLM_ANALYTICS_JUDGE_IMAGES: 'llm-analytics-judge-images', // owner: #team-ai-observabilityLLM_ANALYTICS_TAGS: 'llm-analytics-tags', // owner: #team-ai-observabilityLLM_ANALYTICS_TRACE_NAVIGATION: 'llm-analytics-trace-navigation', // owner: #team-ai-observabilityLLM_OBSERVABILITY_TRACE_SEARCH: 'llm-observability-trace-search', // owner: #team-ai-observabilitydiff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.pyindex 312383fed0c0..2b474a65e0b4 100644--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py@@ -1,6 +1,6 @@import jsonimport math-from dataclasses import dataclass+from dataclasses import dataclass, replacefrom datetime import timedeltafrom typing import Annotated, Any, Literal@@ -13,6 +13,7 @@ from temporalio.common import RetryPolicyfrom temporalio.exceptions import ApplicationErrorfrom posthog.dataclasses import frozen+from posthog.exceptions_capture import capture_exceptionfrom posthog.llm.system_one import (MAX_SCORE_LEVELS,ChoiceAnswer,@@ -40,6 +41,7 @@ from posthog.temporal.ai_observability.evaluation_types import (build_generation_not_found_result,build_skipped_evaluation_result,)+from posthog.temporal.ai_observability.judge_images import ImagePrompt, build_image_promptfrom posthog.temporal.ai_observability.message_utils import extract_text_from_messages, format_tool_definitionsfrom posthog.temporal.ai_observability.metrics import (increment_errors,@@ -78,6 +80,7 @@ from products.ai_observability.backend.llm.errors import (UnsupportedModelError,provider_error_detail,)+from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.llm.providers.openai import OPENAI_DECISIONS_BASE_URLfrom products.ai_observability.backend.llm.providers.openai_decisions import OpenAIDecisionAdapterfrom products.ai_observability.backend.llm.providers.openrouter import OPENROUTER_DECISIONS_BASE_URL@@ -93,6 +96,7 @@ from products.ai_observability.backend.text_repr.formatters import (sanitize_surrogates,truncate_by_line_sampling,)+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaMarkerlogger = structlog.get_logger(__name__)@@ -442,6 +446,25 @@ def _build_unparsable_response_skip_result(return result+# Statuses an attached image can cause. Auth, permission and missing-model errors fail the same way without images.+_IMAGE_REJECTION_STATUSES = frozenset({400, 413, 415, 422})+++class JudgeImagesRejectedError(Exception):+ """Reported without the provider's message, which can quote a live presigned link."""+++def _image_rejection_status(error: Exception) -> int | None:+ """The status of a provider rejection an image can cause, raw or wrapped by an adapter."""+ for candidate in (error, error.__cause__):+ if (+ isinstance(candidate, Exception)+ and (status := _rejected_request_status(candidate)) in _IMAGE_REJECTION_STATUSES+ ):+ return status+ return None++def _rejected_request_status(error: Exception) -> int | None:"""The 4xx status of a provider rejection that no retry can fix, or None.@@ -558,17 +581,19 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActireturn _build_errored_trace_result(allows_na, output_type=output_type)io = extract_event_io(event_type, properties)- tools_raw = extract_event_tools(properties)-- input_data = extract_text_from_messages(io.input_raw)- output_data = extract_text_from_messages(io.output_raw)- tools_data = format_tool_definitions(tools_raw)-+ tools_data = format_tool_definitions(extract_event_tools(properties))system_prompt = build_system_prompt(prompt, allows_na, output_type=output_type, output_config=output_config)- output_section = f"\n\nOutput: {output_data}"- context_budget = EVENT_TEXT_MAX_CHARS - len("Input: ") - len(output_section)- if context_budget < 0:+ user_prompt = _build_user_prompt(io.input_raw, io.output_raw, tools_data)+ image_prompt = build_image_prompt(+ evaluation,+ user_prompt,+ lambda media_marker: _build_user_prompt(io.input_raw, io.output_raw, tools_data, media_marker=media_marker),+ )+ if user_prompt is None and image_prompt is not None:+ # Inline base64 can push the plain text over the limit when the marked text still fits.+ user_prompt = image_prompt.unnumbered_text()+ if user_prompt is None:return build_skipped_evaluation_result(output_type=output_type,allows_na=allows_na,@@ -576,22 +601,34 @@ def _execute_llm_judge_activity(inputs: ExecuteLLMJudgeInputs) -> EvaluationActiskip_reason="generation_output_too_long",)- sections = [input_data]- if tools_data:- sections.append(f"Tools available:\n{tools_data}")- # Keep the full answer being graded outside the sampled context.- context = truncate_by_line_sampling("\n\n".join(sections), context_budget)- user_prompt = sanitize_surrogates(f"Input: {context}{output_section}")-return call_llm_judge(evaluation=evaluation,system_prompt=system_prompt,user_prompt=user_prompt,allows_na=allows_na,backfill=inputs.backfill_id is not None,+ image_prompt=image_prompt,)+def _build_user_prompt(+ input_raw: Any, output_raw: Any, tools_data: str, *, media_marker: MediaMarker | None = None+) -> str | None:+ """The judge's user prompt, or None when the output alone exceeds the judge's text limit."""+ # Render the input first, so its images get the lower numbers in reading order.+ input_text = extract_text_from_messages(input_raw, media_marker)+ output_section = f"\n\nOutput: {extract_text_from_messages(output_raw, media_marker)}"+ context_budget = EVENT_TEXT_MAX_CHARS - len("Input: ") - len(output_section)+ if context_budget < 0:+ return None+ sections = [input_text]+ if tools_data:+ sections.append(f"Tools available:\n{tools_data}")+ # Keep the full answer being graded outside the sampled context.+ context = truncate_by_line_sampling("\n\n".join(sections), context_budget)+ return sanitize_surrogates(f"Input: {context}{output_section}")++def _decision_numeric_score(minimum: float, maximum: float, index: float) -> float:last_index = MAX_SCORE_LEVELS - 1if index == 0:@@ -613,6 +650,7 @@ def call_llm_judge(user_prompt: str,allows_na: bool,backfill: bool = False,+ image_prompt: ImagePrompt | None = None,) -> EvaluationActivityResult:"""Resolve the judge model/key for `evaluation` and run a single judge completion.@@ -821,16 +859,40 @@ def call_llm_judge(),)else:- response = client.complete(- CompletionRequest(- model=model,- system=system_prompt,- messages=[{"role": "user", "content": user_prompt}],- provider=provider,- response_format=response_format,- reasoning_level=evaluation["evaluation_config"].get("reasoning_level"),- )+ content = image_prompt.images.content(image_prompt.text, provider, model) if image_prompt else user_prompt+ sent_images = isinstance(content, list) and any(isinstance(part, ImagePart) for part in content)+ request = CompletionRequest(+ model=model,+ system=system_prompt,+ messages=[{"role": "user", "content": content}],+ provider=provider,+ response_format=response_format,+ reasoning_level=evaluation["evaluation_config"].get("reasoning_level"),)+ status: int | None = None+ try:+ response = client.complete(request)+ except Exception as e:+ status = _image_rejection_status(e) if sent_images else None+ if status is None:+ raise+ # Handle the rejection outside the except: the provider's error can quote a presigned link, and+ # an active exception reaches the logs through capture_exception and chains onto a retry failure.+ if status is not None:+ # A model without image input, or an image link the provider could not download.+ # Grade the plain transcript rather than skip the run.+ context = {+ "evaluation_id": evaluation["id"],+ "team_id": evaluation["team_id"],+ "provider": provider,+ "model": model,+ "status": status,+ }+ logger.warning("judge_images_rejected", **context)+ if not is_byok:+ # On a PostHog key this can mean broken presigned links, so keep it visible.+ capture_exception(JudgeImagesRejectedError(f"Provider rejected judge images ({status})"), context)+ response = client.complete(replace(request, messages=[{"role": "user", "content": user_prompt}]))except (DecisionEndpointBlockedError, ProviderConfigurationError) as e:increment_user_errors("endpoint_blocked", provider=provider)return terminal_user_error_result(diff --git a/posthog/temporal/ai_observability/judge_images.py b/posthog/temporal/ai_observability/judge_images.pynew file mode 100644index 000000000000..121420fed86b--- /dev/null+++ b/posthog/temporal/ai_observability/judge_images.py@@ -0,0 +1,230 @@+"""Images an LLM judge sees next to the transcript that refers to them as `[image N]`."""++import re+import base64+from collections import Counter+from collections.abc import Callable+from functools import lru_cache+from typing import Any, Literal++import structlog+import posthoganalytics++from posthog.dataclasses import frozen+from posthog.models.team import Team+from posthog.storage.object_storage import ObjectStorageError++from products.ai_observability.backend.facade.blobs import BlobPointer+from products.ai_observability.backend.llm.images import IMAGE_MIME_TYPES, MAX_IMAGE_BYTES, ImagePart, image_transport+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, MediaMarker++logger = structlog.get_logger(__name__)++JUDGE_IMAGES_FLAG = "llm-analytics-judge-images"++# Each image costs tokens on the customer's key, and above 20 images Anthropic tightens its per-image dimension limit.+MAX_IMAGES = 20+# Bytes sent inline in one judge request. Links don't count: the provider downloads them.+MAX_INLINE_BYTES = 15_000_000+# Covers the judge call and its retries. Sign right before the call: a link signed with temporary+# AWS credentials stops working when the credentials expire.+PRESIGNED_URL_SECONDS = 15 * 60++Outcome = Literal["link", "inline", "too_large", "unavailable"]++_NOT_ATTACHED_LABELS: dict[Outcome, str] = {+ "too_large": "not attached, too large",+ "unavailable": "could not be attached",+}+++_NUMBERED_MARKER_RE = re.compile(r"\[image \d+\]")+++@frozen+class ImagePrompt:+ """The judge transcript with `[image N]` markers, and the images those markers refer to."""++ text: str+ images: "JudgeImages"++ def unnumbered_text(self) -> str:+ """The transcript for a judge call that sends no images, so no marker points at a missing image."""+ return _NUMBERED_MARKER_RE.sub("[image]", self.text)+++@frozen+class _Attachment:+ outcome: Outcome+ part: ImagePart | None = None+ inline_bytes: int = 0+++@frozen+class _Image:+ number: int+ mime_type: str+ # An offloaded image has a pointer, a small one stays inline as base64.+ pointer: BlobPointer | None+ base64_data: str | None+++class JudgeImages:+ """Numbers the images found while formatting a transcript, then attaches them to the judge request."""++ def __init__(self, team_id: int) -> None:+ self._team_id = team_id+ self._images: list[_Image] = []+ self._numbers: dict[str, int] = {}+ # After one storage failure the rest would fail too, each after its own timeout and retry.+ self._storage_failed = False++ def marker(self, media: MediaBlock) -> str:+ pointer = BlobPointer.parse(media.url) if media.url else None+ mime_type = _normalize_mime_type(media.mime_type or (pointer.mime_type if pointer else None))+ # Links from the customer's app keep a plain marker: we never fetch them.+ key = pointer.sha256 if pointer else media.base64_data+ if media.modality != "image" or mime_type not in IMAGE_MIME_TYPES or key is None:+ return media.marker()+ if (number := self._numbers.get(key)) is None:+ number = self._numbers[key] = len(self._images) + 1+ self._images.append(+ _Image(number=number, mime_type=mime_type, pointer=pointer, base64_data=media.base64_data)+ )+ return f"[image {number}]"++ def content(self, user_prompt: str, provider: str, model: str) -> str | list[Any]:+ """The user message content: each referenced image under its label, then the transcript."""+ # Line sampling can drop a marker from a long transcript, and then its image only costs tokens.+ images = [image for image in self._images if f"[image {image.number}]" in user_prompt]+ if not images:+ return user_prompt++ # Keep the images referenced last: the newest input and the graded output, even when it repeats an old image.+ ranked = sorted(images, key=lambda image: user_prompt.rfind(f"[image {image.number}]"))+ attachments: dict[int, _Attachment] = {}+ inline_bytes = 0+ for image in reversed(ranked[-MAX_IMAGES:]):+ attachment = self._attach(image, provider, model, MAX_INLINE_BYTES - inline_bytes)+ inline_bytes += attachment.inline_bytes+ attachments[image.number] = attachment++ parts: list[Any] = [{"type": "text", "text": "The transcript refers to these images as [image N]."}]+ outcomes: Counter[str] = Counter()+ for image in images:+ if image.number not in attachments:+ continue+ attachment = attachments[image.number]+ outcomes[attachment.outcome] += 1+ if attachment.part is None:+ label = _NOT_ATTACHED_LABELS[attachment.outcome]+ parts.append({"type": "text", "text": f"Image {image.number}: {label}."})+ else:+ parts.extend([{"type": "text", "text": f"Image {image.number}:"}, attachment.part])+ # One line, not one per image: it sits outside the transcript's text budget.+ if over_limit := len(images) - len(attachments):+ outcomes["over_limit"] = over_limit+ parts.append({"type": "text", "text": f"Images not attached, over the judge's image limit: {over_limit}."})+ parts.append({"type": "text", "text": user_prompt})+ logger.info("judge_images_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)+ return parts++ def _attach(self, image: _Image, provider: str, model: str, inline_budget: int) -> _Attachment:+ transport = image_transport(provider, model, image.mime_type)+ if transport is None:+ return _Attachment(outcome="unavailable")+ provider_limit = MAX_IMAGE_BYTES.get(provider)+ if image.pointer is None:+ if len(image.base64_data or "") * 3 // 4 > min(inline_budget, provider_limit or inline_budget):+ return _Attachment(outcome="too_large")+ try:+ data = base64.b64decode(image.base64_data or "", validate=True)+ except ValueError: # binascii.Error, or a non-ASCII string+ return _Attachment(outcome="unavailable")+ return _Attachment(+ outcome="inline", part=ImagePart(mime_type=image.mime_type, data=data), inline_bytes=len(data)+ )+ if self._storage_failed:+ return _Attachment(outcome="unavailable")+ limits = [+ limit for limit in (inline_budget if transport == "inline" else None, provider_limit) if limit is not None+ ]+ try:+ return self._attach_blob(image, image.pointer, transport, min(limits, default=None))+ except ObjectStorageError:+ self._storage_failed = True+ return _Attachment(outcome="unavailable")++ def _attach_blob(+ self, image: _Image, pointer: BlobPointer, transport: Literal["url", "inline"], max_bytes: int | None+ ) -> _Attachment:+ stored_size = 0+ if max_bytes is not None:+ # Check the stored size first, so an oversized blob is never downloaded or sent.+ size = pointer.stored_size(self._team_id)+ if size is None:+ return _Attachment(outcome="unavailable")+ if size > max_bytes:+ return _Attachment(outcome="too_large")+ stored_size = size+ if transport == "url":+ url = pointer.presigned_url(self._team_id, PRESIGNED_URL_SECONDS)+ if url is None:+ return _Attachment(outcome="unavailable")+ return _Attachment(outcome="link", part=ImagePart(mime_type=image.mime_type, url=url))+ # ponytail: sequential reads, a thread pool if traces with many images get slow+ data = pointer.read(self._team_id)+ if data is None:+ return _Attachment(outcome="unavailable")+ part = ImagePart(mime_type=image.mime_type, data=data)+ return _Attachment(outcome="inline", part=part, inline_bytes=stored_size)+++def _normalize_mime_type(mime_type: str | None) -> str | None:+ if mime_type is None:+ return None+ normalized = mime_type.strip().lower()+ return "image/jpeg" if normalized == "image/jpg" else normalized+++@lru_cache(maxsize=4096)+def _project_uuid(team_id: int) -> str | None:+ project_uuid = Team.objects.filter(pk=team_id).values_list("uuid", flat=True).first()+ return str(project_uuid) if project_uuid else None+++def judge_sees_images(evaluation: dict[str, Any]) -> bool:+ if evaluation["evaluation_config"].get("include_media") is False:+ return False+ # Local evaluation keeps the check off the network. An inconclusive result means off.+ try:+ # Team ids repeat across regions and self-hosted instances, so the flag targets the project uuid.+ project_uuid = _project_uuid(evaluation["team_id"])+ if project_uuid is None:+ return False+ return bool(+ posthoganalytics.feature_enabled(+ JUDGE_IMAGES_FLAG,+ project_uuid,+ groups={"project": project_uuid},+ group_properties={"project": {"id": str(evaluation["team_id"]), "uuid": project_uuid}},+ only_evaluate_locally=True,+ send_feature_flag_events=False,+ )+ )+ except Exception:+ return False+++def build_image_prompt(+ evaluation: dict[str, Any], plain_prompt: str | None, render: Callable[[MediaMarker], str | None]+) -> ImagePrompt | None:+ """The marked prompt and its images, or None when this judge run gets the plain prompt only.++ `render` builds the same transcript as the plain prompt, with media blocks rendered by the marker it gets.+ """+ if not judge_sees_images(evaluation):+ return None+ images = JudgeImages(evaluation["team_id"])+ marked = render(images.marker)+ return ImagePrompt(text=marked, images=images) if marked is not None and marked != plain_prompt else Nonediff --git a/posthog/temporal/ai_observability/message_utils.py b/posthog/temporal/ai_observability/message_utils.pyindex 9f8c7b799117..50f922b992f3 100644--- a/posthog/temporal/ai_observability/message_utils.py+++ b/posthog/temporal/ai_observability/message_utils.py@@ -3,10 +3,12 @@import jsonfrom typing import Any, Union-from products.ai_observability.backend.text_repr.formatters.otel_parts import flatten_parts_message+from products.ai_observability.backend.text_repr.formatters.constants import RESPONSES_TOOL_OUTPUT_TYPES+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaMarker, parse_media_block+from products.ai_observability.backend.text_repr.formatters.otel_parts import MEDIA_PART_KEY, flatten_parts_message-def extract_text_from_messages(messages: Union[str, list, dict, None]) -> str:+def extract_text_from_messages(messages: Union[str, list, dict, None], media_marker: MediaMarker | None = None) -> str:"""Extract readable text from LLM message structures.@@ -22,6 +24,8 @@ def extract_text_from_messages(messages: Union[str, list, dict, None]) -> str:Returns formatted string like:"user: Hello\\nassistant: Hi there"++ With `media_marker`, media blocks render as the markers it returns, such as `[image 1]`."""if not messages:return ""@@ -36,7 +40,7 @@ def extract_text_from_messages(messages: Union[str, list, dict, None]) -> str:for msg in messages:if isinstance(msg, dict):for flattened in flatten_parts_message(msg):- rendered_msg = _render_message(flattened)+ rendered_msg = _render_message(flattened, media_marker)if rendered_msg is not None:formatted_parts.append(rendered_msg)elif isinstance(msg, str):@@ -47,7 +51,7 @@ def extract_text_from_messages(messages: Union[str, list, dict, None]) -> str:# Handle single dict message — render it the same way as a one-element# list so that role prefixes and tool_call_id correlation are surfaced# consistently regardless of the wrapper shape.- rendered_messages = [_render_message(flattened) for flattened in flatten_parts_message(messages)]+ rendered_messages = [_render_message(flattened, media_marker) for flattened in flatten_parts_message(messages)]return "\n".join(rendered for rendered in rendered_messages if rendered is not None)@@ -75,15 +79,25 @@ def _only_tool_results(msg: dict) -> bool:)-def _render_message(msg: dict) -> str | None:+def _render_message(msg: dict, media_marker: MediaMarker | None) -> str | None:"""Render a single message dict into a `role: body` line.Returns None when the message has neither a role nor any body to render(so the caller can skip it)."""+ item_type = msg.get("type")+ if media_marker is not None and isinstance(item_type, str) and item_type in RESPONSES_TOOL_OUTPUT_TYPES:+ # A Responses API tool output has no role or content, so render it as a tool message to reach its images.+ msg = {"role": "tool", "tool_call_id": msg.get("call_id"), "content": msg.get("output")}role = msg.get("role", "")content = msg.get("content", "")- text = _extract_content_text(content)+ part = msg.get(MEDIA_PART_KEY)+ media = parse_media_block(part) if media_marker is not None and isinstance(part, dict) else None+ text = (+ media_marker(media)+ if media_marker is not None and media is not None+ else _extract_content_text(content, media_marker)+ )tool_calls_text = _format_tool_calls(msg.get("tool_calls"))rendered = " ".join(part for part in (text, tool_calls_text) if part)@@ -112,13 +126,14 @@ def _render_message(msg: dict) -> str | None:return None-def _extract_content_text(content: Union[str, list, dict, None]) -> str:+def _extract_content_text(content: Union[str, list, dict, None], media_marker: MediaMarker | None = None) -> str:"""Extract text from message content, handling nested structures.Handles multiple provider formats:- Anthropic: [{"type": "text", "text": "..."}]- OpenAI Responses API: [{"text": "...", "annotations": [...]}]- Generic: [{"content": "..."}]+ - With `media_marker`, image, audio and file blocks, as markers such as `[image 1]`- Plain stringsAlways falls back to str() rather than returning empty — an LLM judge@@ -139,7 +154,14 @@ def _extract_content_text(content: Union[str, list, dict, None]) -> str:if "text" in block:text_parts.append(str(block["text"]))elif "content" in block:- text_parts.append(str(block["content"]))+ # Tool results can hold image blocks, which only a judge with images unpacks.+ nested = block["content"]+ if media_marker is not None and isinstance(nested, list):+ text_parts.append(_extract_content_text(nested, media_marker))+ else:+ text_parts.append(str(nested))+ elif media_marker is not None and (media := parse_media_block(block)) is not None:+ text_parts.append(media_marker(media))else:# Unknown block shape — stringify rather than silently droptext_parts.append(str(block))@@ -147,6 +169,9 @@ def _extract_content_text(content: Union[str, list, dict, None]) -> str:text_parts.append(block)return " ".join(text_parts)+ if media_marker is not None and isinstance(content, dict) and (media := parse_media_block(content)) is not None:+ return media_marker(media)+# Fallback: convert to stringreturn str(content)diff --git a/posthog/temporal/ai_observability/run_session_evaluation.py b/posthog/temporal/ai_observability/run_session_evaluation.pyindex 565cb4784ab8..8b683d743139 100644--- a/posthog/temporal/ai_observability/run_session_evaluation.py+++ b/posthog/temporal/ai_observability/run_session_evaluation.py@@ -42,6 +42,7 @@ from posthog.temporal.ai_observability.evaluation_payload import (should_skip_for_payload,)from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult, build_skipped_evaluation_result+from posthog.temporal.ai_observability.judge_images import build_image_promptfrom posthog.temporal.ai_observability.message_utils import extract_text_from_messagesfrom posthog.temporal.ai_observability.run_trace_evaluation import TRACE_EVENTS_LOOKBACKfrom posthog.temporal.common.utils import close_db_connections@@ -56,6 +57,7 @@ from products.ai_observability.backend.text_repr.formatters import (format_trace_text_repr,llm_trace_to_formatter_format,)+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaMarker# Sessions run ~3.5x a trace at p90 (222 events against a trace's typical handful) and the shared# formatter uniformly samples lines rather than truncating, so the trace's 150k budget would hand@@ -358,7 +360,7 @@ user's conversation, in order — according to this criteria:{config.instructions}"""-def format_session_for_judge(traces: list[LLMTrace]) -> str | None:+def format_session_for_judge(traces: list[LLMTrace], media_marker: MediaMarker | None = None) -> str | None:"""Render a session as the canonical text representation, one section per trace.Preserve generation outputs across the full session before truncating them, including@@ -386,6 +388,9 @@ def format_session_for_judge(traces: list[LLMTrace]) -> str | None:"include_line_numbers": True,"max_length": None if preserve_generation_output else per_trace_budget,}+ if media_marker is not None:+ # One marker for the whole session, so an image repeated across traces is attached once.+ options["media_marker"] = media_markersections: list[str] = []rendered_length = 0for index, trace in enumerate(traces, start=1):@@ -486,7 +491,14 @@ def execute_session_llm_judge_activity(inputs: ExecuteSessionEvaluationInputs) -if outcome.skip_reason or outcome.traces is None:return build_session_skip_result(allows_na, outcome.skip_reason or "session_not_found", output_type=output_type)- transcript = format_session_for_judge(outcome.traces)+ traces = outcome.traces+ transcript = format_session_for_judge(traces)+ image_prompt = build_image_prompt(+ evaluation, transcript, lambda media_marker: format_session_for_judge(traces, media_marker)+ )+ if transcript is None and image_prompt is not None:+ # Inline base64 can push the plain transcript over the limit when the marked one still fits.+ transcript = image_prompt.unnumbered_text()if transcript is None:return build_session_skip_result(allows_na, "session_too_long_to_judge", output_type=output_type)@@ -498,6 +510,7 @@ def execute_session_llm_judge_activity(inputs: ExecuteSessionEvaluationInputs) -user_prompt=transcript,allows_na=allows_na,backfill=inputs.backfill_id is not None,+ image_prompt=image_prompt,)diff --git a/posthog/temporal/ai_observability/run_trace_evaluation.py b/posthog/temporal/ai_observability/run_trace_evaluation.pyindex fe92da4f6bf8..3e3b71439dee 100644--- a/posthog/temporal/ai_observability/run_trace_evaluation.py+++ b/posthog/temporal/ai_observability/run_trace_evaluation.py@@ -63,6 +63,7 @@ from posthog.temporal.ai_observability.evaluation_workflow_activities import (emit_internal_telemetry_activity,fetch_evaluation_activity,)+from posthog.temporal.ai_observability.judge_images import build_image_promptfrom posthog.temporal.ai_observability.message_utils import extract_text_from_messagesfrom posthog.temporal.ai_observability.metrics import increment_emit_event_outcome, increment_errorsfrom posthog.temporal.ai_observability.run_evaluation import (@@ -92,6 +93,7 @@ from products.ai_observability.backend.text_repr.formatters import (format_trace_text_repr,llm_trace_to_formatter_format,)+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaMarkerfrom products.ai_observability.backend.text_repr.formatters.message_formatter import has_message_contentif TYPE_CHECKING:@@ -549,7 +551,7 @@ def build_trace_system_prompt({config.instructions}"""-def format_trace_for_judge(trace: LLMTrace) -> str:+def format_trace_for_judge(trace: LLMTrace, media_marker: MediaMarker | None = None) -> str:"""Serialize a trace into the canonical text representation for the LLM judge.Preserve complete generation outputs before truncating them so the judge does not grade@@ -565,6 +567,8 @@ def format_trace_for_judge(trace: LLMTrace) -> str:"max_length": None,"max_render_length": JUDGE_TRACE_MAX_CHARS,}+ if media_marker is not None:+ options["media_marker"] = media_markerfor truncated in (False, True):options["truncated"] = truncatedtry:@@ -694,14 +698,19 @@ def execute_trace_llm_judge_activity(inputs: ExecuteTraceEvaluationInputs) -> Evif not _has_judge_transcript(outcome.trace):return _build_trace_skip_result(allows_na, "trace_not_found", output_type=output_type)+ trace = outcome.trace+ user_prompt = format_trace_for_judge(trace)return call_llm_judge(evaluation=evaluation,system_prompt=build_trace_system_prompt(prompt, allows_na, output_type=output_type, output_config=output_config),- user_prompt=format_trace_for_judge(outcome.trace),+ user_prompt=user_prompt,allows_na=allows_na,backfill=inputs.backfill_id is not None,+ image_prompt=build_image_prompt(+ evaluation, user_prompt, lambda media_marker: format_trace_for_judge(trace, media_marker)+ ),)diff --git a/posthog/temporal/ai_observability/test_judge_images.py b/posthog/temporal/ai_observability/test_judge_images.pynew file mode 100644index 000000000000..3d755b001e4f--- /dev/null+++ b/posthog/temporal/ai_observability/test_judge_images.py@@ -0,0 +1,192 @@+import base64++import pytest+from unittest.mock import patch++from posthog.storage.object_storage import ObjectStorageError+from posthog.temporal.ai_observability.judge_images import MAX_IMAGES, JudgeImages++from products.ai_observability.backend.llm.images import ImagePart+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, parse_media_block++OBJECT_STORAGE = "products.ai_observability.backend.ai_blobs.object_storage"+++def _pointer(digit: str, mime: str = "image%2Fjpeg") -> MediaBlock:+ return MediaBlock(modality="image", url=f"phaiblob://v1/sha256/{digit * 64}?mime={mime}&size=63357")+++def _inline(digit: str, length: int = 4) -> MediaBlock:+ return MediaBlock(modality="image", mime_type="image/png", base64_data=digit * length)+++class TestJudgeImagesMarkers:+ def test_numbers_images_and_reuses_the_number_for_a_repeat(self):+ images = JudgeImages(team_id=7)+ inline = MediaBlock(modality="image", mime_type="image/png", base64_data="aGk=")++ markers = [images.marker(media) for media in (_pointer("a"), inline, _pointer("a"))]++ assert markers == ["[image 1]", "[image 2]", "[image 1]"]++ @pytest.mark.parametrize(+ ("media", "marker"),+ [+ pytest.param(MediaBlock(modality="audio", base64_data="aGk="), "[audio]", id="not_an_image"),+ pytest.param(_pointer("a", mime="image%2Fsvg%2Bxml"), "[image]", id="unsupported_format"),+ pytest.param(+ MediaBlock(modality="image", mime_type="image/png", url="https://example.com/a.png"),+ "[image: https://example.com/a.png]",+ id="customer_link",+ ),+ pytest.param(+ parse_media_block({"file_data": {"mime_type": "image/png", "file_uri": "gs://bucket/a.png"}}),+ "[image]",+ id="gemini_cloud_storage_uri",+ ),+ ],+ )+ def test_images_it_cannot_attach_keep_an_unnumbered_marker(self, media: MediaBlock, marker: str):+ assert JudgeImages(team_id=7).marker(media) == marker+++class TestJudgeImagesContent:+ def test_an_image_referenced_again_at_the_end_stays_attached(self):+ images = JudgeImages(team_id=7)+ markers = [images.marker(_inline(f"{i:02}")) for i in range(MAX_IMAGES + 1)]+ # The output repeats the first input image after every newer one.+ content = images.content(" ".join([*markers, markers[0]]), "openai", "gpt-5.5")++ assert {"type": "text", "text": "Image 1:"} in content+ assert {"type": "text", "text": "Image 2:"} not in content+ assert {"type": "text", "text": "Images not attached, over the judge's image limit: 1."} in content++ def test_attaches_the_newest_images_when_over_the_limit(self):+ images = JudgeImages(team_id=7)+ prompt = " ".join(images.marker(_inline(f"{i:02}")) for i in range(MAX_IMAGES + 1))++ content = images.content(prompt, "openai", "gpt-5.5")++ assert content[1] == {"type": "text", "text": "Image 2:"}+ assert content[-4:-1] == [+ {"type": "text", "text": f"Image {MAX_IMAGES + 1}:"},+ ImagePart(mime_type="image/png", data=base64.b64decode(f"{MAX_IMAGES:02}" * 4)),+ {"type": "text", "text": "Images not attached, over the judge's image limit: 1."},+ ]++ @pytest.mark.parametrize(+ ("provider", "model", "expected_image"),+ [+ pytest.param("openai", "gpt-5.5", ImagePart(mime_type="image/jpeg", url="https://signed"), id="link"),+ pytest.param("together_ai", "any", ImagePart(mime_type="image/jpeg", data=b"hi"), id="inline"),+ pytest.param("gemini", "gemini-2.0-flash", ImagePart(mime_type="image/jpeg", data=b"hi"), id="gemini_2_0"),+ ],+ )+ def test_attaches_each_image_under_its_label(self, provider: str, model: str, expected_image: ImagePart):+ images = JudgeImages(team_id=7)+ prompt = f"Input: user: {images.marker(_pointer('a', mime='image%2FJPG'))} What does this chart show?"++ with (+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed") as presign,+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 2}),+ patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"hi") as read,+ ):+ content = images.content(prompt, provider, model)++ assert content == [+ {"type": "text", "text": "The transcript refers to these images as [image N]."},+ {"type": "text", "text": "Image 1:"},+ expected_image,+ {"type": "text", "text": prompt},+ ]+ storage_call = presign if expected_image.url else read+ assert storage_call.call_args.args[0] == f"aio/7/sha256/{'a' * 64}"++ @pytest.mark.parametrize(+ ("provider", "model", "media", "storage_error"),+ [+ pytest.param("openai", "gpt-5.5", _pointer("a"), None, id="blob_missing_for_a_link"),+ pytest.param("together_ai", "any", _pointer("a"), None, id="blob_missing_for_inline"),+ pytest.param(+ "together_ai", "any", _pointer("a"), ObjectStorageError("head_object failed"), id="storage_error"+ ),+ pytest.param(+ "gemini", "gemini-3-flash", _pointer("a", mime="image%2Fgif"), None, id="format_rejected_by_provider"+ ),+ pytest.param(+ "gemini",+ "gemini-3-flash",+ MediaBlock(modality="image", mime_type="image/png", base64_data="aGk"),+ None,+ id="invalid_inline_base64",+ ),+ pytest.param(+ "gemini",+ "gemini-3-flash",+ MediaBlock(modality="image", mime_type="image/png", base64_data="aGk\u00e9"),+ None,+ id="non_ascii_inline_base64",+ ),+ ],+ )+ def test_reports_an_image_it_could_not_attach(+ self, provider: str, model: str, media: MediaBlock, storage_error: Exception | None+ ):+ images = JudgeImages(team_id=7)+ prompt = f"user: {images.marker(media)}"++ with (+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value=None),+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value=None, side_effect=storage_error),+ ):+ content = images.content(prompt, provider, model)++ assert content[1] == {"type": "text", "text": "Image 1: could not be attached."}++ def test_stops_calling_storage_after_a_storage_error(self):+ images = JudgeImages(team_id=7)+ prompt = f"user: {images.marker(_pointer('a'))} {images.marker(_pointer('b'))}"++ with patch(f"{OBJECT_STORAGE}.head_object_strict", side_effect=ObjectStorageError("timed out")) as head:+ content = images.content(prompt, "together_ai", "any")++ assert head.call_count == 1+ assert {"type": "text", "text": "Image 1: could not be attached."} in content+ assert {"type": "text", "text": "Image 2: could not be attached."} in content++ @pytest.mark.parametrize(+ ("provider", "media", "too_large", "reads"),+ [+ pytest.param("openai", _pointer, set(), 0, id="links_have_no_byte_cap"),+ pytest.param("anthropic", _pointer, {1, 2}, 0, id="provider_limit_on_links"),+ # The pointers claim 63 KB, so this also checks that the budget uses the stored size.+ pytest.param("together_ai", _pointer, {1}, 1, id="stored_size_for_inline_providers"),+ pytest.param(+ "openai", lambda digit: _inline(digit, length=13_400_000), {1}, 0, id="inline_images_on_link_providers"+ ),+ ],+ )+ def test_caps_image_bytes(self, provider: str, media, too_large: set[int], reads: int):+ images = JudgeImages(team_id=7)+ prompt = f"user: {images.marker(media('a'))} {images.marker(media('b'))}"++ with (+ patch(f"{OBJECT_STORAGE}.get_presigned_url", return_value="https://signed"),+ patch(f"{OBJECT_STORAGE}.head_object_strict", return_value={"ContentLength": 10_000_000}),+ patch(f"{OBJECT_STORAGE}.read_bytes", return_value=b"x" * 10_000_000) as read,+ ):+ content = images.content(prompt, provider, "any")++ assert {+ number+ for number in (1, 2)+ if {"type": "text", "text": f"Image {number}: not attached, too large."} in content+ } == too_large+ # An image over the budget is never downloaded.+ assert read.call_count == reads++ def test_skips_images_whose_marker_was_sampled_out_of_the_prompt(self):+ images = JudgeImages(team_id=7)+ images.marker(_pointer("a"))++ assert images.content("user: hello", "openai", "gpt-5.5") == "user: hello"diff --git a/posthog/temporal/ai_observability/test_message_utils.py b/posthog/temporal/ai_observability/test_message_utils.pyindex 548c36ccf6e1..bcdc89602c96 100644--- a/posthog/temporal/ai_observability/test_message_utils.py+++ b/posthog/temporal/ai_observability/test_message_utils.py@@ -4,6 +4,7 @@ import jsonimport pytest+from posthog.temporal.ai_observability.judge_images import JudgeImagesfrom posthog.temporal.ai_observability.message_utils import (extract_text_from_messages,format_tool_definitions,@@ -62,6 +63,96 @@ class TestExtractTextFromMessages:assert expected_substring in resultassert result != ""+ @pytest.mark.parametrize(+ ("block", "marker"),+ [+ pytest.param(+ {"type": "image_url", "image_url": {"url": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fjpeg"}},+ "[image 1]",+ id="openai_offloaded_pointer",+ ),+ pytest.param(+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGk="}},+ "[image 1]",+ id="openai_inline_data_uri",+ ),+ pytest.param(+ {"type": "input_image", "image_url": "https://example.com/cat.jpg"},+ "[image: https://example.com/cat.jpg]",+ id="openai_responses_customer_link",+ ),+ pytest.param(+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}},+ "[image 1]",+ id="anthropic_base64",+ ),+ pytest.param(+ {+ "type": "image",+ "source": {"type": "base64", "media_type": "image/png", "data": "[base64 image redacted]"},+ },+ "[image]",+ id="anthropic_redacted",+ ),+ pytest.param(+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,[base64 image redacted]"}},+ "[image]",+ id="openai_redacted_data_uri",+ ),+ pytest.param(+ {"type": "image_url", "image_url": {"url": "data:image/png;base64," * 3 + "aGk="}},+ "[image]",+ id="nested_data_uri",+ ),+ pytest.param({"inline_data": {"mime_type": "image/png", "data": "aGk="}}, "[image 1]", id="gemini_inline"),+ pytest.param(+ {"type": "image", "inline_data": {"mime_type": "image/png", "data": "aGk="}},+ "[image 1]",+ id="gemini_posthog_sdk",+ ),+ pytest.param(+ {"type": "image", "data": "aGk=", "mime_type": "image/png"}, "[image 1]", id="gemini_interactions"+ ),+ pytest.param({"inlineData": {"mimeType": "audio/wav", "data": "aGk="}}, "[audio]", id="gemini_audio"),+ pytest.param({"type": "input_audio", "input_audio": {"data": "aGk="}}, "[audio]", id="openai_audio"),+ pytest.param({"type": "document", "source": {"type": "base64", "data": "aGk="}}, "[file]", id="file"),+ ],+ )+ def test_judge_images_replace_media_blocks_with_markers(self, block, marker):+ messages = [{"role": "user", "content": [block, {"type": "text", "text": "What does this chart show?"}]}]++ assert (+ extract_text_from_messages(messages, JudgeImages(team_id=7).marker)+ == f"user: {marker} What does this chart show?"+ )++ def test_judge_images_number_otel_parts_and_tool_result_images(self):+ pointer = f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1"+ image = {"type": "image_url", "image_url": {"url": pointer}}+ messages = [+ {+ "role": "user",+ "parts": [{"type": "blob", "modality": "image", "mime_type": "image/png", "content": pointer}],+ },+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [image]}]},+ {+ "type": "function_call_output",+ "call_id": "c1",+ "output": [{"type": "input_image", "image_url": pointer}],+ },+ ]++ assert (+ extract_text_from_messages(messages, JudgeImages(team_id=7).marker)+ == "user: [image 1]\nuser: [image 1]\ntool[c1]: [image 1]"+ )+ assert extract_text_from_messages(messages).startswith("user: [image]\n")++ def test_media_blocks_keep_their_text_without_judge_images(self):+ block = {"type": "file", "file": {"filename": "report.pdf"}}++ assert "report.pdf" in extract_text_from_messages([{"role": "user", "content": [block]}])+def test_mixed_content_blocks(self):messages = [{@@ -499,6 +590,16 @@ class TestExtractTextFromMessages:),pytest.param({"type": "file", "file_id": "file_123"}, "[file: file_123]", id="file_reference"),pytest.param({"type": "blob", "content": "aGVsbG8="}, "[media]", id="blob_without_modality"),+ pytest.param(+ {"type": "blob", "mime_type": "audio/wav", "content": "aGVsbG8="},+ "[audio]",+ id="modality_from_mime_type",+ ),+ pytest.param(+ {"type": "uri", "modality": "image", "mime_type": "image/png", "uri": "gs://bucket/a.png"},+ "[image]",+ id="uri_nobody_else_can_fetch",+ ),],)def test_otel_parts_media_render_markers(self, part, marker):diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.pyindex 3366a68f9844..5ae5ee7572aa 100644--- a/posthog/temporal/ai_observability/test_run_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_evaluation.py@@ -1,3 +1,4 @@+import sysimport jsonimport uuidimport dataclasses@@ -27,6 +28,7 @@ from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Workerfrom posthog.api.capture import CaptureInternalErrorfrom posthog.constants import AvailableFeaturefrom posthog.models import Organization, OrganizationMembership, Team, User+from posthog.temporal.ai_observability.judge_images import ImagePrompt, JudgeImagesfrom posthog.temporal.ai_observability.sentiment.extraction import truncate_to_head_tailfrom posthog.temporal.ai_observability.sentiment.schema import SentimentResultfrom posthog.temporal.common.errors import NonReportableError@@ -34,6 +36,7 @@ from posthog.temporal.common.posthog_client import is_expected_activity_failurefrom products.access_control.backend.models.access_control import AccessControlfrom products.ai_observability.backend.llm.errors import (+ RESPONSE_LIMIT_MESSAGE,AuthenticationError,ClaudeDeploymentOnAzureOpenAIError,ContentFilteredError,@@ -42,16 +45,19 @@ from products.ai_observability.backend.llm.errors import (ModelPermissionError,OutputTokenLimitError,ProviderConnectionError,+ ProviderRequestRejectedError,QuotaExceededError,RateLimitError,StructuredOutputParseError,UnsupportedModelError,)+from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.models.evaluation_config import EvaluationConfigfrom products.ai_observability.backend.models.evaluation_directories import EvaluationDirectoryfrom products.ai_observability.backend.models.evaluations import Evaluationfrom products.ai_observability.backend.models.provider_keys import LLMProviderKeyfrom products.ai_observability.backend.text_repr.formatters import EVENT_TEXT_MAX_CHARS+from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlockfrom . import run_evaluation as run_evaluation_modulefrom .evaluation_errors import (@@ -1137,6 +1143,73 @@ def _openai_status_error(status: int, message: str) -> openai.APIStatusError:return openai.APIStatusError(f"Error code: {status}", response=response, body={"error": {"message": message}})+def _adapter_rejection(cause: Exception | None) -> ProviderRequestRejectedError:+ error = ProviderRequestRejectedError(RESPONSE_LIMIT_MESSAGE if cause is None else "The provider rejected it.")+ error.__cause__ = cause+ return error+++ ("error", "presigned_url", "retries"),+ [+ pytest.param(+ _adapter_rejection(_openai_status_error(400, "Image input is not supported.")),+ "https://signed",+ True,+ id="adapter_wrapped_400",+ ),+ pytest.param(+ _openai_status_error(400, "Error while downloading https://signed."), "https://signed", True, id="raw_400"+ ),+ pytest.param(_adapter_rejection(None), "https://signed", False, id="unreadable_response"),+ pytest.param(_openai_status_error(401, "Invalid API key."), "https://signed", False, id="auth_error"),+ pytest.param(_openai_status_error(400, "Bad request."), None, False, id="no_image_attached"),+ ],+)+def test_judge_falls_back_to_the_plain_prompt_only_when_an_attached_image_is_rejected(+ error: Exception, presigned_url: str | None, retries: bool+) -> None:+ images = JudgeImages(team_id=1)+ pointer = MediaBlock(modality="image", url=f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1")+ image_prompt = ImagePrompt(text=f"user: {images.marker(pointer)} Polite?", images=images)++ with (+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client") as client_class,+ patch(+ "products.ai_observability.backend.ai_blobs.object_storage.get_presigned_url", return_value=presigned_url+ ),+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.capture_exception") as capture,+ ):+ # Only a PostHog key reports the fallback. On BYOK a rejection ends as a skip instead of raising.+ spec.return_value.resolve.return_value = MagicMock(+ provider="openai", model="gpt-5.5", provider_key=MagicMock(), is_byok=not retries+ )+ # The provider error can quote the link, so nothing may log or chain it while the fallback runs.+ active_errors: list[BaseException | None] = []+ capture.side_effect = lambda *_: active_errors.append(sys.exc_info()[1])+ client = client_class.return_value+ client.complete.side_effect = [+ error,+ MagicMock(parsed=BooleanEvalResult(verdict=True, reasoning="ok"), usage=None),+ ]+ result = call_llm_judge(+ evaluation={"id": "test-evaluation", "team_id": 1, "evaluation_config": {"prompt": "Polite?"}},+ system_prompt="",+ user_prompt="user: Polite?",+ allows_na=False,+ image_prompt=image_prompt,+ )++ if retries:+ assert result["verdict"] is True+ assert client.complete.call_args_list[1].args[0].messages == [{"role": "user", "content": "user: Polite?"}]+ assert "https://signed" not in str(capture.call_args)+ assert active_errors == [None]+ else:+ assert client.complete.call_count == 1++@pytest.mark.parametrize("provider, raised_exception, expected_status, expected_detail",[@@ -1499,6 +1572,86 @@ class TestRunEvaluationWorkflow:assert result["reasoning"] == "The answer is correct"mock_client.complete.assert_called_once()+ @pytest.mark.parametrize(+ ("flag_enabled", "evaluation_config", "attaches_image"),+ [+ pytest.param(True, {"prompt": "Is the chart described correctly?"}, True, id="flag_on"),+ pytest.param(False, {"prompt": "Is the chart described correctly?"}, False, id="flag_off"),+ pytest.param(+ True, {"prompt": "Is the chart described correctly?", "include_media": False}, False, id="eval_opts_out"+ ),+ ],+ )+ @pytest.mark.django_db(transaction=True)+ def test_execute_llm_judge_activity_attaches_images_when_enabled(+ self, flag_enabled, evaluation_config, attaches_image, setup_data, active_key_config+ ):+ team = setup_data["team"]+ evaluation = {+ "id": str(setup_data["evaluation"].id),+ "name": "Test Evaluation",+ "evaluation_type": "llm_judge",+ "evaluation_config": evaluation_config,+ "output_type": "boolean",+ "output_config": {},+ "team_id": team.id,+ }+ pointer = f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fjpeg&size=63357"+ event_data = create_mock_event_data(+ team.id,+ properties={+ "$ai_input": [+ {+ "role": "user",+ "content": [+ {"type": "image_url", "image_url": {"url": pointer}},+ {"type": "text", "text": "What does this chart show?"},+ ],+ }+ ],+ "$ai_output_choices": [+ {+ "role": "assistant",+ "content": [+ {"type": "image_url", "image_url": {"url": pointer.replace("a" * 64, "b" * 64)}},+ {"type": "text", "text": "Sales rose in March."},+ ],+ }+ ],+ },+ )++ with (+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client") as mock_client_class,+ patch(+ "posthog.temporal.ai_observability.judge_images.posthoganalytics.feature_enabled",+ return_value=flag_enabled,+ ) as feature_enabled,+ patch(+ "products.ai_observability.backend.ai_blobs.object_storage.get_presigned_url",+ return_value="https://signed",+ ),+ ):+ mock_client = mock_client_class.return_value+ mock_client.complete.return_value = MagicMock(+ parsed=BooleanEvalResult(verdict=True, reasoning="ok"),+ usage=MagicMock(input_tokens=1, output_tokens=1, total_tokens=1),+ )+ execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))++ content = mock_client.complete.call_args.args[0].messages[0]["content"]+ if "include_media" not in evaluation_config:+ # Team ids repeat across regions, so the flag is checked against the project uuid.+ assert feature_enabled.call_args.kwargs["groups"] == {"project": str(team.uuid)}+ if attaches_image:+ assert ImagePart(mime_type="image/jpeg", url="https://signed") in content+ assert "user: [image 1] What does this chart show?" in content[-1]["text"]+ assert "assistant: [image 2] Sales rose in March." in content[-1]["text"]+ else:+ assert isinstance(content, str)+ assert "[image" not in content+ assert "What does this chart show?" in content+@pytest.mark.parametrize("oversized_input",[@@ -4514,6 +4667,7 @@ class TestJudgePromptAssembly:"evaluation_type": "llm_judge","evaluation_config": {"prompt": "Is this correct?"},"output_type": "boolean",+ "team_id": 1,}event_data = create_mock_event_data(1,@@ -4539,6 +4693,7 @@ class TestJudgePromptAssembly:"evaluation_type": "llm_judge","evaluation_config": {"prompt": "Is this correct?"},"output_type": output_type,+ "team_id": 1,"output_config": {"allows_na": allows_na,**({"options": [{"key": "correct", "label": "Correct"}]} if output_type == "categorical" else {}),@@ -4563,6 +4718,39 @@ class TestJudgePromptAssembly:skip_reason="generation_output_too_long",)+ def test_inline_images_that_overflow_the_plain_prompt_are_still_judged(self) -> None:+ evaluation = {+ "evaluation_type": "llm_judge",+ "evaluation_config": {"prompt": "Is this correct?"},+ "output_type": "boolean",+ "team_id": 1,+ "output_config": {},+ }+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64," + "A" * 20_000}}+ event_data = create_mock_event_data(+ 1,+ properties={+ "$ai_input": "Draw eight charts.",+ "$ai_output_choices": [{"role": "assistant", "content": [image] * 8}],+ },+ )++ with (+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.call_llm_judge") as judge,+ patch(+ "posthog.temporal.ai_observability.judge_images.posthoganalytics.feature_enabled",+ return_value=True,+ ),+ patch("posthog.temporal.ai_observability.judge_images._project_uuid", return_value="uuid"),+ ):+ execute_llm_judge_activity(ExecuteLLMJudgeInputs(evaluation=evaluation, event_data=event_data))++ image_prompt = judge.call_args.kwargs["image_prompt"]+ assert image_prompt is not None+ # Decision judges and the text-only retry use this prompt, and they send no images.+ assert judge.call_args.kwargs["user_prompt"] == image_prompt.unnumbered_text()+ assert "[image 1]" not in judge.call_args.kwargs["user_prompt"]+@pytest.mark.parametrize("text,expected_text",[diff --git a/posthog/temporal/ai_observability/test_run_session_evaluation.py b/posthog/temporal/ai_observability/test_run_session_evaluation.pyindex e32bd6ec8d0b..70a4b7cb3608 100644--- a/posthog/temporal/ai_observability/test_run_session_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_session_evaluation.py@@ -20,6 +20,7 @@ from posthog.temporal.ai_observability.evaluation_llm_judge import (get_output_type_config,)from posthog.temporal.ai_observability.evaluation_payload import payload_budget_bytes+from posthog.temporal.ai_observability.judge_images import JudgeImagesfrom posthog.temporal.ai_observability.run_session_evaluation import (_SESSION_EVENT_COUNT_SQL,AI_EVENTS_RETENTION_DAYS,@@ -141,6 +142,18 @@ class TestFormatSessionForJudge:traces = [_trace(f"t{i}", cost=0, latency=0, event_count=100) for i in range(260)]assert format_session_for_judge(traces) is None+ def test_numbers_an_image_once_across_the_session(self):+ image = {"type": "image_url", "image_url": {"url": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1"}}+ traces = [_trace(f"t{i}", cost=0, latency=0) for i in range(2)]+ for trace in traces:+ trace.events[0].properties["$ai_input"] = [{"role": "user", "content": [image]}]++ rendered = format_session_for_judge(traces, JudgeImages(team_id=1).marker)++ assert rendered is not None+ assert rendered.count("[image 1]") == 2+ assert "[image 2]" not in rendered+def test_many_small_traces_are_judged_rather_than_skipped_on_count_alone(self):"""The overflow check must measure the rendered transcript, not `per_trace_budget * n`.The product form made this a hard cliff at 251 traces however little they contained, so adiff --git a/posthog/temporal/ai_observability/test_run_trace_evaluation.py b/posthog/temporal/ai_observability/test_run_trace_evaluation.pyindex a78b2f6f9b12..3b94246a8d4e 100644--- a/posthog/temporal/ai_observability/test_run_trace_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_trace_evaluation.py@@ -23,6 +23,7 @@ from posthog.hogql_queries.ai.trace_query_runner import TraceQueryRunnerfrom posthog.models import Organization, PropertyDefinition, Team, Userfrom posthog.models.ai_events.test_util import bulk_create_ai_eventsfrom posthog.models.event.util import bulk_create_events+from posthog.temporal.ai_observability.judge_images import JudgeImagesfrom posthog.temporal.ai_observability.run_session_evaluation import fetch_session_for_evaluationfrom posthog.temporal.ai_observability.trace_summarization.fetch_and_format import _fetch_and_format_trace@@ -160,6 +161,47 @@ class TestFormatTraceForJudge:assert "gpt-4o" in transcriptassert "What is 2+2?" in transcript+ def test_numbers_images_only_for_a_judge_with_images(self):+ image = {"type": "image_url", "image_url": {"url": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng&size=1"}}+ otel_blob = {"type": "blob", "modality": "image", "mime_type": "image/png", "content": "aGk="}+ trace = create_trace(+ [+ create_trace_event(+ "$ai_generation",+ **{+ "$ai_input": [+ {"role": "user", "content": [image, {"type": "text", "text": "Describe the chart."}]},+ {+ "role": "user",+ "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [image]}],+ },+ {"role": "user", "parts": [otel_blob]},+ {+ "type": "function_call_output",+ "call_id": "c1",+ "output": [+ {+ "type": "input_image",+ "image_url": f"phaiblob://v1/sha256/{'c' * 64}?mime=image%2Fpng&size=1",+ }+ ],+ },+ ],+ "$ai_output": "A rising line.",+ },+ )+ ]+ )++ plain = format_trace_for_judge(trace)+ marked = format_trace_for_judge(trace, JudgeImages(team_id=1).marker)++ assert "[image 1]" not in plain+ # The message and the tool result repeat one image, and the OTel part adds a second.+ assert marked.count("[image 1]") == 2+ assert "[image 2]" in marked+ assert "[image 3]" in marked+def test_includes_tool_catalog(self):tools = [{"type": "function", "function": {"name": "search_docs", "description": "Search the docs"}}]trace = create_trace(diff --git a/products/ai_observability/backend/ai_blobs.py b/products/ai_observability/backend/ai_blobs.pynew file mode 100644index 000000000000..80b2e4d0308c--- /dev/null+++ b/products/ai_observability/backend/ai_blobs.py@@ -0,0 +1,46 @@+"""`phaiblob://` pointers that ingestion writes in place of offloaded AI media. Twin of `frontend/aiBlob.ts`."""++import re+from urllib.parse import parse_qs++from django.conf import settings++from posthog.dataclasses import frozen+from posthog.storage import object_storage++_POINTER_RE = re.compile(r"^phaiblob://v1/sha256/([0-9a-f]{64})(?:\?(.*))?$")+++def blob_key(team_id: int, sha256: str) -> str:+ return f"{settings.AI_BLOB_S3_PREFIX}{team_id}/sha256/{sha256}"+++@frozen+class BlobPointer:+ sha256: str+ mime_type: str | None = None++ @classmethod+ def parse(cls, value: str) -> "BlobPointer | None":+ match = _POINTER_RE.match(value)+ if match is None:+ return None+ # The pointer's `size=` comes from event data, so readers ask storage for the real size instead.+ mime_type = parse_qs(match.group(2) or "").get("mime", [None])[0]+ return cls(sha256=match.group(1), mime_type=mime_type)++ # The team id always comes from the caller's own scope, never from event data,+ # so a pointer copied from another team's event can't reach that team's blob.+ def read(self, team_id: int) -> bytes | None:+ return object_storage.read_bytes(+ blob_key(team_id, self.sha256), bucket=settings.AI_BLOB_S3_BUCKET, missing_ok=True+ )++ def stored_size(self, team_id: int) -> int | None:+ head = object_storage.head_object_strict(blob_key(team_id, self.sha256), bucket=settings.AI_BLOB_S3_BUCKET)+ return int(head["ContentLength"]) if head else None++ def presigned_url(self, team_id: int, expiration_seconds: int) -> str | None:+ return object_storage.get_presigned_url(+ blob_key(team_id, self.sha256), expiration=expiration_seconds, bucket=settings.AI_BLOB_S3_BUCKET+ )diff --git a/products/ai_observability/backend/api/ai_blob.py b/products/ai_observability/backend/api/ai_blob.pyindex 22ab3c8da8a5..64977523acb8 100644--- a/products/ai_observability/backend/api/ai_blob.py+++ b/products/ai_observability/backend/api/ai_blob.py@@ -9,6 +9,8 @@ from rest_framework.request import Requestfrom posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.storage import object_storage+from products.ai_observability.backend.ai_blobs import blob_key+SCRIPTABLE_IMAGE_MIMES = {"image/svg+xml"}@@ -27,8 +29,9 @@ class AIBlobViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):@extend_schema(exclude=True)@action(detail=False, methods=["GET"], url_path=r"v1/sha256/(?P<hash>[0-9a-f]{64})")def serve(self, request: Request, hash: str, **kwargs) -> HttpResponse:- key = f"{settings.AI_BLOB_S3_PREFIX}{self.team_id}/sha256/{hash}"- result = object_storage.read_object(key, bucket=settings.AI_BLOB_S3_BUCKET, missing_ok=True)+ result = object_storage.read_object(+ blob_key(self.team_id, hash), bucket=settings.AI_BLOB_S3_BUCKET, missing_ok=True+ )if result is None:return HttpResponse(status=404)body, content_type = resultdiff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.pyindex 7798809a85bf..a8f971a1d99f 100644--- a/products/ai_observability/backend/api/evaluations.py+++ b/products/ai_observability/backend/api/evaluations.py@@ -126,6 +126,10 @@ logger = structlog.get_logger(__name__)"nullable": True,"description": "Judge method. Omit to keep the saved method; null restores automatic routing.",},+ "include_media": {+ "type": "boolean",+ "description": "Set to false to judge text only.",+ },"reasoning_level": {"type": "string","enum": ["minimal", "low", "medium", "high"],@@ -602,10 +606,10 @@ class EvaluationSerializer(UserAccessControlSerializerMixin, serializers.ModelSeif not isinstance(config, dict):raise serializers.ValidationError({"evaluation_config": "Must be an object."})if self.partial and self.instance is not None and evaluation_type == EvaluationType.LLM_JUDGE.value:- # A prompt-only PATCH must not reset the saved judge settings.+ # A prompt-only PATCH must not reset the saved judge settings or image opt-out.saved = {key: self.instance.evaluation_config[key]- for key in ("judge_method", "reasoning_level")+ for key in ("judge_method", "reasoning_level", "include_media")if key not in config and key in self.instance.evaluation_config}data["evaluation_config"] = {**saved, **config}diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.pyindex 8d155c62fc1b..d4a88f9e76e6 100644--- a/products/ai_observability/backend/api/test/test_evaluations.py+++ b/products/ai_observability/backend/api/test/test_evaluations.py@@ -137,17 +137,19 @@ class TestModelConfigurationSerializer(SimpleTestCase):@parameterized.expand([- ("llm", {}, {"judge_method": "llm"}),- ("decision", {}, {"judge_method": "decision"}),- ("llm", {"judge_method": None}, {}),- ("decision", {"judge_method": "llm"}, {"judge_method": "llm"}),- (None, {}, {}),+ ({"judge_method": "llm"}, {}, {"judge_method": "llm"}),+ ({"judge_method": "decision"}, {}, {"judge_method": "decision"}),+ ({"judge_method": "llm"}, {"judge_method": None}, {}),+ ({"judge_method": "decision"}, {"judge_method": "llm"}, {"judge_method": "llm"}),+ ({}, {}, {}),+ ({"include_media": False}, {}, {"include_media": False}),+ ({"include_media": False}, {"include_media": True}, {"include_media": True}),])- def test_prompt_patch_preserves_method_unless_explicitly_changed(- self, method: str | None, patch_config: dict[str, object], expected_method: dict[str, str]+ def test_prompt_patch_preserves_saved_choices_unless_explicitly_changed(+ self, saved: dict[str, object], patch_config: dict[str, object], expected_method: dict[str, object]) -> None:- stored_config = {"prompt": "Check the response.", **({"judge_method": method} if method else {})}+ stored_config = {"prompt": "Check the response.", **saved}evaluation = Evaluation(evaluation_type="llm_judge",evaluation_config=stored_config.copy(),diff --git a/products/ai_observability/backend/facade/blobs.py b/products/ai_observability/backend/facade/blobs.pynew file mode 100644index 000000000000..d46305f3be64--- /dev/null+++ b/products/ai_observability/backend/facade/blobs.py@@ -0,0 +1,9 @@+"""Facade re-export for offloaded AI blobs.++Isolated from ``facade/api.py``, which imports the clustering workflows, so the eval workers can read blobs without an+import cycle.+"""++from products.ai_observability.backend.ai_blobs import BlobPointer++__all__ = ["BlobPointer"]diff --git a/products/ai_observability/backend/llm/images.py b/products/ai_observability/backend/llm/images.pynew file mode 100644index 000000000000..9d54236ca84d--- /dev/null+++ b/products/ai_observability/backend/llm/images.py@@ -0,0 +1,52 @@+import base64+from collections.abc import Callable+from dataclasses import field+from typing import Any, Literal++from posthog.dataclasses import frozen++IMAGE_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/webp", "image/gif"})+# Providers whose image blocks take a link that they download themselves.+_URL_PROVIDERS = frozenset({"openai", "azure_openai", "anthropic", "openrouter", "gemini"})+# Raw bytes per image. Anthropic allows 10 MB base64-encoded, about 7.5 MB raw, so keep a margin under it.+MAX_IMAGE_BYTES = {"anthropic": 7_000_000}+++@frozen+class ImagePart:+ """An image in a message's content list. Each adapter converts it to its provider's image block.++ Set `url` for a link the provider downloads, or `data` for inline bytes.+ """++ mime_type: str+ url: str | None = None+ data: bytes | None = field(default=None, repr=False)++ def base64(self) -> str:+ return base64.b64encode(self.data or b"").decode()++ def data_uri(self) -> str:+ return f"data:{self.mime_type};base64,{self.base64()}"+++def image_transport(provider: str, model: str, mime_type: str) -> Literal["url", "inline"] | None:+ """How a provider accepts an image of this type: a link, inline base64, or not at all."""+ if mime_type not in IMAGE_MIME_TYPES:+ return None+ if provider == "gemini":+ if mime_type == "image/gif":+ return None+ # The Gemini 2.0 family can't fetch external links.+ return "inline" if model.startswith("gemini-2.0") else "url"+ return "url" if provider in _URL_PROVIDERS else "inline"+++def convert_image_parts(messages: list[dict[str, Any]], convert: Callable[[ImagePart], Any]) -> list[dict[str, Any]]:+ """Replace each ImagePart in the messages' content lists with the provider block that `convert` returns."""+ return [+ {**message, "content": [convert(part) if isinstance(part, ImagePart) else part for part in message["content"]]}+ if isinstance(message.get("content"), list)+ else message+ for message in messages+ ]diff --git a/products/ai_observability/backend/llm/providers/anthropic.py b/products/ai_observability/backend/llm/providers/anthropic.pyindex 306dbb9b0f97..3cd96efa68a5 100644--- a/products/ai_observability/backend/llm/providers/anthropic.py+++ b/products/ai_observability/backend/llm/providers/anthropic.py@@ -34,6 +34,7 @@ from products.ai_observability.backend.llm.errors import (is_output_limit_error_message,stream_error_chunk,)+from products.ai_observability.backend.llm.images import ImagePart, convert_image_partsfrom products.ai_observability.backend.llm.providers.azure_openai import is_allowed_azure_endpointfrom products.ai_observability.backend.llm.types import (AnalyticsContext,@@ -229,7 +230,7 @@ class AnthropicAdapter:effective_api_key = api_key or self._get_default_api_key()client = self._create_client(effective_api_key, analytics)- messages: Any = request.messages+ messages: Any = convert_image_parts(request.messages, _anthropic_image_block)system_prompt = request.system or ""use_structured = request.response_format and issubclass(request.response_format, BaseModel)@@ -359,7 +360,7 @@ class AnthropicAdapter:tools = self._convert_tools(request.tools) if request.tools else Nonesystem = request.system or ""- messages = request.messages+ messages = convert_image_parts(request.messages, _anthropic_image_block)# Handle cache control for supported modelssystem_prompt: list[TextBlockParam] = []@@ -638,3 +639,9 @@ class AnthropicAdapter:prepared_messages.append(prepared_message)return prepared_messages+++def _anthropic_image_block(part: ImagePart) -> dict[str, Any]:+ if part.url is not None:+ return {"type": "image", "source": {"type": "url", "url": part.url}}+ return {"type": "image", "source": {"type": "base64", "media_type": part.mime_type, "data": part.base64()}}diff --git a/products/ai_observability/backend/llm/providers/openai.py b/products/ai_observability/backend/llm/providers/openai.pyindex 5be139fd6a08..b69b662732c0 100644--- a/products/ai_observability/backend/llm/providers/openai.py+++ b/products/ai_observability/backend/llm/providers/openai.py@@ -36,6 +36,7 @@ from products.ai_observability.backend.llm.errors import (is_output_limit_error_message,stream_error_chunk,)+from products.ai_observability.backend.llm.images import ImagePart, convert_image_partsfrom products.ai_observability.backend.llm.types import (AnalyticsContext,CompletionRequest,@@ -310,7 +311,7 @@ You must respond with valid JSON that matches this schema:Return ONLY the JSON object, no other text or markdown formatting."""enhanced_messages = [{"role": "system", "content": enhanced_system}]- enhanced_messages.extend(request.messages)+ enhanced_messages.extend(convert_image_parts(request.messages, _openai_image_block))create_response = client.chat.completions.create(model=request.model,@@ -519,7 +520,7 @@ Return ONLY the JSON object, no other text or markdown formatting."""messages = []if request.system:messages.append({"role": "system", "content": request.system})- messages.extend(request.messages)+ messages.extend(convert_image_parts(request.messages, _openai_image_block))return messagesdef _build_analytics_kwargs(self, analytics: AnalyticsContext, client) -> dict:@@ -570,3 +571,7 @@ Return ONLY the JSON object, no other text or markdown formatting."""result = handler.convert_to(ToolFormat.OPENAI)assert result is not None, "tools must be non-empty when calling _convert_tools"return result+++def _openai_image_block(part: ImagePart) -> dict[str, Any]:+ return {"type": "image_url", "image_url": {"url": part.url or part.data_uri()}}diff --git a/products/ai_observability/backend/llm/providers/test/test_anthropic.py b/products/ai_observability/backend/llm/providers/test/test_anthropic.pyindex 656bb0399ad2..56d0a632a23c 100644--- a/products/ai_observability/backend/llm/providers/test/test_anthropic.py+++ b/products/ai_observability/backend/llm/providers/test/test_anthropic.py@@ -12,6 +12,7 @@ from products.ai_observability.backend.llm.errors import (OutputTokenLimitError,ProviderConfigurationError,)+from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.llm.providers.anthropic import (AnthropicAdapter,AnthropicConfig,@@ -172,6 +173,40 @@ class TestAnthropicStreamOutputSchema:assert "output_config" not in self._stream_kwargs(None)+class TestAnthropicImageParts:+ @parameterized.expand(+ [+ (+ "link",+ ImagePart(mime_type="image/png", url="https://example.com/a.png"),+ {"type": "url", "url": "https://example.com/a.png"},+ ),+ (+ "inline",+ ImagePart(mime_type="image/png", data=b"hi"),+ {"type": "base64", "media_type": "image/png", "data": "aGk="},+ ),+ ]+ )+ def test_image_parts_are_sent_as_image_blocks(self, _name: str, image: ImagePart, expected_source: dict):+ request = CompletionRequest(+ model="claude-haiku-4-5",+ messages=[{"role": "user", "content": [image, {"type": "text", "text": "hi"}]}],+ provider="anthropic",+ )+ with patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.Anthropic") as mock_cls:+ mock_client = MagicMock()+ mock_cls.return_value = mock_client+ mock_client.messages.create.return_value.content = []+ mock_client.messages.create.return_value.usage = MagicMock(input_tokens=1, output_tokens=1)+ AnthropicAdapter().complete(request, api_key="sk-ant-test", analytics=AnalyticsContext(capture=False))++ assert mock_client.messages.create.call_args.kwargs["messages"][0]["content"] == [+ {"type": "image", "source": expected_source},+ {"type": "text", "text": "hi"},+ ]++def _make_bad_request_error(message: str) -> anthropic.BadRequestError:request = httpx.Request("POST", "https://api.anthropic.com/v1/messages")response = httpx.Response(status_code=400, request=request, json={"error": {"message": message}})diff --git a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.pyindex aa92919c8818..93cce2da4829 100644--- a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py+++ b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py@@ -19,6 +19,7 @@ from products.ai_observability.backend.llm.errors import (QuotaExceededError,StructuredOutputParseError,)+from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.llm.providers.openai import OpenAIAdapter, OpenAIConfigfrom products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema@@ -177,6 +178,42 @@ class TestOpenAIStreamOutputSchema:assert "response_format" not in self._stream_kwargs(None)+class TestOpenAIImageParts:+ @pytest.mark.parametrize(+ ("image", "expected_url"),+ [+ pytest.param(+ ImagePart(mime_type="image/png", url="https://example.com/a.png"),+ "https://example.com/a.png",+ id="link",+ ),+ pytest.param(ImagePart(mime_type="image/png", data=b"hi"), "data:image/png;base64,aGk=", id="inline"),+ ],+ )+ @pytest.mark.parametrize("json_fallback", [False, True], ids=["plain", "json_fallback"])+ def test_image_parts_are_sent_as_image_url_blocks(self, image: ImagePart, expected_url: str, json_fallback: bool):+ request = CompletionRequest(+ model="gpt-4.1",+ messages=[{"role": "user", "content": [image, {"type": "text", "text": "hi"}]}],+ provider="openai",+ response_format=_Verdict if json_fallback else None,+ )+ mock_client = MagicMock()+ mock_client.beta.chat.completions.parse.side_effect = _make_bad_request_error(+ "response_format is not supported"+ )+ mock_client.chat.completions.create.return_value.usage = None+ mock_client.chat.completions.create.return_value.choices[0].message.content = '{"verdict": true}'+ with patch("products.ai_observability.backend.llm.providers.openai.openai.OpenAI", return_value=mock_client):+ OpenAIAdapter().complete(request, api_key="sk-test", analytics=AnalyticsContext(capture=False))++ sent = mock_client.chat.completions.create.call_args.kwargs["messages"]+ assert next(message for message in sent if message["role"] == "user")["content"] == [+ {"type": "image_url", "image_url": {"url": expected_url}},+ {"type": "text", "text": "hi"},+ ]++class TestOpenAICompleteReasoningEffort:@parameterized.expand([diff --git a/products/ai_observability/backend/models/evaluation_configs.py b/products/ai_observability/backend/models/evaluation_configs.pyindex 46f2bf5dca78..cdfe11cde92f 100644--- a/products/ai_observability/backend/models/evaluation_configs.py+++ b/products/ai_observability/backend/models/evaluation_configs.py@@ -30,6 +30,7 @@ class LLMJudgeConfig(BaseModel):judge_method: Literal["llm", "decision"] | None = Field(default=None, description="Judge method. Omit to keep the saved method; null restores automatic routing.")+ include_media: bool | None = Field(default=None, description="Set to false to judge text only.")# Matches llm.types.ReasoningLevel; importing the llm package here would load provider SDKs at startup.reasoning_level: Literal["minimal", "low", "medium", "high"] | None = Field(default=None,diff --git a/products/ai_observability/backend/providers/formatters/gemini_formatter.py b/products/ai_observability/backend/providers/formatters/gemini_formatter.pyindex 904f7b28ee16..209e5f27aaa8 100644--- a/products/ai_observability/backend/providers/formatters/gemini_formatter.py+++ b/products/ai_observability/backend/providers/formatters/gemini_formatter.py@@ -2,8 +2,9 @@ import jsonimport base64from typing import Any, cast-from google.genai.types import Blob, Content, ContentListUnion, FunctionCall, FunctionResponse, Part+from google.genai.types import Blob, Content, ContentListUnion, FileData, FunctionCall, FunctionResponse, Part+from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.providers.formatters.anthropic_typeguards import (is_base64_image_param,is_image_block_param,@@ -84,6 +85,12 @@ def _tool_response_payload(content: Any, *, is_error: bool = False) -> dict[str,return value if isinstance(value, dict) else {"result": value}+def _gemini_image_part(image: ImagePart) -> Part:+ if image.url is not None:+ return Part(file_data=FileData(file_uri=image.url, mime_type=image.mime_type))+ return Part(inline_data=Blob(data=image.data or b"", mime_type=image.mime_type))++def convert_anthropic_messages_to_gemini(messages: list[dict[str, Any]]) -> ContentListUnion:contents: list[Content] = [] # Sticking to Content, as we don't support other formats yet# Gemini keys a function response by function name, but Anthropic-style tool_result@@ -96,7 +103,9 @@ def convert_anthropic_messages_to_gemini(messages: list[dict[str, Any]]) -> Contparts.append(Part(text=message["content"]))elif isinstance(message["content"], list):for block in message["content"]:- if is_text_block_param(block):+ if isinstance(block, ImagePart):+ parts.append(_gemini_image_part(block))+ elif is_text_block_param(block):parts.append(Part(text=block["text"]))elif is_image_block_param(block):if not is_base64_image_param(block["source"]):diff --git a/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py b/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.pyindex bca4b7dd194d..676458ec055f 100644--- a/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py+++ b/products/ai_observability/backend/providers/formatters/test/test_gemini_formatter.py@@ -2,9 +2,10 @@ from typing import Any, castimport pytest-from google.genai.types import Content, FunctionCall, FunctionResponse, Part+from google.genai.types import Blob, Content, FileData, FunctionCall, FunctionResponse, Partfrom parameterized import parameterized+from products.ai_observability.backend.llm.images import ImagePartfrom products.ai_observability.backend.providers.formatters.gemini_formatter import (MessageConversionError,convert_anthropic_messages_to_gemini,@@ -92,6 +93,25 @@ class TestConvertAnthropicMessagesToGemini:assert function_response.name == "get_weather"assert function_response.response == expected_response+ @parameterized.expand(+ [+ (+ "link",+ ImagePart(mime_type="image/png", url="https://example.com/a.png"),+ Part(file_data=FileData(file_uri="https://example.com/a.png", mime_type="image/png")),+ ),+ (+ "inline",+ ImagePart(mime_type="image/png", data=b"hi"),+ Part(inline_data=Blob(data=b"hi", mime_type="image/png")),+ ),+ ]+ )+ def test_image_part_becomes_gemini_part(self, _name: str, image: ImagePart, expected: Part):+ contents = _convert([{"role": "user", "content": [image]}])++ assert _part(contents[0], 0) == expected+def test_first_function_call_in_a_turn_gets_the_skip_signature(self):contents = _convert([diff --git a/products/ai_observability/backend/text_repr/formatters/media_blocks.py b/products/ai_observability/backend/text_repr/formatters/media_blocks.pynew file mode 100644index 000000000000..210d9eb50ab0--- /dev/null+++ b/products/ai_observability/backend/text_repr/formatters/media_blocks.py@@ -0,0 +1,115 @@+import re+from collections.abc import Callable++from posthog.dataclasses import frozen++_IMAGE_TYPES = frozenset({"image", "image_url", "input_image"})+_AUDIO_TYPES = frozenset({"audio", "input_audio"})+_FILE_TYPES = frozenset({"document", "file", "input_file"})+_GEMINI_DATA_KEYS = ("inline_data", "inlineData", "file_data", "fileData")+_OTEL_TYPES = frozenset({"blob", "uri"})+_LINK_SCHEMES = ("https://", "http://", "phaiblob://")+# Without full capture the SDKs replace the bytes with a placeholder such as `[base64 image redacted]`.+# Same pattern as the frontend's `isRedactedMediaSentinel`.+_REDACTED_RE = re.compile(r"^\s*\[base64\b[^\]]*\bredacted\]\s*$", re.IGNORECASE)+++@frozen+class MediaBlock:+ """An image, audio, video or file content block, and where its bytes are.++ `url` holds an https link or a `phaiblob://` pointer, `base64_data` holds inline bytes.+ """++ modality: str+ mime_type: str | None = None+ url: str | None = None+ base64_data: str | None = None++ def marker(self) -> str:+ # A web link can tell the judge what the media shows. Base64 and pointers are token noise.+ if self.url is not None and self.url.startswith(("https://", "http://")):+ return f"[{self.modality}: {self.url}]"+ return f"[{self.modality}]"+++# Turns a media block into the text a transcript shows in its place, for example `[image 1]`.+MediaMarker = Callable[[MediaBlock], str]+++def parse_media_block(block: dict) -> MediaBlock | None:+ """Read an OpenAI chat or Responses, Anthropic, Gemini, or OTel GenAI media block."""+ # Before the `type` check: the PostHog Gemini SDK records `{"type": "image", "inline_data": {...}}`.+ for key in _GEMINI_DATA_KEYS:+ data = block.get(key)+ if isinstance(data, dict):+ mime_type = data.get("mime_type") or data.get("mimeType")+ mime_type = mime_type if isinstance(mime_type, str) else None+ modality = mime_type.split("/")[0] if mime_type else ""+ if modality not in ("image", "audio", "video"):+ modality = "file"+ uri = data.get("file_uri") or data.get("fileUri")+ if isinstance(uri, str):+ # A file URI is never inline bytes, and a `gs://` URI is not a link anyone else can fetch.+ return MediaBlock(+ modality=modality, mime_type=mime_type, url=uri if uri.startswith(_LINK_SCHEMES) else None+ )+ return _with_source(modality, mime_type, data.get("data"))+ block_type = block.get("type")+ if isinstance(block_type, str):+ if block_type in _OTEL_TYPES:+ return _otel_part(block)+ if block_type in _IMAGE_TYPES:+ return _image_block(block)+ if block_type in _AUDIO_TYPES:+ return MediaBlock(modality="audio")+ if block_type in _FILE_TYPES:+ return MediaBlock(modality="file")+ return None+++def _otel_part(part: dict) -> MediaBlock:+ mime_type = part.get("mime_type")+ mime_type = mime_type if isinstance(mime_type, str) else None+ modality = part.get("modality")+ if not isinstance(modality, str) or not modality:+ modality = mime_type.split("/")[0] if mime_type else "media"+ if part["type"] == "blob":+ return _with_source(modality, mime_type, part.get("content"))+ uri = part.get("uri")+ # A `uri` part never holds inline bytes, and a `gs://` URI is not a link anyone else can fetch.+ return MediaBlock(+ modality=modality,+ mime_type=mime_type,+ url=uri if isinstance(uri, str) and uri.startswith(_LINK_SCHEMES) else None,+ )+++def _image_block(block: dict) -> MediaBlock:+ image_url = block.get("image_url")+ if isinstance(image_url, dict):+ image_url = image_url.get("url")+ source = block.get("source")+ if isinstance(source, dict):+ media_type = source.get("media_type")+ return _with_source(+ "image", media_type if isinstance(media_type, str) else None, source.get("data") or source.get("url")+ )+ # The Gemini Interactions API puts `data` and `mime_type` on the block itself.+ mime_type = block.get("mime_type") or block.get("mimeType")+ return _with_source("image", mime_type if isinstance(mime_type, str) else None, image_url or block.get("data"))+++def _with_source(modality: str, mime_type: str | None, value: object) -> MediaBlock:+ if not isinstance(value, str) or not value:+ return MediaBlock(modality=modality, mime_type=mime_type)+ if value.startswith("data:") and ";base64," in value:+ # Unwrap one envelope only. Its payload is bytes or a wrapped pointer, never another data URI,+ # and unwrapping nested prefixes would copy the payload once per level.+ header, value = value[len("data:") :].split(";base64,", 1)+ mime_type = header or mime_type+ if _REDACTED_RE.match(value) or value.startswith("data:"):+ return MediaBlock(modality=modality, mime_type=mime_type)+ if value.startswith(_LINK_SCHEMES):+ return MediaBlock(modality=modality, mime_type=mime_type, url=value)+ return MediaBlock(modality=modality, mime_type=mime_type, base64_data=value)diff --git a/products/ai_observability/backend/text_repr/formatters/message_formatter.py b/products/ai_observability/backend/text_repr/formatters/message_formatter.pyindex 49f9e69cd75a..430fa59958c5 100644--- a/products/ai_observability/backend/text_repr/formatters/message_formatter.py+++ b/products/ai_observability/backend/text_repr/formatters/message_formatter.py@@ -32,7 +32,8 @@ from .constants import (SAMPLING_REDUCTION_FACTOR,SPECIAL_BLOCK_TYPES,)-from .otel_parts import flatten_parts_message+from .media_blocks import MediaMarker, parse_media_block+from .otel_parts import MEDIA_PART_KEY, flatten_parts_messagefrom .tool_formatter import format_tools@@ -47,6 +48,7 @@ class FormatterOptions(TypedDict, total=False):include_line_numbers: bool # Prefix each line with line number (default: False)max_length: int | None # Max output length; randomly drop lines if exceeded (default: None)max_render_length: int # Abort oversized renders before sampling or assembling the full transcript+ media_marker: MediaMarker # Render media blocks with this instead of dropping them (LLM judges with images)class ToolCall(TypedDict, total=False):@@ -344,7 +346,7 @@ def _string_type(msg: dict[str, Any]) -> str | None:return msg_type if isinstance(msg_type, str) else None-def safe_extract_text(content: Any) -> str:+def safe_extract_text(content: Any, media_marker: MediaMarker | None = None) -> str:"""Safely extract text from various content formats.Handles strings, dicts with 'text' key, arrays of content blocks, etc.@@ -360,7 +362,9 @@ def safe_extract_text(content: Any) -> str:if "text" in content:return str(content["text"])if "content" in content:- return safe_extract_text(content["content"])+ return safe_extract_text(content["content"], media_marker)+ if media_marker is not None and (media := parse_media_block(content)) is not None:+ return media_marker(media)# Fallback to JSON representationreturn json.dumps(content)@@ -371,6 +375,13 @@ def safe_extract_text(content: Any) -> str:item_type = _string_type(item)# Try both "text" and "content" keys (tool_result uses "content")text_value = item.get("text") or item.get("content")+ if media_marker is not None:+ if (media := parse_media_block(item)) is not None:+ text_value = media_marker(media)+ item_type = None+ elif isinstance(text_value, list):+ # Tool results can hold image blocks.+ text_value = safe_extract_text(text_value, media_marker)if text_value:# Add spacing between all content blocks in the array@@ -421,7 +432,7 @@ def _is_special_block(block: Any) -> bool:return False-def _format_special_block(block: dict) -> str | None:+def _format_special_block(block: dict, media_marker: MediaMarker | None = None) -> str | None:"""Format special blocks (tool calls, tool_use, etc).@@ -467,10 +478,10 @@ def _format_special_block(block: dict) -> str | None:return f"[Tool result: {tool_name}]"# Block not handled by this function- return safe_extract_text(block)+ return safe_extract_text(block, media_marker)-def extract_text_content(content: Any) -> str:+def extract_text_content(content: Any, media_marker: MediaMarker | None = None) -> str:"""Extract text content from various message content formats.Uses safe extraction with fallback for unparseable content.@@ -484,7 +495,7 @@ def extract_text_content(content: Any) -> str:text_parts: list[str] = []for block in content:if isinstance(block, dict):- formatted = _format_special_block(block)+ formatted = _format_special_block(block, media_marker)if formatted is not None and formatted:text_parts.append(formatted)# Handle non-dict items in list@@ -495,10 +506,10 @@ def extract_text_content(content: Any) -> str:return "\n\n".join(text_parts)# Use safe extraction for non-special content (handles type labels for text/reasoning/etc)- return safe_extract_text(content)+ return safe_extract_text(content, media_marker)-def extract_payload_text(payload: Any) -> str:+def extract_payload_text(payload: Any, media_marker: MediaMarker | None = None) -> str:"""Extract text from a tool payload, returning "" only when nothing was recorded.A falsey scalar such as `0` or `False` is a real result, so it must not collapse to "" the@@ -513,7 +524,7 @@ def extract_payload_text(payload: Any) -> str:# `extract_text_content`'s repr fallback would render an empty container as a literal# "[]" / "{}", which reads as recorded output.return ""- return extract_text_content(payload)+ return extract_text_content(payload, media_marker)def _is_responses_item(msg: dict[str, Any]) -> bool:@@ -596,7 +607,8 @@ def _format_responses_item(msg: dict[str, Any], options: FormatterOptions | Noneif item_type in RESPONSES_TOOL_OUTPUT_TYPES:output = msg.get("output")- text = extract_payload_text(msg.get("content") if output is None else output)+ media_marker = options.get("media_marker") if options else None+ text = extract_payload_text(msg.get("content") if output is None else output, media_marker)if not text:return [MISSING_TOOL_OUTPUT_NOTE]lines, _ = truncate_content(text, options)@@ -648,10 +660,14 @@ def _format_message_body(msg: dict[str, Any], options: FormatterOptions | None)responses_lines = _format_responses_item(msg, options)content = msg.get("content", "")- if responses_lines is not None:+ media_marker = options.get("media_marker") if options else None+ media_part = msg.get(MEDIA_PART_KEY)+ if media_marker is not None and isinstance(media_part, dict) and (media := parse_media_block(media_part)):+ lines.append(media_marker(media))+ elif responses_lines is not None:lines.extend(responses_lines)elif content:- text_content = extract_text_content(content)+ text_content = extract_text_content(content, media_marker)if text_content:content_lines, _ = truncate_content(text_content, options)lines.extend(content_lines)diff --git a/products/ai_observability/backend/text_repr/formatters/otel_parts.py b/products/ai_observability/backend/text_repr/formatters/otel_parts.pyindex 388e1162fcdd..51c040208df6 100644--- a/products/ai_observability/backend/text_repr/formatters/otel_parts.py+++ b/products/ai_observability/backend/text_repr/formatters/otel_parts.py@@ -1,5 +1,10 @@import json+from .media_blocks import parse_media_block++# A media part's flattened message keeps the original part here, so a judge with images can attach it.+MEDIA_PART_KEY = "media_part"+def flatten_parts_message(msg: dict) -> list[dict]:"""Convert an OTel GenAI semconv parts-shaped message into flat chat message(s).@@ -27,7 +32,7 @@ def flatten_parts_message(msg: dict) -> list[dict]:- `blob`/`uri`/`file` parts become bracketed text markers (`[image]`,`[image: <uri>]`, `[file: <id>]`). The trace view renders image blobs and urisas actual images; the judge gets markers instead because base64 payloads are- token noise to an LLM+ token noise to an LLM. The original part stays under `MEDIA_PART_KEY`- `compaction` parts become follow-up messages carrying the compacted summary,or a `[conversation compacted]` marker when the summary is absent@@ -76,7 +81,9 @@ def flatten_parts_message(msg: dict) -> list[dict]:if isinstance(reasoning, str) and reasoning:followups.append({"role": "thinking", "content": reasoning})elif part_type in ("blob", "uri", "file"):- followups.append({"role": msg["role"], "content": _media_part_marker(part_type, part)})+ followups.append(+ {"role": msg["role"], "content": _media_part_marker(part_type, part), MEDIA_PART_KEY: part}+ )elif part_type == "compaction":summary = part.get("content")if not (isinstance(summary, str) and summary):@@ -101,13 +108,9 @@ def flatten_parts_message(msg: dict) -> list[dict]:def _media_part_marker(part_type: str, part: dict) -> str:- modality = part.get("modality")- if not isinstance(modality, str) or not modality:- modality = "media"- if part_type == "blob":- return f"[{modality}]"- if part_type == "uri":- uri = part.get("uri")- return f"[{modality}: {uri}]" if isinstance(uri, str) and uri else f"[{modality}]"+ if part_type in ("blob", "uri"):+ # The same rule a judge with images falls back to, so both transcripts match where no image attaches.+ media = parse_media_block(part)+ return media.marker() if media else "[media]"file_id = part.get("file_id")return f"[file: {file_id}]" if isinstance(file_id, str) and file_id else "[file]"diff --git a/products/ai_observability/backend/text_repr/formatters/span_formatter.py b/products/ai_observability/backend/text_repr/formatters/span_formatter.pyindex 80d699f813b5..7e24b6f5e023 100644--- a/products/ai_observability/backend/text_repr/formatters/span_formatter.py+++ b/products/ai_observability/backend/text_repr/formatters/span_formatter.py@@ -105,7 +105,7 @@ def _format_tool_result(state: dict, options: FormatterOptions | None) -> list[slines.append("")# `content` is often a list of content blocks, which `str()` would render as a Python repr.- text = extract_payload_text(content)+ text = extract_payload_text(content, options.get("media_marker") if options else None)if text:content_lines, _ = truncate_content(text, options)lines.extend(content_lines)diff --git a/products/ai_observability/backend/text_repr/formatters/test/test_event_formatter.py b/products/ai_observability/backend/text_repr/formatters/test/test_event_formatter.pyindex e24a9b19dacb..0f7aa32d3bf0 100644--- a/products/ai_observability/backend/text_repr/formatters/test/test_event_formatter.py+++ b/products/ai_observability/backend/text_repr/formatters/test/test_event_formatter.py@@ -475,3 +475,10 @@ class TestSpanToolResultFormatting:result = format_event_text_repr(self._span(state))assert "3 issues found" in resultassert "'type': 'text'" not in result++ def test_tool_result_images_reach_the_media_marker(self):+ image = {"type": "input_image", "image_url": "data:image/png;base64,aGk="}+ state = {"type": "tool", "name": "screenshot", "status": "success", "content": [image]}+ result = format_event_text_repr(self._span(state), {"media_marker": lambda media: "[image 1]"})+ assert "[image 1]" in result+ assert "aGk=" not in resultdiff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsxindex 1645b679a376..f75f3d11e37b 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx@@ -1,6 +1,8 @@import type { Meta, StoryObj } from '@storybook/react'import { BindLogic } from 'kea'+import { FEATURE_FLAGS } from 'lib/constants'+import { useStorybookMocks } from '~/mocks/browser'import { llmEvaluationLogic } from '../llmEvaluationLogic'@@ -119,6 +121,10 @@ export const DecisionNarrow: Story = { args: { method: 'decision', narrow: trueexport const NoDecisionModels: Story = { args: { method: 'decision', empty: true } }export const NativeOpenAI: Story = { args: { method: 'decision', nativeOpenAI: true } }export const NativeOpenAINarrow: Story = { args: { method: 'decision', nativeOpenAI: true, narrow: true } }+export const WithAttachments: Story = {+ args: { method: 'llm' },+ parameters: { featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_JUDGE_IMAGES], testOptions: { snapshotBrowsers: [] } },+}export const ReasoningLevelNarrow: Story = {args: { method: 'llm', nativeOpenAI: true, narrow: true, reasoningLevel: 'medium' },}diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsxindex 79801c6a2d87..55b6aa67405e 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx@@ -1,8 +1,10 @@import { useActions, useValues } from 'kea'-import { LemonSegmentedButton } from '@posthog/lemon-ui'+import { LemonSegmentedButton, LemonSwitch } from '@posthog/lemon-ui'+import { FEATURE_FLAGS } from 'lib/constants'import { LemonField } from 'lib/lemon-ui/LemonField'+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'import { modelPickerLogic } from '../../modelPickerLogic'import { booleanOutputConfigError, decisionModelHint } from '../constants'@@ -13,8 +15,9 @@ import { EvaluationJudgeModelPicker } from './EvaluationJudgeModelPicker'export function EvaluationModelPicker(): JSX.Element {const { byokModelsLoading, providerKeysLoading } = useValues(modelPickerLogic)- const { evaluation, usesDecisionModel, judgeMethod } = useValues(llmEvaluationLogic)- const { setJudgeMethod } = useActions(llmEvaluationLogic)+ const { evaluation, usesDecisionModel, judgeMethod, includeMedia } = useValues(llmEvaluationLogic)+ const { setJudgeMethod, setIncludeMedia } = useActions(llmEvaluationLogic)+ const { featureFlags } = useValues(featureFlagLogic)const loading = byokModelsLoading || providerKeysLoading@@ -61,6 +64,22 @@ export function EvaluationModelPicker(): JSX.Element {)}</div></LemonField.Pure>+ {featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_JUDGE_IMAGES] &&+ evaluation?.evaluation_type === 'llm_judge' &&+ !usesDecisionModel && (+ <LemonField.Pure label="Attachments">+ <LemonSwitch+ checked={includeMedia}+ onChange={setIncludeMedia}+ bordered+ label="Send to the judge"+ data-attr="llma-evaluation-include-attachments"+ />+ <p className="text-muted text-sm">+ The judge sees the images in what it evaluates. Other file types are not supported yet.+ </p>+ </LemonField.Pure>+ )}{evaluation?.evaluation_type === 'llm_judge' &&evaluation.output_type === 'boolean' &&usesDecisionModel && (diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.tsindex 5b58de8384bc..f3553e77ca71 100644--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.test.ts@@ -444,6 +444,20 @@ describe('llmEvaluationLogic', () => {})})+ it('sends an explicit attachments choice so turning it back on is saved', async () => {+ await expectLogic(logic).toDispatchActions(['loadEvaluationSuccess'])++ logic.actions.setIncludeMedia(false)+ expect(logic.values.evaluation?.evaluation_config).toEqual(+ expect.objectContaining({ include_media: false })+ )+ expect(logic.values.includeMedia).toBe(false)++ logic.actions.setIncludeMedia(true)+ expect(logic.values.evaluation?.evaluation_config).toEqual(expect.objectContaining({ include_media: true }))+ expect(logic.values.includeMedia).toBe(true)+ })+it('records the polarity on the boolean output config', async () => {await expectLogic(logic, () => {logic.actions.loadEvaluationSuccess({ ...mockEvaluation })diff --git a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.tsindex 502d0b0cc659..e798e7ee913e 100644--- a/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts+++ b/products/ai_observability/frontend/evaluations/llmEvaluationLogic.ts@@ -343,6 +343,7 @@ export interface llmEvaluationLogicValues {hogTestMessage: string | nullhogTestResults: TestHogResultItemApi[] | nullhogTestResultsLoading: boolean+ includeMedia: booleanisForceRefresh: booleanisNewEvaluation: booleanisReportableEvaluation: boolean@@ -536,6 +537,9 @@ export interface llmEvaluationLogicActions {setHogTestMessage: (message: string | null) => {message: string | null}+ setIncludeMedia: (includeMedia: boolean) => {+ includeMedia: boolean+ }setJudgeMethod: (judgeMethod: JudgeMethod) => {judgeMethod: JudgeMethod}@@ -607,6 +611,7 @@ export interface llmEvaluationLogicMeta {selectedModelOption: (evaluation: EvaluationConfig | null, byokModels: ModelOption[]) => ModelOption | nulljudgeMethod: (evaluation: EvaluationConfig | null, selectedModelOption: ModelOption | null) => JudgeMethodusesDecisionModel: (judgeMethod: JudgeMethod) => boolean+ includeMedia: (evaluation: EvaluationConfig | null) => booleanreasoningLevel: (evaluation: EvaluationConfig | null) => ReasoningLevel | nullselectedModelSupportsReasoning: (selectedModelOption: ModelOption | null, usesDecisionModel: boolean) => booleanselectedModelName: (@@ -704,6 +709,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([setTriggerConditions: (conditions: EvaluationConditionSet[]) => ({ conditions }),setModelConfiguration: (modelConfiguration: ModelConfiguration | null) => ({ modelConfiguration }),setJudgeMethod: (judgeMethod: JudgeMethod) => ({ judgeMethod }),+ setIncludeMedia: (includeMedia: boolean) => ({ includeMedia }),setReasoningLevel: (reasoningLevel: ReasoningLevel | null) => ({ reasoningLevel }),setEvaluationType: (evaluationType: EvaluationType) => ({ evaluationType }),setEvaluationTarget: (target: EvaluationTarget) => ({ target }),@@ -1042,6 +1048,11 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([: state.output_config,}: state,+ // Always explicit: a PATCH without the field keeps the saved value.+ setIncludeMedia: (state, { includeMedia }) =>+ state && isLLMJudgeEvaluation(state)+ ? { ...state, evaluation_config: { ...state.evaluation_config, include_media: includeMedia } }+ : state,setEvaluationType: (state, { evaluationType }) => {if (!state) {return null@@ -1192,6 +1203,7 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([setTriggerConditions: () => true,setModelConfiguration: () => true,setJudgeMethod: () => true,+ setIncludeMedia: () => true,setReasoningLevel: () => true,setEvaluationType: () => true,setEvaluationTarget: () => true,@@ -1776,6 +1788,11 @@ export const llmEvaluationLogic = kea<llmEvaluationLogicType>([},],usesDecisionModel: [(s) => [s.judgeMethod], (judgeMethod: JudgeMethod): boolean => judgeMethod === 'decision'],+ includeMedia: [+ (s) => [s.evaluation],+ (evaluation: EvaluationConfig | null): boolean =>+ !isLLMJudgeEvaluation(evaluation) || evaluation.evaluation_config?.include_media !== false,+ ],reasoningLevel: [(s) => [s.evaluation],(evaluation: EvaluationConfig | null): ReasoningLevel | null =>diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.tsindex c352b9432d93..22e5876ab9f2 100644--- a/products/ai_observability/frontend/generated/api.schemas.ts+++ b/products/ai_observability/frontend/generated/api.schemas.ts@@ -2363,6 +2363,8 @@ export type EvaluationApiEvaluationConfig =* @nullable*/judge_method?: 'llm' | 'decision' | null+ /** Set to false to judge text only. */+ include_media?: boolean/*** How much the judge model reasons before it answers. Only Anthropic and OpenAI reasoning models with the 'llm' judge method support it. Omit to keep the saved level; null uses the model's default.* @nullable@@ -2697,6 +2699,8 @@ export type PatchedEvaluationApiEvaluationConfig =* @nullable*/judge_method?: 'llm' | 'decision' | null+ /** Set to false to judge text only. */+ include_media?: boolean/*** How much the judge model reasons before it answers. Only Anthropic and OpenAI reasoning models with the 'llm' judge method support it. Omit to keep the saved level; null uses the model's default.* @nullablediff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.tsindex 91e12dbbdce9..477a9fa04acc 100644--- a/products/ai_observability/frontend/generated/api.zod.ts+++ b/products/ai_observability/frontend/generated/api.zod.ts@@ -620,6 +620,7 @@ export const EvaluationsCreateBody = /* @__PURE__ */ zod.enum(['llm', 'decision']).nullish().describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),reasoning_level: zod.enum(['minimal', 'low', 'medium', 'high']).nullish()@@ -1021,6 +1022,7 @@ export const EvaluationsUpdateBody = /* @__PURE__ */ zod.enum(['llm', 'decision']).nullish().describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),reasoning_level: zod.enum(['minimal', 'low', 'medium', 'high']).nullish()@@ -1325,6 +1327,7 @@ export const EvaluationsPartialUpdateBody = /* @__PURE__ */ zod.enum(['llm', 'decision']).nullish().describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),reasoning_level: zod.enum(['minimal', 'low', 'medium', 'high']).nullish()diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 91fc3813119d..62de6013b5ed 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -9228,6 +9228,7 @@ export namespace Schemas {judge_method?:| (("llm" | "decision") | null)| undefined;+ include_media?: boolean | undefined;reasoning_level?:| (("minimal" | "low" | "medium" | "high") | null)| undefined;@@ -19483,6 +19484,7 @@ export namespace Schemas {| {prompt: string;judge_method?: (("llm" | "decision") | null) | undefined;+ include_media?: boolean | undefined;reasoning_level?:| (("minimal" | "low" | "medium" | "high") | null)| undefined;diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 508a1bcacbba..11e7cdeec398 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -45316,6 +45316,8 @@ export namespace Schemas {* @nullable*/judge_method?: 'llm' | 'decision' | null;+ /** Set to false to judge text only. */+ include_media?: boolean;/*** How much the judge model reasons before it answers. Only Anthropic and OpenAI reasoning models with the 'llm' judge method support it. Omit to keep the saved level; null uses the model's default.* @nullable@@ -87695,6 +87697,8 @@ export namespace Schemas {* @nullable*/judge_method?: 'llm' | 'decision' | null;+ /** Set to false to judge text only. */+ include_media?: boolean;/*** How much the judge model reasons before it answers. Only Anthropic and OpenAI reasoning models with the 'llm' judge method support it. Omit to keep the saved level; null uses the model's default.* @nullablediff --git a/services/mcp/src/generated/ai_observability/api.ts b/services/mcp/src/generated/ai_observability/api.tsindex 7bdbe19643b3..4944eee9f83f 100644--- a/services/mcp/src/generated/ai_observability/api.ts+++ b/services/mcp/src/generated/ai_observability/api.ts@@ -1435,6 +1435,7 @@ export const EvaluationsCreateBody = () => zod.enum(['llm', 'decision']).nullish().describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),reasoning_level: zod.enum(['minimal', 'low', 'medium', 'high']).nullish()@@ -1916,6 +1917,7 @@ export const EvaluationsPartialUpdateBody = () => zod.enum(['llm', 'decision']).nullish().describe('Judge method. Omit to keep the saved method; null restores automatic routing.'),+ include_media: zod.boolean().optional().describe('Set to false to judge text only.'),reasoning_level: zod.enum(['minimal', 'low', 'medium', 'high']).nullish()diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.jsonindex 2211a7d9905a..7e7e386739ed 100644--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-create.json@@ -63,6 +63,10 @@"anyOf": [{"properties": {+ "include_media": {+ "description": "Set to false to judge text only.",+ "type": "boolean"+ },"judge_method": {"anyOf": [{diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.jsonindex d4eba03e9095..e0d197215fc8 100644--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/llma-evaluation-update.json@@ -62,6 +62,10 @@"anyOf": [{"properties": {+ "include_media": {+ "description": "Set to false to judge text only.",+ "type": "boolean"+ },"judge_method": {"anyOf": [{