Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): send judge attachments to openrouter by model input (#127209)

Bernat Torres7b1665ae8055
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 5829646c31a7..08a413a44c0d 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -45,9 +45,10 @@ If the output and section labels alone exceed that limit, the run is skipped wit
An exception applies when the judge sees images (the `llm-analytics-judge-images` flag is on and the evaluation does not turn attachments off).
Then each image becomes an `[image N]` marker and each PDF a `[file N]` marker instead of its base64 or link, so an output with large inline attachments can fit and the run continues.
The attachment labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 attachments, plus one summary line for the rest.
-PDFs reach only OpenAI, Anthropic and Gemini judges, up to 10 MB each.
+PDFs reach OpenAI, Anthropic and Gemini judges, up to 10 MB each.
WAV and MP3 audio becomes an `[audio N]` marker and reaches Gemini judges and OpenAI audio models, always inline.
-MP4, MPEG, QuickTime and WebM video becomes a `[video N]` marker and reaches Gemini judges only, up to 15 MB each.
+MP4, MPEG, QuickTime and WebM video becomes a `[video N]` marker and reaches Gemini judges, up to 15 MB each.
+OpenRouter judges get each type that OpenRouter's model catalogue lists as an input of their model. PDFs need the `file` input, so OpenRouter never parses them with its paid OCR engine. Everything except images goes inline, and a model the catalogue doesn't list gets images only.
Attachments a judge can't read take none of its 20 slots, and one summary line counts them.
Generation evaluation and LLM tagger user prompts repair UTF-16 surrogate sequences after sampling.
diff --git a/posthog/temporal/ai_observability/judge_attachments.py b/posthog/temporal/ai_observability/judge_attachments.py
index 056f94dcf3eb..c8aee9e764c4 100644
--- a/posthog/temporal/ai_observability/judge_attachments.py
+++ b/posthog/temporal/ai_observability/judge_attachments.py
@@ -129,8 +129,12 @@ class JudgeAttachments:
if not images:
return user_prompt
+ # One lookup per file type: on OpenRouter each one reads the whole model catalogue from the cache.
+ transports = {
+ mime: attachment_transport(provider, model, mime) for mime in {image.mime_type for image in images}
+ }
# Attachments this judge can't read take no slot, so they can't push out ones it can.
- readable = [image for image in images if attachment_transport(provider, model, image.mime_type) is not None]
+ readable = [image for image in images if transports[image.mime_type] is not None]
# Keep the images referenced last: the newest input and the graded output, even when it repeats an old image.
ranked = sorted(readable, key=lambda image: user_prompt.rfind(image.marker))
# Gemini rejects a request over its video limit, and the text-only retry would drop everything else too.
@@ -147,7 +151,7 @@ class JudgeAttachments:
attachments: dict[str, _Attachment] = {}
inline_bytes = 0
for image in kept:
- attachment = self._attach(image, provider, model, MAX_INLINE_BYTES - inline_bytes)
+ attachment = self._attach(image, provider, transports[image.mime_type], MAX_INLINE_BYTES - inline_bytes)
inline_bytes += attachment.inline_bytes
attachments[image.marker] = attachment
@@ -181,8 +185,9 @@ class JudgeAttachments:
logger.info("judge_attachments_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)
return parts
- def _attach(self, image: _Media, provider: str, model: str, inline_budget: int) -> _Attachment:
- transport = attachment_transport(provider, model, image.mime_type)
+ def _attach(
+ self, image: _Media, provider: str, transport: Literal["url", "inline"] | None, inline_budget: int
+ ) -> _Attachment:
if transport is None:
return _Attachment(outcome="unavailable")
provider_limit = MAX_IMAGE_BYTES.get(provider) if image.kind == "image" else _BYTE_LIMITS.get(image.kind)
diff --git a/posthog/temporal/ai_observability/test_judge_attachments.py b/posthog/temporal/ai_observability/test_judge_attachments.py
index 0d64a5be387b..e1d9052c6f88 100644
--- a/posthog/temporal/ai_observability/test_judge_attachments.py
+++ b/posthog/temporal/ai_observability/test_judge_attachments.py
@@ -15,6 +15,11 @@ from products.ai_observability.backend.llm.attachments import (
from products.ai_observability.backend.text_repr.formatters.media_blocks import MediaBlock, parse_media_block
OBJECT_STORAGE = "products.ai_observability.backend.ai_blobs.object_storage"
+OPENROUTER_CATALOGUE = {
+ "openai/gpt-5-mini": frozenset({"text", "image", "file"}),
+ "google/gemini-2.5-flash": frozenset({"text", "image", "file", "audio", "video"}),
+ "example/image-only": frozenset({"text", "image"}),
+}
def _pointer(digit: str, mime: str = "image%2Fjpeg") -> MediaBlock:
@@ -59,6 +64,14 @@ class TestJudgeAttachmentsMarkers:
class TestJudgeAttachmentsContent:
+ @pytest.fixture(autouse=True)
+ def openrouter_catalogue(self):
+ with patch(
+ "products.ai_observability.backend.llm.providers.openrouter.model_input_modalities",
+ side_effect=OPENROUTER_CATALOGUE.get,
+ ) as lookup:
+ yield lookup
+
def test_an_image_referenced_again_at_the_end_stays_attached(self):
images = JudgeAttachments(team_id=7)
markers = [images.marker(_inline(f"{i:02}")) for i in range(MAX_ATTACHMENTS + 1)]
@@ -191,11 +204,43 @@ class TestJudgeAttachmentsContent:
),
pytest.param(
"openrouter",
- "any",
+ "openai/gpt-5-mini",
+ "application%2Fpdf",
+ "File 1:",
+ AttachmentPart(mime_type="application/pdf", data=b"hi"),
+ id="pdf_on_an_openrouter_model_that_reads_files",
+ ),
+ pytest.param(
+ "openrouter",
+ "google/gemini-2.5-flash",
+ "video%2Fmp4",
+ "Video 1:",
+ AttachmentPart(mime_type="video/mp4", data=b"hi"),
+ id="video_on_an_openrouter_model_that_reads_video",
+ ),
+ pytest.param(
+ "openrouter",
+ "openai/gpt-5-mini",
+ "video%2Fmp4",
+ "Not attached, this judge can't read their file type: 1.",
+ None,
+ id="video_on_an_openrouter_model_without_video_input",
+ ),
+ pytest.param(
+ "openrouter",
+ "example/unlisted",
"application%2Fpdf",
"Not attached, this judge can't read their file type: 1.",
None,
- id="pdf_unsupported_provider",
+ id="pdf_on_a_model_the_openrouter_catalogue_does_not_list",
+ ),
+ pytest.param(
+ "openrouter",
+ "example/unlisted",
+ "image%2Fpng",
+ "Image 1:",
+ AttachmentPart(mime_type="image/png", url="https://signed"),
+ id="image_on_a_model_the_openrouter_catalogue_does_not_list",
),
],
)
@@ -349,7 +394,7 @@ class TestJudgeAttachmentsContent:
assert f"Video {11 - attached}:" not in labels
assert f"Not attached, over the judge's attachment limit: {11 - attached}." in labels
- def test_attachments_the_judge_cannot_read_take_no_slot(self):
+ def test_attachments_the_judge_cannot_read_take_no_slot(self, openrouter_catalogue):
images = JudgeAttachments(team_id=7)
image = images.marker(_inline("aa"))
pdfs = " ".join(
@@ -357,13 +402,15 @@ class TestJudgeAttachmentsContent:
for i in range(MAX_ATTACHMENTS)
)
- content = images.content(f"user: {image} {pdfs}", "openrouter", "openai/gpt-5-mini")
+ content = images.content(f"user: {image} {pdfs}", "openrouter", "example/image-only")
assert content[1:4] == [
{"type": "text", "text": "Image 1:"},
AttachmentPart(mime_type="image/png", data=base64.b64decode("aa" * 4)),
{"type": "text", "text": f"Not attached, this judge can't read their file type: {MAX_ATTACHMENTS}."},
]
+ # Each lookup reads the whole OpenRouter catalogue, so it runs once per file type, not per attachment.
+ assert openrouter_catalogue.call_count == 2
def test_an_inline_image_exactly_at_the_provider_limit_is_attached(self):
images = JudgeAttachments(team_id=7)
diff --git a/products/ai_observability/backend/llm/attachments.py b/products/ai_observability/backend/llm/attachments.py
index 6dac120e9f9c..2e19222c2a84 100644
--- a/products/ai_observability/backend/llm/attachments.py
+++ b/products/ai_observability/backend/llm/attachments.py
@@ -9,11 +9,10 @@ IMAGE_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/webp", "image/gi
PDF_MIME_TYPE = "application/pdf"
# The formats OpenAI's chat API takes as input audio, which Gemini reads too.
AUDIO_FORMATS = {"audio/wav": "wav", "audio/mpeg": "mp3"}
-# Only Gemini reads video.
VIDEO_MIME_TYPES = frozenset({"video/mp4", "video/mpeg", "video/quicktime", "video/webm"})
# Providers whose image blocks take a link that they download themselves.
-_URL_PROVIDERS = frozenset({"openai", "azure_openai", "anthropic", "openrouter", "gemini"})
-# Providers that read PDFs natively. OpenRouter is left out: it parses PDFs for other models with a paid OCR engine.
+_URL_PROVIDERS = frozenset({"openai", "azure_openai", "anthropic", "gemini"})
+# Providers that read PDFs natively.
_PDF_PROVIDERS = frozenset({"openai", "anthropic", "gemini"})
# Raw bytes per image. Anthropic allows 10 MB base64-encoded, about 7.5 MB raw, so keep a margin under it.
MAX_IMAGE_BYTES = {"anthropic": 7_000_000, "gemini": 15_000_000}
@@ -51,6 +50,8 @@ class AttachmentPart:
def attachment_transport(provider: str, model: str, mime_type: str) -> Literal["url", "inline"] | None:
"""How a provider accepts an attachment of this type: a link, inline base64, or not at all."""
+ if provider == "openrouter":
+ return _openrouter_transport(model, mime_type)
openai_audio_model = provider == "openai" and "audio" in model
if mime_type in AUDIO_FORMATS:
# OpenAI takes audio only as inline base64, and only on its audio models, so audio always goes inline.
@@ -79,9 +80,34 @@ def attachment_transport(provider: str, model: str, mime_type: str) -> Literal["
return "url" if provider in _URL_PROVIDERS else "inline"
+def _openrouter_transport(model: str, mime_type: str) -> Literal["url", "inline"] | None:
+ """OpenRouter routes many models, so its catalogue says what each one reads."""
+ # Avoids a circular import: the OpenRouter adapter imports this module through the OpenAI one.
+ from products.ai_observability.backend.llm.providers.openrouter import model_input_modalities # noqa: PLC0415
+
+ if mime_type in IMAGE_MIME_TYPES:
+ modality = "image"
+ elif mime_type == PDF_MIME_TYPE:
+ # "file" means the model reads PDFs itself. Other models would get OpenRouter's paid OCR.
+ modality = "file"
+ elif mime_type in AUDIO_FORMATS:
+ modality = "audio"
+ elif mime_type in VIDEO_MIME_TYPES:
+ modality = "video"
+ else:
+ return None
+ modalities = model_input_modalities(model)
+ if modalities is None:
+ return "url" if modality == "image" else None
+ if modality not in modalities:
+ return None
+ # OpenRouter forwards a link only where the model's provider downloads it, and Gemini on AI Studio doesn't.
+ return "url" if modality == "image" else "inline"
+
+
def max_videos(model: str) -> int:
"""Videos Gemini accepts in one request."""
- return 1 if model.startswith(("gemini-1.", "gemini-2.0")) else 10
+ return 1 if model.removeprefix("google/").startswith(("gemini-1.", "gemini-2.0")) else 10
def convert_attachment_parts(
diff --git a/products/ai_observability/backend/llm/providers/openai.py b/products/ai_observability/backend/llm/providers/openai.py
index 90e6316ae8b8..d19ee107da8c 100644
--- a/products/ai_observability/backend/llm/providers/openai.py
+++ b/products/ai_observability/backend/llm/providers/openai.py
@@ -311,7 +311,7 @@ You must respond with valid JSON that matches this schema:
Return ONLY the JSON object, no other text or markdown formatting."""
enhanced_messages = [{"role": "system", "content": enhanced_system}]
- enhanced_messages.extend(convert_attachment_parts(request.messages, _openai_attachment_block))
+ enhanced_messages.extend(convert_attachment_parts(request.messages, self._attachment_block))
create_response = client.chat.completions.create(
model=request.model,
@@ -516,11 +516,14 @@ Return ONLY the JSON object, no other text or markdown formatting."""
def _get_default_headers(self) -> dict[str, str]:
return {}
+ def _attachment_block(self, part: AttachmentPart) -> dict[str, Any]:
+ return _openai_attachment_block(part)
+
def _build_messages(self, request: CompletionRequest) -> list[dict]:
messages = []
if request.system:
messages.append({"role": "system", "content": request.system})
- messages.extend(convert_attachment_parts(request.messages, _openai_attachment_block))
+ messages.extend(convert_attachment_parts(request.messages, self._attachment_block))
return messages
def _build_analytics_kwargs(self, analytics: AnalyticsContext, client) -> dict:
diff --git a/products/ai_observability/backend/llm/providers/openrouter.py b/products/ai_observability/backend/llm/providers/openrouter.py
index ead3a364441a..a85bee667f46 100644
--- a/products/ai_observability/backend/llm/providers/openrouter.py
+++ b/products/ai_observability/backend/llm/providers/openrouter.py
@@ -6,13 +6,14 @@ from many providers. It is BYOKEY-only (no PostHog-funded key).
import logging
from collections.abc import Generator
-from typing import Any
+from typing import Any, Literal
from django.core.cache import cache
import httpx
import openai
+from products.ai_observability.backend.llm.attachments import VIDEO_MIME_TYPES, AttachmentPart
from products.ai_observability.backend.llm.errors import UnsupportedModelError
from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter, OpenAIConfig
from products.ai_observability.backend.llm.types import (
@@ -38,40 +39,62 @@ OPENROUTER_ALL_MODELS_URL = f"{OPENROUTER_BASE_URL}/models?output_modalities=all
# Older workers expect a non-chat-only snapshot, so use a separate cache namespace.
MODEL_MODALITIES_CACHE_KEY = "ai_observability:openrouter:model_output_modalities:v1"
MODEL_MODALITIES_LAST_GOOD_CACHE_KEY = f"{MODEL_MODALITIES_CACHE_KEY}:last_good"
+MODEL_INPUT_MODALITIES_CACHE_KEY = "ai_observability:openrouter:model_input_modalities:v1"
+MODEL_INPUT_MODALITIES_LAST_GOOD_CACHE_KEY = f"{MODEL_INPUT_MODALITIES_CACHE_KEY}:last_good"
MODEL_MODALITIES_CACHE_TTL_SECONDS = 60 * 60
MODEL_MODALITIES_FETCH_TIMEOUT_SECONDS = 5.0
# Short, so a catalogue outage adds the fetch timeout at most once a minute.
MODEL_MODALITIES_UNAVAILABLE_TTL_SECONDS = 60
_CATALOGUE_UNAVAILABLE = "unavailable"
+_CACHE_KEYS = {
+ "output": (MODEL_MODALITIES_CACHE_KEY, MODEL_MODALITIES_LAST_GOOD_CACHE_KEY),
+ "input": (MODEL_INPUT_MODALITIES_CACHE_KEY, MODEL_INPUT_MODALITIES_LAST_GOOD_CACHE_KEY),
+}
-def _model_output_modalities(*, refresh: bool = True) -> dict[str, list[str]] | None:
- """Output modalities of OpenRouter models.
+def _model_modalities(direction: Literal["input", "output"], *, refresh: bool = True) -> dict[str, list[str]] | None:
+ """Input or output modalities of OpenRouter models.
Returns the last successful catalogue during outages, or None before the first success.
"""
- cached = cache.get(MODEL_MODALITIES_CACHE_KEY)
+ cache_key, last_good_key = _CACHE_KEYS[direction]
+ cached = cache.get(cache_key)
if cached == _CATALOGUE_UNAVAILABLE:
- return cache.get(MODEL_MODALITIES_LAST_GOOD_CACHE_KEY)
+ return cache.get(last_good_key)
if cached is not None:
return cached
if not refresh:
- return cache.get(MODEL_MODALITIES_LAST_GOOD_CACHE_KEY)
+ return cache.get(last_good_key)
try:
response = httpx.get(OPENROUTER_ALL_MODELS_URL, timeout=MODEL_MODALITIES_FETCH_TIMEOUT_SECONDS)
response.raise_for_status()
models = response.json()["data"]
- modalities = {
- model["id"]: (model.get("architecture") or {}).get("output_modalities") or ["text"] for model in models
+ catalogue = {
+ name: {
+ model["id"]: (model.get("architecture") or {}).get(f"{name}_modalities") or ["text"] for model in models
+ }
+ for name in _CACHE_KEYS
}
except Exception:
logger.warning("Could not fetch the OpenRouter model catalogue", exc_info=True)
- cache.set(MODEL_MODALITIES_CACHE_KEY, _CATALOGUE_UNAVAILABLE, MODEL_MODALITIES_UNAVAILABLE_TTL_SECONDS)
- return cache.get(MODEL_MODALITIES_LAST_GOOD_CACHE_KEY)
- # An outage must not discard the last successful catalogue.
- cache.set(MODEL_MODALITIES_LAST_GOOD_CACHE_KEY, modalities, timeout=None)
- cache.set(MODEL_MODALITIES_CACHE_KEY, modalities, MODEL_MODALITIES_CACHE_TTL_SECONDS)
- return modalities
+ for key, _ in _CACHE_KEYS.values():
+ cache.set(key, _CATALOGUE_UNAVAILABLE, MODEL_MODALITIES_UNAVAILABLE_TTL_SECONDS)
+ return cache.get(last_good_key)
+ for name, (key, last_good) in _CACHE_KEYS.items():
+ # An outage must not discard the last successful catalogue.
+ cache.set(last_good, catalogue[name], timeout=None)
+ cache.set(key, catalogue[name], MODEL_MODALITIES_CACHE_TTL_SECONDS)
+ return catalogue[direction]
+
+
+def _model_output_modalities(*, refresh: bool = True) -> dict[str, list[str]] | None:
+ return _model_modalities("output", refresh=refresh)
+
+
+def model_input_modalities(model: str) -> frozenset[str] | None:
+ """What a model reads, such as text, image, file, audio or video. None when the catalogue doesn't list it."""
+ models = _model_modalities("input")
+ return frozenset(models[model]) if models is not None and model in models else None
def non_chat_model_ids() -> frozenset[str] | None:
@@ -110,6 +133,11 @@ class OpenRouterAdapter(OpenAIAdapter):
def _get_default_headers(self) -> dict[str, str]:
return OPENROUTER_HEADERS
+ def _attachment_block(self, part: AttachmentPart) -> dict[str, Any]:
+ if part.mime_type in VIDEO_MIME_TYPES:
+ return {"type": "video_url", "video_url": {"url": part.data_uri()}}
+ return super()._attachment_block(part)
+
def complete(
self,
request: CompletionRequest,
diff --git a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
index 7280577a605a..6b28aa0fcfa2 100644
--- a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
+++ b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
@@ -21,6 +21,7 @@ from products.ai_observability.backend.llm.errors import (
StructuredOutputParseError,
)
from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter, OpenAIConfig
+from products.ai_observability.backend.llm.providers.openrouter import OpenRouterAdapter
from products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema
@@ -230,6 +231,22 @@ class TestOpenAIAttachmentParts:
{"type": "text", "text": "hi"},
]
+ def test_openrouter_sends_video_as_a_video_url_block(self):
+ request = CompletionRequest(
+ model="google/gemini-2.5-flash",
+ messages=[{"role": "user", "content": [AttachmentPart(mime_type="video/mp4", data=b"hi")]}],
+ provider="openrouter",
+ )
+ mock_client = MagicMock()
+ mock_client.chat.completions.create.return_value.usage = None
+ mock_client.chat.completions.create.return_value.choices[0].message.content = "ok"
+ with patch("products.ai_observability.backend.llm.providers.openai.openai.OpenAI", return_value=mock_client):
+ OpenRouterAdapter().complete(request, api_key="sk-test", analytics=AnalyticsContext(capture=False))
+
+ assert mock_client.chat.completions.create.call_args.kwargs["messages"][0]["content"] == [
+ {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,aGk="}}
+ ]
+
class TestOpenAICompleteReasoningEffort:
@parameterized.expand(
diff --git a/products/ai_observability/backend/llm/providers/test/test_openrouter.py b/products/ai_observability/backend/llm/providers/test/test_openrouter.py
index 5d7dacb9110c..8b3db6940862 100644
--- a/products/ai_observability/backend/llm/providers/test/test_openrouter.py
+++ b/products/ai_observability/backend/llm/providers/test/test_openrouter.py
@@ -18,6 +18,7 @@ from products.ai_observability.backend.llm.providers.openrouter import (
OPENROUTER_HEADERS,
OpenRouterAdapter,
decision_model_ids,
+ model_input_modalities,
non_chat_model_ids,
)
@@ -180,7 +181,10 @@ class TestOpenRouterNonChatModels:
mock_response = MagicMock()
mock_response.json.return_value = {
"data": [
- {"id": "openai/gpt-4o", "architecture": {"output_modalities": ["text"]}},
+ {
+ "id": "openai/gpt-4o",
+ "architecture": {"output_modalities": ["text"], "input_modalities": ["text", "image", "file"]},
+ },
{"id": "google/image-model", "architecture": {"output_modalities": ["image", "text"]}},
{"id": "example/dual-model", "architecture": {"output_modalities": ["text", "decisions"]}},
{"id": "typesafe/jev-1.13", "architecture": {"output_modalities": ["decisions"]}},
@@ -227,6 +231,9 @@ class TestOpenRouterNonChatModels:
)
assert decision_model_ids() == expected_decisions
assert decision_model_ids(decision_only=True) == expected_decisions - {"example/dual-model"}
+ assert model_input_modalities("openai/gpt-4o") == frozenset({"text", "image", "file"})
+ assert model_input_modalities("no-architecture/model") == frozenset({"text"})
+ assert model_input_modalities("example/unlisted") is None
@pytest.mark.parametrize("cached_decisions", [None, False, True])
def test_catalogue_failure_keeps_last_success_and_recovers(self, cached_decisions: bool | None) -> None:
diff --git a/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx
index e4fd336525fe..de67dc3745a1 100644
--- a/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx
+++ b/products/ai_observability/frontend/evaluations/create/EvaluationModelPanel.tsx
@@ -46,7 +46,8 @@ export function EvaluationModelPanel(): JSX.Element | null {
<p className="text-muted text-sm">
The judge sees the images, PDFs, audio and video in what it evaluates. PDFs need an OpenAI,
Anthropic or Gemini judge, audio a Gemini judge or an OpenAI audio model, and video a Gemini
- judge. Other file types are not supported yet.
+ judge. On OpenRouter, the judge gets the types its model reads. Other file types are not
+ supported yet.
</p>
</LemonField.Pure>
)}