Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): read more audio formats and mark media the judge never got (#128923)

Bernat Torres1f20106c610f
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 74ff1526d9b3..eb2202b99938 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -46,10 +46,12 @@ An exception applies when the judge sees images (the `llm-analytics-judge-images
Then each image becomes an `[image N]` marker and each PDF a `[file N]` marker instead of its base64 or link, so an output with large inline attachments can fit and the run continues.
The attachment labels sent next to the transcript sit outside the 150,000-character limit, but they are bounded: at most 20 attachments, plus one summary line that names the rest.
PDFs reach OpenAI, Anthropic and Gemini judges, up to 10 MB each.
-WAV and MP3 audio becomes an `[audio N]` marker and reaches Gemini judges and OpenAI audio models, always inline.
+WAV, MP3, AAC, AIFF, FLAC and OGG audio becomes an `[audio N]` marker and always goes inline.
+Gemini judges read all six. OpenAI audio models and non-Gemini OpenRouter models get WAV and MP3 only.
MP4, MPEG, QuickTime and WebM video becomes a `[video N]` marker and reaches Gemini judges, up to 15 MB each.
OpenRouter judges get each type that OpenRouter's model catalogue lists as an input of their model. PDFs need the `file` input, so OpenRouter never parses them with its paid OCR engine. Everything except images goes inline, and a model the catalogue doesn't list gets images only.
Attachments a judge can't read take none of its 20 slots, and one summary line names them.
+Media the event never carried, such as an SDK's `[base64 image redacted]` placeholder, becomes `[image, not attached]`.
When any attachment is left out, the judge is told it can't see it and not to guess what it contains.
Generation evaluation and LLM tagger user prompts repair UTF-16 surrogate sequences after sampling.
diff --git a/posthog/temporal/ai_observability/judge_attachments.py b/posthog/temporal/ai_observability/judge_attachments.py
index bb8ac3e8b7d3..b26436c12503 100644
--- a/posthog/temporal/ai_observability/judge_attachments.py
+++ b/posthog/temporal/ai_observability/judge_attachments.py
@@ -119,7 +119,10 @@ class JudgeAttachments:
key = pointer.sha256 if pointer else media.base64_data
kind = _kind(media.modality, mime_type)
if kind is None or mime_type is None or key is None:
- return media.marker()
+ if media.url is not None and media.url.startswith(("https://", "http://")):
+ return media.marker()
+ # Covers an SDK placeholder like `[base64 image redacted]`: a bare `[image]` reads as if the judge had it.
+ return f"[{media.modality}, not attached]"
if (image := self._media_by_key.get(key)) is None:
self._counts[kind] += 1
image = self._media_by_key[key] = _Media(
@@ -136,8 +139,9 @@ class JudgeAttachments:
"""The user message content: each referenced attachment under its label, then the transcript."""
# Line sampling can drop a marker from a long transcript, and then its attachment only costs tokens.
images = [image for image in self._media if image.marker in user_prompt]
+ unattached = ", not attached]" in user_prompt
if not images:
- return user_prompt
+ return f"{_NOT_ATTACHED_NOTE}\n\n{user_prompt}" if unattached else user_prompt
# One lookup per file type: on OpenRouter each one reads the whole model catalogue from the cache.
transports = {
@@ -196,7 +200,12 @@ class JudgeAttachments:
"text": f"Not attached, this judge can't read their file type: {_names(unreadable)}.",
}
)
- if over_limit or unreadable or any(attachment.part is None for attachment in attachments.values()):
+ if (
+ unattached
+ or over_limit
+ or unreadable
+ or any(attachment.part is None for attachment in attachments.values())
+ ):
parts.append({"type": "text", "text": _NOT_ATTACHED_NOTE})
parts.append({"type": "text", "text": user_prompt})
logger.info("judge_attachments_attached", provider=provider, model=model, inline_bytes=inline_bytes, **outcomes)
@@ -272,6 +281,9 @@ _MIME_ALIASES = {
"audio/mp3": "audio/mpeg",
"audio/x-wav": "audio/wav",
"audio/wave": "audio/wav",
+ "audio/x-aac": "audio/aac",
+ "audio/x-aiff": "audio/aiff",
+ "audio/x-flac": "audio/flac",
"video/mpg": "video/mpeg",
}
@@ -279,7 +291,8 @@ _MIME_ALIASES = {
def _normalize_mime_type(mime_type: str | None) -> str | None:
if mime_type is None:
return None
- normalized = mime_type.strip().lower()
+ # Parameters such as `codecs=opus` don't change which file type a provider accepts.
+ normalized = mime_type.split(";", 1)[0].strip().lower()
return _MIME_ALIASES.get(normalized, normalized)
diff --git a/posthog/temporal/ai_observability/test_judge_attachments.py b/posthog/temporal/ai_observability/test_judge_attachments.py
index dd939d4a453e..2a090b579231 100644
--- a/posthog/temporal/ai_observability/test_judge_attachments.py
+++ b/posthog/temporal/ai_observability/test_judge_attachments.py
@@ -19,6 +19,7 @@ OPENROUTER_CATALOGUE = {
"openai/gpt-5-mini": frozenset({"text", "image", "file"}),
"google/gemini-2.5-flash": frozenset({"text", "image", "file", "audio", "video"}),
"example/image-only": frozenset({"text", "image"}),
+ "openai/gpt-audio-mini": frozenset({"text", "audio"}),
}
@@ -45,8 +46,8 @@ class TestJudgeAttachmentsMarkers:
@pytest.mark.parametrize(
("media", "marker"),
[
- pytest.param(MediaBlock(modality="audio", base64_data="aGk="), "[audio]", id="not_an_image"),
- pytest.param(_pointer("a", mime="image%2Fsvg%2Bxml"), "[image]", id="unsupported_format"),
+ pytest.param(MediaBlock(modality="audio", base64_data="aGk="), "[audio, not attached]", id="no_file_type"),
+ pytest.param(_pointer("a", mime="image%2Fsvg%2Bxml"), "[image, not attached]", id="unsupported_format"),
pytest.param(
MediaBlock(modality="image", mime_type="image/png", url="https://example.com/a.png"),
"[image: https://example.com/a.png]",
@@ -54,12 +55,12 @@ class TestJudgeAttachmentsMarkers:
),
pytest.param(
parse_media_block({"file_data": {"mime_type": "image/png", "file_uri": "gs://bucket/a.png"}}),
- "[image]",
+ "[image, not attached]",
id="gemini_cloud_storage_uri",
),
],
)
- def test_images_it_cannot_attach_keep_an_unnumbered_marker(self, media: MediaBlock, marker: str):
+ def test_media_it_cannot_attach_gets_an_unnumbered_marker(self, media: MediaBlock, marker: str):
assert JudgeAttachments(team_id=7).marker(media) == marker
@@ -155,6 +156,30 @@ class TestJudgeAttachmentsContent:
AttachmentPart(mime_type="audio/wav", data=b"hi"),
id="audio_inline_on_gemini",
),
+ pytest.param(
+ "gemini",
+ "gemini-2.5-flash",
+ "audio%2Fogg%3B%20codecs%3Dopus",
+ "Audio 1:",
+ AttachmentPart(mime_type="audio/ogg", data=b"hi"),
+ id="ogg_audio_with_a_codec_on_gemini",
+ ),
+ pytest.param(
+ "openrouter",
+ "google/gemini-2.5-flash",
+ "audio%2Fflac",
+ "Audio 1:",
+ AttachmentPart(mime_type="audio/flac", data=b"hi"),
+ id="flac_audio_on_an_openrouter_gemini_model",
+ ),
+ pytest.param(
+ "openrouter",
+ "openai/gpt-audio-mini",
+ "audio%2Fflac",
+ "Not attached, this judge can't read their file type: Audio 1.",
+ None,
+ id="flac_audio_on_an_openrouter_model_outside_gemini",
+ ),
pytest.param(
"gemini",
"gemini-3-flash",
@@ -187,6 +212,14 @@ class TestJudgeAttachmentsContent:
AttachmentPart(mime_type="audio/mpeg", data=b"hi"),
id="audio_on_an_openai_audio_model",
),
+ pytest.param(
+ "openai",
+ "gpt-audio-mini",
+ "audio%2Fogg",
+ "Not attached, this judge can't read their file type: Audio 1.",
+ None,
+ id="ogg_audio_on_an_openai_audio_model",
+ ),
pytest.param(
"openai",
"gpt-5.5",
@@ -428,8 +461,19 @@ class TestJudgeAttachmentsContent:
assert content[1] == {"type": "text", "text": "Image 1:"}
- def test_skips_images_whose_marker_was_sampled_out_of_the_prompt(self):
+ @pytest.mark.parametrize(
+ ("prompt", "expected"),
+ [
+ pytest.param("user: hello", "user: hello", id="marker_sampled_out"),
+ pytest.param(
+ "user: [image, not attached] what is this?",
+ f"{_NOT_ATTACHED_NOTE}\n\nuser: [image, not attached] what is this?",
+ id="only_unattached_media",
+ ),
+ ],
+ )
+ def test_a_prompt_without_attachments_stays_text(self, prompt: str, expected: str):
images = JudgeAttachments(team_id=7)
images.marker(_pointer("a"))
- assert images.content("user: hello", "openai", "gpt-5.5") == "user: hello"
+ assert images.content(prompt, "openai", "gpt-5.5") == expected
diff --git a/posthog/temporal/ai_observability/test_message_utils.py b/posthog/temporal/ai_observability/test_message_utils.py
index c3e1804d8449..84e10968eeba 100644
--- a/posthog/temporal/ai_observability/test_message_utils.py
+++ b/posthog/temporal/ai_observability/test_message_utils.py
@@ -91,17 +91,17 @@ class TestExtractTextFromMessages:
"type": "image",
"source": {"type": "base64", "media_type": "image/png", "data": "[base64 image redacted]"},
},
- "[image]",
+ "[image, not attached]",
id="anthropic_redacted",
),
pytest.param(
{"type": "image_url", "image_url": {"url": "data:image/png;base64,[base64 image redacted]"}},
- "[image]",
+ "[image, not attached]",
id="openai_redacted_data_uri",
),
pytest.param(
{"type": "image_url", "image_url": {"url": "data:image/png;base64," * 3 + "aGk="}},
- "[image]",
+ "[image, not attached]",
id="nested_data_uri",
),
pytest.param({"inline_data": {"mime_type": "image/png", "data": "aGk="}}, "[image 1]", id="gemini_inline"),
@@ -133,8 +133,12 @@ class TestExtractTextFromMessages:
"[audio 1]",
id="openai_audio_offloaded",
),
- pytest.param({"type": "input_audio", "input_audio": {"data": "aGk="}}, "[audio]", id="openai_audio"),
- pytest.param({"type": "document", "source": {"type": "base64", "data": "aGk="}}, "[file]", id="file"),
+ pytest.param(
+ {"type": "input_audio", "input_audio": {"data": "aGk="}}, "[audio, not attached]", id="openai_audio"
+ ),
+ pytest.param(
+ {"type": "document", "source": {"type": "base64", "data": "aGk="}}, "[file, not attached]", id="file"
+ ),
pytest.param(
{"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "aGk="}},
"[file 1]",
@@ -151,7 +155,9 @@ class TestExtractTextFromMessages:
id="openai_responses_pdf",
),
pytest.param(
- {"type": "file", "file": {"file_id": "file-abc"}}, "[file]", id="openai_uploaded_file_without_bytes"
+ {"type": "file", "file": {"file_id": "file-abc"}},
+ "[file, not attached]",
+ id="openai_uploaded_file_without_bytes",
),
pytest.param(
{"type": "file", "mediaType": "image/png", "file": f"phaiblob://v1/sha256/{'a' * 64}?mime=image%2Fpng"},
@@ -188,7 +194,7 @@ class TestExtractTextFromMessages:
),
pytest.param(
{"type": "file", "mediaType": "image/png", "file": "raw files not supported"},
- "[image]",
+ "[image, not attached]",
id="vercel_sdk_placeholder",
),
],
diff --git a/products/ai_observability/backend/llm/attachments.py b/products/ai_observability/backend/llm/attachments.py
index 2e19222c2a84..4e2905abce3c 100644
--- a/products/ai_observability/backend/llm/attachments.py
+++ b/products/ai_observability/backend/llm/attachments.py
@@ -7,8 +7,16 @@ from posthog.dataclasses import frozen
IMAGE_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/webp", "image/gif"})
PDF_MIME_TYPE = "application/pdf"
-# The formats OpenAI's chat API takes as input audio, which Gemini reads too.
-AUDIO_FORMATS = {"audio/wav": "wav", "audio/mpeg": "mp3"}
+# The formats OpenAI's chat API takes as input audio.
+OPENAI_AUDIO_FORMATS = {"audio/wav": "wav", "audio/mpeg": "mp3"}
+# Gemini reads these as well. The values are the `format` names OpenRouter's input_audio block takes.
+AUDIO_FORMATS = {
+ **OPENAI_AUDIO_FORMATS,
+ "audio/aac": "aac",
+ "audio/aiff": "aiff",
+ "audio/flac": "flac",
+ "audio/ogg": "ogg",
+}
VIDEO_MIME_TYPES = frozenset({"video/mp4", "video/mpeg", "video/quicktime", "video/webm"})
# Providers whose image blocks take a link that they download themselves.
_URL_PROVIDERS = frozenset({"openai", "azure_openai", "anthropic", "gemini"})
@@ -55,7 +63,8 @@ def attachment_transport(provider: str, model: str, mime_type: str) -> Literal["
openai_audio_model = provider == "openai" and "audio" in model
if mime_type in AUDIO_FORMATS:
# OpenAI takes audio only as inline base64, and only on its audio models, so audio always goes inline.
- return "inline" if provider == "gemini" or openai_audio_model else None
+ reads_format = provider == "gemini" or (openai_audio_model and mime_type in OPENAI_AUDIO_FORMATS)
+ return "inline" if reads_format else None
if mime_type in VIDEO_MIME_TYPES:
if provider != "gemini":
return None
@@ -101,6 +110,9 @@ def _openrouter_transport(model: str, mime_type: str) -> Literal["url", "inline"
return "url" if modality == "image" else None
if modality not in modalities:
return None
+ # Only Gemini models are known to read audio beyond WAV and MP3. A rejection would drop every attachment.
+ if modality == "audio" and mime_type not in OPENAI_AUDIO_FORMATS and not model.startswith("google/"):
+ return None
# OpenRouter forwards a link only where the model's provider downloads it, and Gemini on AI Studio doesn't.
return "url" if modality == "image" else "inline"