Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): ingest Gemini Interactions modalities and render images (#119692)

Carlos Marchal3ebe0c90ada2
diff --git a/docs/onboarding/ai-observability/google.tsx b/docs/onboarding/ai-observability/google.tsx
index 8894c87eb43e..ef07209d2b5a 100644
--- a/docs/onboarding/ai-observability/google.tsx
+++ b/docs/onboarding/ai-observability/google.tsx
@@ -108,6 +108,17 @@ export const getGoogleSteps = (ctx: OnboardingComponentsContext): StepDefinition
`}
</Markdown>
+ <Markdown>
+ {dedent`
+ Audio output usage is captured separately from text output when Gemini reports the
+ modality breakdown. Dedicated standard Gemini Developer API audio pricing is available
+ for \`gemini-2.5-pro-preview-tts\` and \`gemini-3.1-flash-tts-preview\`. Other audio models,
+ including newer TTS and Live models, need their own catalog rates or custom pricing;
+ capturing their usage alone does not establish accurate audio pricing. Unlisted providers
+ use the catalog's default estimate, rather than a verified provider-specific rate.
+ `}
+ </Markdown>
+
<CodeBlock
blocks={[
{
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index f02dc835034e..616afb37858b 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -8678,6 +8678,14 @@ snapshots:
hash: v1.k794b7964.1cd199ebfc3ecf7f75c065b7f670778fed080997c6afb56bac02e49b49637c96.nFRmsF5CaYLyxKdaRS0UtS6ba8esGKqBv6MPu7HwOBo
scenes-app-ai-observability-evaluation-explanation--system-one-zero-probability--light:
hash: v1.k794b7964.f56325c23a185f032b8e7c331463345ea780e4e1b9a85f44a2f560869f06afa4.KcU1H2opmRuSoQcBHacqKya9WVvI5GATsFn_lgLD8rM
+ scenes-app-ai-observability-image-preview--failed-image--dark:
+ hash: v1.k794b7964.e4f4b0993222ae8f8a3f0e59e4c0528312f13a1e358df37e53d4331b4dacbae5.oDWA2N3BGPuGJvd9voAvMpM2rCR7tfrLjEwDqJT6MYk
+ scenes-app-ai-observability-image-preview--failed-image--light:
+ hash: v1.k794b7964.4eb3b049f06ae96184a668696fdadf51050faacc18876e799b3ab0451257ee4b.OeohSQhdo2gAaTqtBzxi8qndmw5hPx_I7FCT5jlQIH8
+ scenes-app-ai-observability-image-preview--loaded-image--dark:
+ hash: v1.k794b7964.f5616b3a5f8f3fce4e63331f5d758f96b0581c374a7b3b9a670910a8599dde49.e0crK2eCymbeMfVduQVtV_ziu9buu7z919QTrTFJ3S4
+ scenes-app-ai-observability-image-preview--loaded-image--light:
+ hash: v1.k794b7964.378003676808bae57233a4563aca4fc19931291b3c13909b063401167dcc0c91.aBhz7hKTJqq2T8g4AfYlRhGmXVsZvRtyyM5qqZVpr1s
scenes-app-ai-observability-instrumentation-checklist-card--collecting--dark:
hash: v1.k794b7964.6f9f209d5daaf8573020fb50615421d426055c8090d0ee73a2629487d30cbf29.g4iTGpNCX68yiOgdFhRMYO2xEGf0WhTu4ji2Rqijroc
scenes-app-ai-observability-instrumentation-checklist-card--collecting--light:
diff --git a/frontend/src/taxonomy/core-filter-definitions-by-group.json b/frontend/src/taxonomy/core-filter-definitions-by-group.json
index a7200b58c990..8627d2d725c0 100644
--- a/frontend/src/taxonomy/core-filter-definitions-by-group.json
+++ b/frontend/src/taxonomy/core-filter-definitions-by-group.json
@@ -155,6 +155,12 @@
"label": "AI cache read audio tokens (LLM)",
"type": "Numeric"
},
+ "$ai_cache_read_image_tokens": {
+ "description": "The number of image tokens read from the prompt cache.",
+ "examples": [16],
+ "label": "AI cache read image tokens (LLM)",
+ "type": "Numeric"
+ },
"$ai_cache_read_input_tokens": {
"description": "The number of tokens read from the cache for the input prompt.",
"examples": [23],
@@ -3984,6 +3990,12 @@
"label": "AI cache read audio tokens (LLM)",
"type": "Numeric"
},
+ "$ai_cache_read_image_tokens": {
+ "description": "The number of image tokens read from the prompt cache.",
+ "examples": [16],
+ "label": "AI cache read image tokens (LLM)",
+ "type": "Numeric"
+ },
"$ai_cache_read_input_tokens": {
"description": "The number of tokens read from the cache for the input prompt.",
"examples": [23],
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/cost-model-matching.test.ts b/nodejs/src/ingestion/pipelines/ai/costs/cost-model-matching.test.ts
index e298efe14d41..fef436075f34 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/cost-model-matching.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/cost-model-matching.test.ts
@@ -544,6 +544,14 @@ describe('requireSpecialCost()', () => {
expect(requireSpecialCost('gemini-2.5-pro-preview')).toBe(true)
})
+ it.each(['gemini-2.5-pro-preview-tts', 'google/gemini-2.5-pro-preview-tts', 'GEMINI-2.5-PRO-PREVIEW-TTS'])(
+ 'keeps %s out of the text-model context threshold rewrite',
+ (model) => {
+ expect(requireSpecialCost(model)).toBe(false)
+ expect(getNewModelName(model, 200001)).toBe(model)
+ }
+ )
+
it('returns true for gemini-2.5-pro-preview variants', () => {
expect(requireSpecialCost('gemini-2.5-pro-preview-0514')).toBe(true)
})
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/cost-model-matching.ts b/nodejs/src/ingestion/pipelines/ai/costs/cost-model-matching.ts
index 968e954642e6..a25637ede048 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/cost-model-matching.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/cost-model-matching.ts
@@ -202,12 +202,16 @@ const searchModelInCosts = (model: string, costsDict: Record<string, ModelCostRo
export const requireSpecialCost = (aiModel: string): boolean => {
const lowerAiModel = aiModel.toLowerCase()
- return SPECIAL_COST_MODELS.some((model) => lowerAiModel.includes(model.toLowerCase()))
+ // TTS is a separate audio-only model, not a context-size variant of Pro.
+ return (
+ !lowerAiModel.endsWith('-tts') &&
+ SPECIAL_COST_MODELS.some((model) => lowerAiModel.includes(model.toLowerCase()))
+ )
}
export function getNewModelName(model: string, inputTokens: unknown): string {
// Gemini 2.5 Pro Preview has a limit of 200k input tokens before the price changes, we store the other price in the :large suffix
- if (model.toLowerCase().includes('gemini-2.5-pro-preview')) {
+ if (requireSpecialCost(model)) {
const tokenCountExceeded = inputTokens ? Number(inputTokens) > 200000 : false
return tokenCountExceeded ? 'gemini-2.5-pro-preview:large' : 'gemini-2.5-pro-preview'
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/index.ts b/nodejs/src/ingestion/pipelines/ai/costs/index.ts
index 583d7811409d..c1bac24ecdbb 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/index.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/index.ts
@@ -74,9 +74,11 @@ const trackCostOutcome = (totalCost: number): void => {
}
// Every token count each cost calculator reads, split by the side it prices.
-// Presence is what matters, not value: `0` is a usage report that says the
+// For independently priced pools, presence matters: `0` is a usage report that says the
// model consumed nothing, while an absent property means the provider never
-// reported usage at all. The sides gate independently, because an interrupted
+// reported usage at all. Positive cached modality breakdowns need independently
+// priced parent pools; those counts are also in the input list.
+// The sides gate independently, because an interrupted
// stream often reports one side only — Anthropic sends input tokens on
// message_start and the output count in the final delta.
//
@@ -97,6 +99,7 @@ const INPUT_TOKEN_COUNT_PROPERTIES = [
'$ai_audio_input_tokens',
'$ai_image_input_tokens',
'$ai_cache_read_audio_tokens',
+ '$ai_cache_read_image_tokens',
] as const
const OUTPUT_TOKEN_COUNT_PROPERTIES = [
@@ -107,8 +110,13 @@ const OUTPUT_TOKEN_COUNT_PROPERTIES = [
'$ai_image_output_tokens',
] as const
-const hasAnyTokenCount = (properties: Properties, keys: readonly string[]): boolean => {
- return keys.some((key) => finiteNumberOrUndefined(properties[key]) !== undefined)
+const hasAnyPricedTokenCount = (properties: Properties, keys: readonly string[]): boolean => {
+ return keys.some((key) => {
+ const value = finiteNumberOrUndefined(properties[key])
+ // A cached subset alone has no priced pool; zero still reports known usage.
+ const isCachedModality = key === '$ai_cache_read_audio_tokens' || key === '$ai_cache_read_image_tokens'
+ return value !== undefined && (!isCachedModality || value === 0)
+ })
}
const setCostsOnEvent = (event: EventWithProperties, cost: ResolvedModelCost): void => {
@@ -125,8 +133,8 @@ const setCostsOnEvent = (event: EventWithProperties, cost: ResolvedModelCost): v
// output, so they have to be caught here.
const hasClientCost = PRECALCULATED_COST_PROPERTIES.some((key) => typeof event.properties[key] === 'number')
- const hasInputTokenCount = hasAnyTokenCount(event.properties, INPUT_TOKEN_COUNT_PROPERTIES)
- const hasOutputTokenCount = hasAnyTokenCount(event.properties, OUTPUT_TOKEN_COUNT_PROPERTIES)
+ const hasInputTokenCount = hasAnyPricedTokenCount(event.properties, INPUT_TOKEN_COUNT_PROPERTIES)
+ const hasOutputTokenCount = hasAnyPricedTokenCount(event.properties, OUTPUT_TOKEN_COUNT_PROPERTIES)
// A token rate multiplied by no usage is 0, which reads as "this call was
// free" rather than "we never learned what this call used". The two are
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/input-costs.test.ts b/nodejs/src/ingestion/pipelines/ai/costs/input-costs.test.ts
index d1fdaa7936e6..95ba9a7ae2b0 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/input-costs.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/input-costs.test.ts
@@ -1081,6 +1081,111 @@ describe('calculateInputCost()', () => {
expectCostToBeCloseTo(result, 0.0003)
})
+ const imageCacheModel: ResolvedModelCost = {
+ model: 'google/gemini-3.1-flash-lite',
+ provider: 'google',
+ cost: {
+ prompt_token: 2.5e-7,
+ completion_token: 0.0000015,
+ image: 2.5e-7,
+ cache_read_token: 2.5e-8,
+ },
+ }
+
+ it('bills fully cached Gemini image input only at the cache-read rate', () => {
+ const event = createAIEvent({
+ $ai_provider: 'google',
+ $ai_model: 'gemini-3.1-flash-lite',
+ $ai_input_tokens: 100,
+ $ai_image_input_tokens: 100,
+ $ai_cache_read_input_tokens: 100,
+ $ai_cache_read_image_tokens: 100,
+ })
+
+ const result = calculateInputCost(event, imageCacheModel)
+
+ expectCostToBeCloseTo(result, 100 * 2.5e-8, 12)
+ })
+
+ it('bills uncached Gemini text plus cached image without double-counting', () => {
+ const event = createAIEvent({
+ $ai_provider: 'google',
+ $ai_model: 'gemini-3.1-flash-lite',
+ $ai_input_tokens: 300,
+ $ai_text_input_tokens: 100,
+ $ai_image_input_tokens: 200,
+ $ai_cache_read_input_tokens: 200,
+ $ai_cache_read_image_tokens: 200,
+ })
+
+ const result = calculateInputCost(event, imageCacheModel)
+
+ expectCostToBeCloseTo(result, 100 * 2.5e-7 + 200 * 2.5e-8, 12)
+ })
+
+ it('does not subtract cached image from an exclusive uncached input count', () => {
+ const event = createAIEvent({
+ $ai_provider: 'google',
+ $ai_model: 'gemini-3.1-flash-lite',
+ $ai_cache_reporting_exclusive: true,
+ $ai_input_tokens: 150, // 50 text + 100 uncached image
+ $ai_image_input_tokens: 200, // 100 cached + 100 uncached
+ $ai_cache_read_input_tokens: 100,
+ $ai_cache_read_image_tokens: 100,
+ })
+
+ const result = calculateInputCost(event, imageCacheModel)
+
+ expectCostToBeCloseTo(result, 50 * 2.5e-7 + 100 * 2.5e-7 + 100 * 2.5e-8, 12)
+ })
+
+ it('keeps uncached image plus cached text at their existing rates', () => {
+ const event = createAIEvent({
+ $ai_provider: 'google',
+ $ai_model: 'gemini-3.1-flash-lite',
+ $ai_input_tokens: 300,
+ $ai_image_input_tokens: 100,
+ $ai_cache_read_input_tokens: 200,
+ })
+
+ const result = calculateInputCost(event, imageCacheModel)
+
+ expectCostToBeCloseTo(result, 100 * 2.5e-7 + 200 * 2.5e-8, 12)
+ })
+
+ it.each([
+ { cacheReadTokens: 200, cachedImageTokens: 100, uncachedImageTokens: 100 },
+ { cacheReadTokens: 150, cachedImageTokens: 50, uncachedImageTokens: 150 },
+ ])(
+ 'bills cached audio and image without exceeding a $cacheReadTokens-token cache pool',
+ ({ cacheReadTokens, cachedImageTokens, uncachedImageTokens }) => {
+ const event = createAIEvent({
+ $ai_provider: 'google',
+ $ai_model: 'gemini-3.1-flash-lite',
+ $ai_input_tokens: 500, // 100 text + 200 audio + 200 image
+ $ai_audio_input_tokens: 200,
+ $ai_image_input_tokens: 200,
+ $ai_cache_read_input_tokens: cacheReadTokens,
+ $ai_cache_read_audio_tokens: 100,
+ $ai_cache_read_image_tokens: 100,
+ })
+ const model: ResolvedModelCost = {
+ ...imageCacheModel,
+ cost: { ...imageCacheModel.cost, audio: 5e-7, input_audio_cache: 5e-8 },
+ }
+
+ const result = calculateInputCost(event, model)
+
+ const expected =
+ 100 * 2.5e-7 + // uncached text
+ 100 * 5e-7 + // uncached audio
+ 100 * 5e-8 + // cached audio
+ uncachedImageTokens * 2.5e-7 +
+ cachedImageTokens * 2.5e-8
+ expectCostToBeCloseTo(result, expected, 12)
+ }
+ )
+
it('clamps the text pool to zero when modality tokens exceed it', () => {
// 500 input total, 400 cache read, 200 audio — the residual text pool
// would be 500 - 400 - 200 = -100; clamp to 0 so we don't silently
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/input-costs.ts b/nodejs/src/ingestion/pipelines/ai/costs/input-costs.ts
index 723ab0b1f932..85b43a68e351 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/input-costs.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/input-costs.ts
@@ -171,22 +171,27 @@ export const calculateInputCost = (event: PluginEvent, cost: ResolvedModelCost):
// would under-bill here.
const rawCachedAudioTokens = numericProperty(event, '$ai_cache_read_audio_tokens')
const cachedAudioInputTokens = Math.max(0, Math.min(rawCachedAudioTokens, audioInputTokens, cacheReadTokens))
+ const cachedNonAudioTokens = cacheReadTokens - cachedAudioInputTokens
+ const rawCachedImageTokens = numericProperty(event, '$ai_cache_read_image_tokens')
+ const cachedImageInputTokens = Math.max(0, Math.min(rawCachedImageTokens, imageInputTokens, cachedNonAudioTokens))
// Audio/image input tokens are reported by providers (OpenAI, Gemini) as a subset
// of the total input token count. We bill them separately at modality rates and
// subtract them from the text pool to avoid double-counting at the prompt rate.
- // For audio, split further into cached / uncached so cached audio bills at the
- // (typically much cheaper) audio-cache rate.
+ // Split cached audio and images from their uncached portions. Cached audio
+ // has its own rate; cached images use the standard cache-read rate.
const uncachedAudioInputTokens = audioInputTokens - cachedAudioInputTokens
+ const uncachedImageInputTokens = imageInputTokens - cachedImageInputTokens
const audioInputCost = computeAudioInputCost(event, cost, uncachedAudioInputTokens)
const cachedAudioInputCost = computeCachedAudioInputCost(event, cost, cachedAudioInputTokens)
- const imageInputCost = computeImageInputCost(event, cost, imageInputTokens)
+ const imageInputCost = computeImageInputCost(event, cost, uncachedImageInputTokens)
const modalityInputCost = bigDecimal.add(bigDecimal.add(audioInputCost, cachedAudioInputCost), imageInputCost)
const hasModalityTokens = audioInputTokens > 0 || imageInputTokens > 0
- // Text-only portion of the cache pool. Subtracting cached_audio gives us
- // the cached-text count, which we bill at the standard cache_read rate.
- const cachedTextTokens = cacheReadTokens - cachedAudioInputTokens
+ // Cached image tokens use the standard cache-read rate, but remain part of
+ // image_input and must not also reduce the regular text-token pool.
+ const cachedTextTokens = cachedNonAudioTokens - cachedImageInputTokens
+ const imageTokensToSubtract = exclusive ? uncachedImageInputTokens : imageInputTokens
if (matchProvider(event, 'anthropic')) {
const aggregateCacheWriteTokens = numericProperty(event, '$ai_cache_creation_input_tokens')
@@ -214,15 +219,15 @@ export const calculateInputCost = (event: PluginEvent, cost: ResolvedModelCost):
const cacheReadCost =
cost.cost.cache_read_token !== undefined
- ? bigDecimal.multiply(cost.cost.cache_read_token, cachedTextTokens)
- : bigDecimal.multiply(bigDecimal.multiply(cost.cost.prompt_token, 0.1), cachedTextTokens)
+ ? bigDecimal.multiply(cost.cost.cache_read_token, cachedNonAudioTokens)
+ : bigDecimal.multiply(bigDecimal.multiply(cost.cost.prompt_token, 0.1), cachedNonAudioTokens)
const totalCacheCost = bigDecimal.add(writeCost, cacheReadCost)
const baseUncachedTokens = exclusive
? inputTokens
: bigDecimal.subtract(bigDecimal.subtract(inputTokens, cachedTextTokens), cacheWriteTokens)
const uncachedTextTokens = clampTextTokens(
- bigDecimal.subtract(bigDecimal.subtract(baseUncachedTokens, audioInputTokens), imageInputTokens),
+ bigDecimal.subtract(bigDecimal.subtract(baseUncachedTokens, audioInputTokens), imageTokensToSubtract),
hasModalityTokens
)
const uncachedCost = bigDecimal.multiply(cost.cost.prompt_token, uncachedTextTokens)
@@ -240,7 +245,7 @@ export const calculateInputCost = (event: PluginEvent, cost: ResolvedModelCost):
? bigDecimal.add(inputTokens, isGeminiCatalogCost ? cacheWriteTokens : 0)
: bigDecimal.subtract(bigDecimal.subtract(inputTokens, cachedTextTokens), separateCacheWriteTokens)
const regularTextTokens = clampTextTokens(
- bigDecimal.subtract(bigDecimal.subtract(baseRegularTokens, audioInputTokens), imageInputTokens),
+ bigDecimal.subtract(bigDecimal.subtract(baseRegularTokens, audioInputTokens), imageTokensToSubtract),
hasModalityTokens
)
@@ -248,12 +253,12 @@ export const calculateInputCost = (event: PluginEvent, cost: ResolvedModelCost):
if (cost.cost.cache_read_token !== undefined) {
// Use explicit cache read cost if available
- cacheReadCost = bigDecimal.multiply(cost.cost.cache_read_token, cachedTextTokens)
+ cacheReadCost = bigDecimal.multiply(cost.cost.cache_read_token, cachedNonAudioTokens)
} else {
// Use default multiplier of 0.5 for all providers when cache_read_token is not defined
const multiplier = 0.5
- if (cachedTextTokens > 0) {
+ if (cachedNonAudioTokens > 0) {
logger.warn('Using default cache read multiplier for model', {
multiplier,
model: cost.model,
@@ -261,7 +266,10 @@ export const calculateInputCost = (event: PluginEvent, cost: ResolvedModelCost):
})
}
- cacheReadCost = bigDecimal.multiply(bigDecimal.multiply(cost.cost.prompt_token, multiplier), cachedTextTokens)
+ cacheReadCost = bigDecimal.multiply(
+ bigDecimal.multiply(cost.cost.prompt_token, multiplier),
+ cachedNonAudioTokens
+ )
}
const cacheWriteRate = cost.cost.cache_write_token ?? cost.cost.prompt_token
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/modality-tokens.test.ts b/nodejs/src/ingestion/pipelines/ai/costs/modality-tokens.test.ts
index 3d6c627024f6..54de155768ba 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/modality-tokens.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/modality-tokens.test.ts
@@ -5,6 +5,90 @@ import { createAIEvent } from './test-helpers'
describe('extractModalityTokens()', () => {
describe('Gemini direct usage metadata', () => {
+ it('extracts Gemini Interactions input, output, and cached modality tokens', () => {
+ const event = createAIEvent({
+ $ai_usage: {
+ input_tokens_by_modality: [
+ { modality: 'text', tokens: 250 },
+ { modality: 'audio', tokens: 750 },
+ { modality: 'image', tokens: 400 },
+ ],
+ output_tokens_by_modality: [
+ { modality: 'text', tokens: 10 },
+ { modality: 'audio', tokens: 20 },
+ { modality: 'image', tokens: 1290 },
+ ],
+ cached_tokens_by_modality: [{ modality: 'audio', tokens: 50 }],
+ },
+ })
+
+ const result = extractModalityTokens(event)
+
+ expect(result.properties).toMatchObject({
+ $ai_text_input_tokens: 250,
+ $ai_audio_input_tokens: 750,
+ $ai_image_input_tokens: 400,
+ $ai_text_output_tokens: 10,
+ $ai_audio_output_tokens: 20,
+ $ai_image_output_tokens: 1290,
+ $ai_cache_read_audio_tokens: 50,
+ })
+ expect(result.properties['$ai_usage']).toBeUndefined()
+ })
+
+ it('extracts cached image tokens from Gemini Interactions usage', () => {
+ const event = createAIEvent({
+ $ai_usage: {
+ input_tokens_by_modality: [{ modality: 'image', tokens: 100 }],
+ cached_tokens_by_modality: [{ modality: 'image', tokens: 100 }],
+ },
+ })
+
+ const result = extractModalityTokens(event)
+
+ expect(result.properties).toMatchObject({
+ $ai_image_input_tokens: 100,
+ $ai_cache_read_image_tokens: 100,
+ })
+ expect(result.properties['$ai_usage']).toBeUndefined()
+ })
+
+ it.each([-1, 1.5, Number.MAX_SAFE_INTEGER + 1, Number.POSITIVE_INFINITY, Number.NaN])(
+ 'ignores invalid Gemini cached image count %s',
+ (tokens) => {
+ const event = createAIEvent({
+ $ai_usage: { cached_tokens_by_modality: [{ modality: 'image', tokens }] },
+ })
+
+ const result = extractModalityTokens(event)
+
+ expect(result.properties['$ai_cache_read_image_tokens']).toBeUndefined()
+ expect(result.properties['$ai_usage']).toBeUndefined()
+ }
+ )
+
+ it.each([
+ { tokens: -5, expected: undefined },
+ { tokens: 1.5, expected: undefined },
+ { tokens: Number.MAX_SAFE_INTEGER + 1, expected: undefined },
+ { tokens: Number.POSITIVE_INFINITY, expected: undefined },
+ { tokens: Number.NaN, expected: undefined },
+ { tokens: 0, expected: 0 },
+ ])('accepts only nonnegative safe integer Interactions token counts ($tokens)', ({ tokens, expected }) => {
+ const event = createAIEvent({
+ $ai_usage: {
+ input_tokens_by_modality: [{ modality: 'text', tokens }],
+ output_tokens_by_modality: [{ modality: 'text', tokens }],
+ },
+ })
+
+ const result = extractModalityTokens(event)
+
+ expect(result.properties['$ai_text_input_tokens']).toBe(expected)
+ expect(result.properties['$ai_text_output_tokens']).toBe(expected)
+ expect(result.properties['$ai_usage']).toBeUndefined()
+ })
+
it('extracts image and text tokens from candidatesTokensDetails array format', () => {
const event = createAIEvent({
$ai_usage: {
@@ -24,7 +108,7 @@ describe('extractModalityTokens()', () => {
expect(result.properties['$ai_usage']).toBeUndefined()
})
- it('extracts image and text tokens from candidatesTokensDetails object format (fallback)', () => {
+ it('extracts audio, image and text tokens from candidatesTokensDetails object format (fallback)', () => {
const event = createAIEvent({
$ai_usage: {
promptTokenCount: 100,
@@ -32,6 +116,7 @@ describe('extractModalityTokens()', () => {
candidatesTokensDetails: {
textTokens: 10,
imageTokens: 1290,
+ audioTokens: 800,
},
},
})
@@ -40,6 +125,30 @@ describe('extractModalityTokens()', () => {
expect(result.properties['$ai_image_output_tokens']).toBe(1290)
expect(result.properties['$ai_text_output_tokens']).toBe(10)
+ expect(result.properties['$ai_audio_output_tokens']).toBe(800)
+ expect(result.properties['$ai_usage']).toBeUndefined()
+ })
+
+ it.each([
+ { tokens: -5, expected: undefined },
+ { tokens: 1.5, expected: undefined },
+ { tokens: Number.MAX_SAFE_INTEGER + 1, expected: undefined },
+ { tokens: Number.POSITIVE_INFINITY, expected: undefined },
+ { tokens: Number.NaN, expected: undefined },
+ { tokens: 0, expected: 0 },
+ ])('accepts only nonnegative safe integer object-form tokens ($tokens)', ({ tokens, expected }) => {
+ const event = createAIEvent({
+ $ai_usage: {
+ promptTokensDetails: { textTokens: tokens },
+ candidatesTokensDetails: { textTokens: tokens, audioTokens: tokens },
+ },
+ })
+
+ const result = extractModalityTokens(event)
+
+ expect(result.properties['$ai_text_input_tokens']).toBe(expected)
+ expect(result.properties['$ai_text_output_tokens']).toBe(expected)
+ expect(result.properties['$ai_audio_output_tokens']).toBe(expected === 0 ? undefined : expected)
expect(result.properties['$ai_usage']).toBeUndefined()
})
@@ -62,7 +171,7 @@ describe('extractModalityTokens()', () => {
expect(result.properties['$ai_usage']).toBeUndefined()
})
- it('extracts image and text tokens from outputTokenDetails object format (fallback)', () => {
+ it('extracts audio, image and text tokens from outputTokenDetails object format (fallback)', () => {
const event = createAIEvent({
$ai_usage: {
promptTokenCount: 100,
@@ -70,6 +179,7 @@ describe('extractModalityTokens()', () => {
outputTokenDetails: {
textTokens: 10,
imageTokens: 1290,
+ audioTokens: 800,
},
},
})
@@ -78,6 +188,7 @@ describe('extractModalityTokens()', () => {
expect(result.properties['$ai_image_output_tokens']).toBe(1290)
expect(result.properties['$ai_text_output_tokens']).toBe(10)
+ expect(result.properties['$ai_audio_output_tokens']).toBe(800)
expect(result.properties['$ai_usage']).toBeUndefined()
})
@@ -199,6 +310,7 @@ describe('extractModalityTokens()', () => {
candidatesTokensDetails: {
textTokens: 10,
imageTokens: 1290,
+ audioTokens: 800,
},
},
},
@@ -209,6 +321,7 @@ describe('extractModalityTokens()', () => {
expect(result.properties['$ai_image_output_tokens']).toBe(1290)
expect(result.properties['$ai_text_output_tokens']).toBe(10)
+ expect(result.properties['$ai_audio_output_tokens']).toBe(800)
expect(result.properties['$ai_usage']).toBeUndefined()
})
@@ -249,6 +362,7 @@ describe('extractModalityTokens()', () => {
outputTokenDetails: {
textTokens: 10,
imageTokens: 1290,
+ audioTokens: 800,
},
},
},
@@ -259,6 +373,7 @@ describe('extractModalityTokens()', () => {
expect(result.properties['$ai_image_output_tokens']).toBe(1290)
expect(result.properties['$ai_text_output_tokens']).toBe(10)
+ expect(result.properties['$ai_audio_output_tokens']).toBe(800)
})
it('handles non-google provider metadata', () => {
@@ -573,6 +688,19 @@ describe('extractModalityTokens()', () => {
expect(result.properties['$ai_cache_read_audio_tokens']).toBe(50)
})
+ it('extracts cached image from Gemini cacheTokensDetails object format', () => {
+ const event = createAIEvent({
+ $ai_usage: {
+ cacheTokensDetails: { imageTokens: 100 },
+ },
+ })
+
+ const result = extractModalityTokens(event)
+
+ expect(result.properties['$ai_cache_read_image_tokens']).toBe(100)
+ expect(result.properties['$ai_usage']).toBeUndefined()
+ })
+
it('does not set $ai_cache_read_audio_tokens when cacheTokensDetails has no audio entry', () => {
const event = createAIEvent({
$ai_usage: {
@@ -794,6 +922,27 @@ describe('extractModalityTokens()', () => {
expect(result.properties['$ai_cache_read_audio_tokens']).toBe(50)
})
+ it.each([1.5, Number.MAX_SAFE_INTEGER + 1, Number.POSITIVE_INFINITY])(
+ 'ignores invalid OpenAI audio token counts (%s)',
+ (audioTokens) => {
+ const event = createAIEvent({
+ $ai_usage: {
+ prompt_tokens_details: {
+ audio_tokens: audioTokens,
+ cached_tokens_details: { audio_tokens: audioTokens },
+ },
+ },
+ })
+
+ const result = extractModalityTokens(event)
+
+ expect({
+ input: result.properties['$ai_audio_input_tokens'],
+ cached: result.properties['$ai_cache_read_audio_tokens'],
+ }).toEqual({ input: undefined, cached: undefined })
+ }
+ )
+
it('does not set audio input when OpenAI prompt audio_tokens is zero', () => {
const event = createAIEvent({
$ai_usage: {
@@ -902,18 +1051,21 @@ describe('extractModalityTokens()', () => {
expect(labelsSpy).toHaveBeenCalledWith({ status: 'no_details', source: 'none' })
})
- it('emits source=gemini_output for output-only extractions', () => {
- extractModalityTokens(
- createAIEvent({
- $ai_usage: {
- candidatesTokensDetails: [{ modality: 'IMAGE', tokenCount: 1290 }],
- },
- })
- )
+ it.each([[{ modality: 'IMAGE', tokenCount: 1290 }], { audioTokens: 800 }])(
+ 'emits source=gemini_output for output-only extractions (%j)',
+ (candidatesTokensDetails) => {
+ extractModalityTokens(
+ createAIEvent({
+ $ai_usage: {
+ candidatesTokensDetails,
+ },
+ })
+ )
- expect(labelsSpy).toHaveBeenCalledTimes(1)
- expect(labelsSpy).toHaveBeenCalledWith({ status: 'extracted', source: 'gemini_output' })
- })
+ expect(labelsSpy).toHaveBeenCalledTimes(1)
+ expect(labelsSpy).toHaveBeenCalledWith({ status: 'extracted', source: 'gemini_output' })
+ }
+ )
it('emits source=gemini_cache for cache-only Gemini extractions', () => {
extractModalityTokens(
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/modality-tokens.ts b/nodejs/src/ingestion/pipelines/ai/costs/modality-tokens.ts
index d78a023d1754..e02d765d72ed 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/modality-tokens.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/modality-tokens.ts
@@ -25,9 +25,13 @@ const pick = (metadata: Record<string, unknown>, ...keys: string[]): unknown =>
return undefined
}
+const isValidTokenCount = (value: unknown): value is number => {
+ return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
+}
+
const tokenCountOf = (detail: Record<string, unknown>): number | null => {
- const tokenCount = detail['tokenCount'] ?? detail['token_count']
- return typeof tokenCount === 'number' ? tokenCount : null
+ const tokenCount = detail['tokenCount'] ?? detail['token_count'] ?? detail['tokens']
+ return isValidTokenCount(tokenCount) ? tokenCount : null
}
const modalityOf = (detail: Record<string, unknown>): string | null => {
@@ -94,15 +98,15 @@ export const extractModalityTokens = (event: EventWithProperties): EventWithProp
}
}
} else if (isObject(tokenDetails)) {
- if (typeof tokenDetails['audioTokens'] === 'number' && tokenDetails['audioTokens'] > 0) {
+ if (isValidTokenCount(tokenDetails['audioTokens']) && tokenDetails['audioTokens'] > 0) {
event.properties['$ai_audio_input_tokens'] = tokenDetails['audioTokens']
extractedSources.add('gemini_input')
}
- if (typeof tokenDetails['imageTokens'] === 'number' && tokenDetails['imageTokens'] > 0) {
+ if (isValidTokenCount(tokenDetails['imageTokens']) && tokenDetails['imageTokens'] > 0) {
event.properties['$ai_image_input_tokens'] = tokenDetails['imageTokens']
extractedSources.add('gemini_input')
}
- if (typeof tokenDetails['textTokens'] === 'number') {
+ if (isValidTokenCount(tokenDetails['textTokens'])) {
event.properties['$ai_text_input_tokens'] = tokenDetails['textTokens']
extractedSources.add('gemini_input')
}
@@ -129,17 +133,25 @@ export const extractModalityTokens = (event: EventWithProperties): EventWithProp
event.properties['$ai_image_output_tokens'] = tokenCount
extractedSources.add('gemini_output')
}
+ if (modality === 'audio' && tokenCount > 0) {
+ event.properties['$ai_audio_output_tokens'] = tokenCount
+ extractedSources.add('gemini_output')
+ }
if (modality === 'text') {
event.properties['$ai_text_output_tokens'] = tokenCount
extractedSources.add('gemini_output')
}
}
} else if (isObject(tokenDetails)) {
- if (typeof tokenDetails['imageTokens'] === 'number' && tokenDetails['imageTokens'] > 0) {
+ if (isValidTokenCount(tokenDetails['audioTokens']) && tokenDetails['audioTokens'] > 0) {
+ event.properties['$ai_audio_output_tokens'] = tokenDetails['audioTokens']
+ extractedSources.add('gemini_output')
+ }
+ if (isValidTokenCount(tokenDetails['imageTokens']) && tokenDetails['imageTokens'] > 0) {
event.properties['$ai_image_output_tokens'] = tokenDetails['imageTokens']
extractedSources.add('gemini_output')
}
- if (typeof tokenDetails['textTokens'] === 'number') {
+ if (isValidTokenCount(tokenDetails['textTokens'])) {
event.properties['$ai_text_output_tokens'] = tokenDetails['textTokens']
extractedSources.add('gemini_output')
}
@@ -166,14 +178,20 @@ export const extractModalityTokens = (event: EventWithProperties): EventWithProp
event.properties['$ai_cache_read_audio_tokens'] = tokenCount
extractedSources.add('gemini_cache')
}
+ if (modality === 'image' && tokenCount > 0) {
+ event.properties['$ai_cache_read_image_tokens'] = tokenCount
+ extractedSources.add('gemini_cache')
+ }
+ }
+ } else if (isObject(tokenDetails)) {
+ if (isValidTokenCount(tokenDetails['audioTokens']) && tokenDetails['audioTokens'] > 0) {
+ event.properties['$ai_cache_read_audio_tokens'] = tokenDetails['audioTokens']
+ extractedSources.add('gemini_cache')
+ }
+ if (isValidTokenCount(tokenDetails['imageTokens']) && tokenDetails['imageTokens'] > 0) {
+ event.properties['$ai_cache_read_image_tokens'] = tokenDetails['imageTokens']
+ extractedSources.add('gemini_cache')
}
- } else if (
- isObject(tokenDetails) &&
- typeof tokenDetails['audioTokens'] === 'number' &&
- tokenDetails['audioTokens'] > 0
- ) {
- event.properties['$ai_cache_read_audio_tokens'] = tokenDetails['audioTokens']
- extractedSources.add('gemini_cache')
}
}
@@ -186,7 +204,7 @@ export const extractModalityTokens = (event: EventWithProperties): EventWithProp
return
}
const audioTokens = promptDetails['audio_tokens']
- if (typeof audioTokens === 'number' && audioTokens > 0) {
+ if (isValidTokenCount(audioTokens) && audioTokens > 0) {
event.properties['$ai_audio_input_tokens'] = audioTokens
extractedSources.add('openai_input')
}
@@ -202,7 +220,7 @@ export const extractModalityTokens = (event: EventWithProperties): EventWithProp
return
}
const audioTokens = cachedDetails['audio_tokens']
- if (typeof audioTokens === 'number' && audioTokens > 0) {
+ if (isValidTokenCount(audioTokens) && audioTokens > 0) {
event.properties['$ai_cache_read_audio_tokens'] = audioTokens
extractedSources.add('openai_cache')
}
@@ -278,11 +296,21 @@ export const extractModalityTokens = (event: EventWithProperties): EventWithProp
return
}
extractAnthropicCacheCreation(metadata)
- extractInputModality(pick(metadata, 'promptTokensDetails', 'prompt_tokens_details'))
+ extractInputModality(
+ pick(metadata, 'promptTokensDetails', 'prompt_tokens_details', 'input_tokens_by_modality')
+ )
extractOutputModality(
- pick(metadata, 'candidatesTokensDetails', 'candidates_tokens_details', 'outputTokenDetails')
+ pick(
+ metadata,
+ 'candidatesTokensDetails',
+ 'candidates_tokens_details',
+ 'outputTokenDetails',
+ 'output_tokens_by_modality'
+ )
+ )
+ extractCacheModality(
+ pick(metadata, 'cacheTokensDetails', 'cache_tokens_details', 'cached_tokens_by_modality')
)
- extractCacheModality(pick(metadata, 'cacheTokensDetails', 'cache_tokens_details'))
extractOpenAIInputModality(metadata)
extractOpenAICacheModality(metadata)
}
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
index f2824fe114c2..ef39671d966e 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.test.ts
@@ -1,3 +1,7 @@
+import { findCostFromModel } from '~/ingestion/pipelines/ai/costs/cost-model-matching'
+import { processCost } from '~/ingestion/pipelines/ai/costs/index'
+import { createAIEvent } from '~/ingestion/pipelines/ai/costs/test-helpers'
+
import { manualCosts } from './manual-providers'
describe('manualCosts', () => {
@@ -82,4 +86,63 @@ describe('manualCosts', () => {
expect(modelEntry?.cost.default.cache_read_token).toBe(expected.cache_read_token)
}
})
+
+ // These TTS models accept text input and emit audio output. They must resolve
+ // their own price rather than fuzzy-matching the ordinary Gemini text model.
+ describe('Gemini TTS pricing through ingestion', () => {
+ it.each([
+ ['gemini-2.5-pro-preview-tts', 0.000001, 0.00002],
+ ['gemini-3.1-flash-tts-preview', 0.000001, 0.00002],
+ ])('prices %s using its dedicated published audio rate', (model, inputRate, audioRate) => {
+ for (const provider of ['google', 'gemini', 'google-ai-studio']) {
+ for (const usage of [
+ { candidatesTokensDetails: [{ modality: 'AUDIO', tokenCount: 800 }] },
+ { output_tokens_by_modality: [{ modality: 'audio', tokens: 800 }] },
+ ]) {
+ const event = createAIEvent({
+ $ai_provider: provider,
+ $ai_model: model,
+ $ai_input_tokens: 100,
+ $ai_output_tokens: 800,
+ $ai_usage: usage,
+ })
+ const resolved = findCostFromModel(model, event.properties)
+ expect(resolved?.cost.model).toBe(model)
+ expect(resolved?.cost.cost.audio_output).toBe(audioRate)
+
+ const result = processCost(event)
+
+ expect(result.properties.$ai_audio_output_tokens).toBe(800)
+ expect(result.properties.$ai_input_cost_usd).toBeCloseTo(100 * inputRate, 8)
+ expect(result.properties.$ai_output_cost_usd).toBeCloseTo(800 * audioRate, 8)
+ expect(result.properties.$ai_total_cost_usd).toBeCloseTo(100 * inputRate + 800 * audioRate, 8)
+ expect(result.properties.$ai_usage).toBeUndefined()
+ }
+ }
+ })
+
+ it.each(['gemini-2.5-pro-preview-tts', 'google/gemini-3.1-flash-tts-preview'])(
+ 'prices audio-only %s even without a modality breakdown',
+ (model) => {
+ const result = processCost(createAIEvent({ $ai_model: model, $ai_output_tokens: 800 }))
+
+ expect(result.properties.$ai_output_cost_usd).toBeCloseTo(0.016, 8)
+ expect(result.properties.$ai_input_cost_usd).toBeUndefined()
+ expect(result.properties.$ai_cost_model_provider).toBe('default')
+ }
+ )
+
+ it('labels an unlisted provider as fallback pricing rather than verified Vertex pricing', () => {
+ const result = processCost(
+ createAIEvent({
+ $ai_provider: 'vertex',
+ $ai_model: 'gemini-3.1-flash-tts-preview',
+ $ai_output_tokens: 800,
+ })
+ )
+
+ expect(result.properties.$ai_cost_model_provider).toBe('default')
+ expect(result.properties.$ai_output_cost_usd).toBeCloseTo(0.016, 8)
+ })
+ })
})
diff --git a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
index 0781a502ea6c..a1f4faa5b3d5 100644
--- a/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
+++ b/nodejs/src/ingestion/pipelines/ai/costs/providers/manual-providers.ts
@@ -30,6 +30,18 @@ const manualProviderCosts: ModelCostRow[] = [
},
},
},
+ // Gemini TTS models accept text-only input and emit audio-only output.
+ // Standard Gemini Developer API rates, USD per token:
+ // https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview-tts
+ // https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-tts-preview
+ // The completion rate also prices audio when a client omits the modality breakdown.
+ ...['gemini-2.5-pro-preview-tts', 'gemini-3.1-flash-tts-preview'].map((model) => ({
+ model,
+ cost: {
+ default: { prompt_token: 0.000001, completion_token: 0.00002, audio_output: 0.00002 },
+ 'google-ai-studio': { prompt_token: 0.000001, completion_token: 0.00002, audio_output: 0.00002 },
+ },
+ })),
// Other
{
model: 'deepseek-v3-fireworks',
diff --git a/nodejs/src/ingestion/pipelines/ai/metrics.ts b/nodejs/src/ingestion/pipelines/ai/metrics.ts
index 492b14e2cb54..cecf2baa7180 100644
--- a/nodejs/src/ingestion/pipelines/ai/metrics.ts
+++ b/nodejs/src/ingestion/pipelines/ai/metrics.ts
@@ -20,7 +20,7 @@ export const aiCostModalityExtractionCounter = new Counter({
export const aiCostTotalOutcomeCounter = new Counter({
name: 'llma_ai_cost_outcome_total',
- // "unknown" means nothing priced the event — no token counts, no request or
+ // "unknown" means nothing priced the event — no priceable token counts, no request or
// web search charge, and no cost the client sent — so no cost was recorded.
// "zero" is a usage report of nothing, which is a different fact. "error"
// means the computed total came out NaN, which takes an Infinity-sized cost
diff --git a/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts b/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
index 45f1e31ebd24..9c0a8b8c41b1 100644
--- a/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
+++ b/nodejs/src/ingestion/pipelines/ai/process-ai-event.test.ts
@@ -3,6 +3,7 @@ import { PluginEvent } from '~/plugin-scaffold'
import { CostModelSource } from './costs/cost-model-matching'
import type { ModelCost, ModelCostRow } from './costs/providers/types'
+import { aiCostTotalOutcomeCounter } from './metrics'
import { normalizeTraceProperties, processAiEvent } from './process-ai-event'
jest.mock('~/common/utils/logger', () => ({
@@ -106,6 +107,15 @@ const costsByModel: Record<string, MockModelRow> = {
image_output: 0.00003,
},
},
+ 'google/gemini-3.1-flash-lite': {
+ model: 'google/gemini-3.1-flash-lite',
+ cost: {
+ prompt_token: 2.5e-7,
+ completion_token: 0.0000015,
+ image: 2.5e-7,
+ cache_read_token: 2.5e-8,
+ },
+ },
'google/gemini-3-flash-preview': {
model: 'google/gemini-3-flash-preview',
cost: {
@@ -139,6 +149,7 @@ jest.mock('./costs/providers', () => {
'openai/gpt-4o-audio-preview': 'openai',
'openai/gpt-audio-mini': 'openai',
'google/gemini-2.5-flash-image': 'google-ai-studio',
+ 'google/gemini-3.1-flash-lite': 'google-ai-studio',
'google/gemini-3-flash-preview': 'google-ai-studio',
}
@@ -489,6 +500,120 @@ describe('processAiEvent()', () => {
expect(result.properties!.$ai_output_cost_usd).toBeCloseTo(0.0003, 7)
expect(result.properties!.$ai_total_cost_usd).toBeCloseTo(0.00155, 7)
})
+
+ it('bills Gemini Interactions cached audio at the audio cache rate', () => {
+ event.properties!.$ai_model = 'gemini-3-flash-preview'
+ event.properties!.$ai_provider = 'gemini'
+ event.properties!.$ai_input_tokens = 1000
+ event.properties!.$ai_output_tokens = 100
+ event.properties!.$ai_cache_read_input_tokens = 50
+ event.properties!.$ai_usage = {
+ input_tokens_by_modality: [
+ { modality: 'audio', tokens: 750 },
+ { modality: 'text', tokens: 250 },
+ ],
+ output_tokens_by_modality: [{ modality: 'text', tokens: 100 }],
+ cached_tokens_by_modality: [{ modality: 'audio', tokens: 50 }],
+ }
+
+ const result = processAiEvent(event)
+
+ expect(result.properties).toMatchObject({
+ $ai_audio_input_tokens: 750,
+ $ai_text_input_tokens: 250,
+ $ai_text_output_tokens: 100,
+ $ai_cache_read_audio_tokens: 50,
+ })
+ expect(result.properties!.$ai_usage).toBeUndefined()
+ expect(result.properties!.$ai_input_cost_usd).toBeCloseTo(0.0011825, 8)
+ expect(result.properties!.$ai_output_cost_usd).toBeCloseTo(0.0003, 8)
+ expect(result.properties!.$ai_total_cost_usd).toBeCloseTo(0.0014825, 8)
+ })
+
+ it('bills Gemini Interactions image output at the image rate', () => {
+ event.properties!.$ai_model = 'gemini-2.5-flash-image'
+ event.properties!.$ai_provider = 'google'
+ event.properties!.$ai_input_tokens = 600
+ event.properties!.$ai_output_tokens = 1300
+ event.properties!.$ai_usage = {
+ input_tokens_by_modality: [
+ { modality: 'text', tokens: 200 },
+ { modality: 'image', tokens: 400 },
+ ],
+ output_tokens_by_modality: [
+ { modality: 'text', tokens: 10 },
+ { modality: 'image', tokens: 1290 },
+ ],
+ }
+
+ const result = processAiEvent(event)
+
+ expect(result.properties).toMatchObject({
+ $ai_text_input_tokens: 200,
+ $ai_image_input_tokens: 400,
+ $ai_text_output_tokens: 10,
+ $ai_image_output_tokens: 1290,
+ })
+ expect(result.properties!.$ai_usage).toBeUndefined()
+ expect(result.properties!.$ai_input_cost_usd).toBeCloseTo(0.00018, 7)
+ expect(result.properties!.$ai_output_cost_usd).toBeCloseTo(0.038725, 7)
+ expect(result.properties!.$ai_total_cost_usd).toBeCloseTo(0.038905, 7)
+ })
+
+ it('bills Gemini Interactions cached image input once at the cache-read rate', () => {
+ event.properties!.$ai_model = 'gemini-3.1-flash-lite'
+ event.properties!.$ai_provider = 'google'
+ event.properties!.$ai_input_tokens = 100
+ event.properties!.$ai_output_tokens = 0
+ event.properties!.$ai_cache_read_input_tokens = 100
+ event.properties!.$ai_usage = {
+ input_tokens_by_modality: [{ modality: 'image', tokens: 100 }],
+ cached_tokens_by_modality: [{ modality: 'image', tokens: 100 }],
+ }
+
+ const result = processAiEvent(event)
+
+ expect(result.properties).toMatchObject({
+ $ai_image_input_tokens: 100,
+ $ai_cache_read_image_tokens: 100,
+ })
+ expect(result.properties!.$ai_usage).toBeUndefined()
+ expect(result.properties!.$ai_input_cost_usd).toBeCloseTo(0.0000025, 12)
+ expect(result.properties!.$ai_total_cost_usd).toBeCloseTo(0.0000025, 12)
+ })
+
+ it('ignores invalid object-form Gemini text tokens before costing image output', () => {
+ event.properties!.$ai_model = 'gemini-2.5-flash-image'
+ event.properties!.$ai_provider = 'google'
+ event.properties!.$ai_input_tokens = 600
+ event.properties!.$ai_output_tokens = 1300
+ event.properties!.$ai_usage = {
+ candidatesTokensDetails: { textTokens: -5, imageTokens: 1290 },
+ }
+
+ const result = processAiEvent(event)
+
+ expect(result.properties!.$ai_text_output_tokens).toBeUndefined()
+ expect(result.properties!.$ai_image_output_tokens).toBe(1290)
+ expect(result.properties!.$ai_output_cost_usd).toBeCloseTo(0.038725, 6)
+ })
+
+ it('does not bill a negative Interactions text token count', () => {
+ event.properties!.$ai_model = 'gemini-3-flash-preview'
+ event.properties!.$ai_provider = 'gemini'
+ event.properties!.$ai_input_tokens = 0
+ event.properties!.$ai_output_tokens = 10
+ event.properties!.$ai_usage = {
+ output_tokens_by_modality: [{ modality: 'text', tokens: -5 }],
+ }
+
+ const result = processAiEvent(event)
+
+ expect(result.properties!.$ai_text_output_tokens).toBeUndefined()
+ expect(result.properties!.$ai_output_cost_usd).toBeCloseTo(0.00003, 8)
+ expect(result.properties!.$ai_total_cost_usd).toBeCloseTo(0.00003, 8)
+ expect(result.properties!.$ai_usage).toBeUndefined()
+ })
})
describe('cost calculation', () => {
@@ -532,6 +657,85 @@ describe('processAiEvent()', () => {
}
)
+ it.each(['image', 'audio'] as const)(
+ 'keeps orphan cached %s usage unknown without inventing parent counts',
+ async (modality) => {
+ aiCostTotalOutcomeCounter.reset()
+ delete event.properties!.$ai_input_tokens
+ delete event.properties!.$ai_output_tokens
+ event.properties![`$ai_cache_read_${modality}_tokens`] = '100'
+
+ const result = processAiEvent(event)
+
+ expect(result.properties!.$ai_input_cost_usd).toBeUndefined()
+ expect(result.properties!.$ai_total_cost_usd).toBeUndefined()
+ expect(result.properties!.$ai_output_cost_usd).toBeUndefined()
+ expect(result.properties![`$ai_${modality}_input_tokens`]).toBeUndefined()
+ expect(result.properties!.$ai_cache_read_input_tokens).toBeUndefined()
+ expect((await aiCostTotalOutcomeCounter.get()).values).toEqual([
+ { labels: { outcome: 'unknown' }, value: 1 },
+ ])
+ }
+ )
+
+ it.each(['image', 'audio'] as const)('preserves explicit zero cached %s usage', async (modality) => {
+ aiCostTotalOutcomeCounter.reset()
+ delete event.properties!.$ai_input_tokens
+ delete event.properties!.$ai_output_tokens
+ event.properties![`$ai_cache_read_${modality}_tokens`] = '0'
+
+ const result = processAiEvent(event)
+
+ expect(result.properties!.$ai_input_cost_usd).toBe(0)
+ expect(result.properties!.$ai_total_cost_usd).toBe(0)
+ expect(result.properties!.$ai_output_cost_usd).toBeUndefined()
+ expect((await aiCostTotalOutcomeCounter.get()).values).toEqual([{ labels: { outcome: 'zero' }, value: 1 }])
+ })
+
+ it.each(['image', 'audio'] as const)(
+ 'ignores unusable parent counts when cached %s usage is the only valid report',
+ (modality) => {
+ event.properties!.$ai_input_tokens = 'invalid'
+ event.properties![`$ai_${modality}_input_tokens`] = NaN
+ event.properties!.$ai_cache_read_input_tokens = Infinity
+ event.properties![`$ai_cache_read_${modality}_tokens`] = 100
+ delete event.properties!.$ai_output_tokens
+
+ const result = processAiEvent(event)
+
+ expect(result.properties!.$ai_input_cost_usd).toBeUndefined()
+ expect(result.properties!.$ai_total_cost_usd).toBeUndefined()
+ }
+ )
+
+ it.each([
+ { properties: { $ai_input_tokens: 100 }, costProperty: '$ai_input_cost_usd', expected: 20 },
+ { properties: { $ai_output_tokens: 50 }, costProperty: '$ai_output_cost_usd', expected: 10 },
+ { properties: { $ai_input_cost_usd: 3 }, costProperty: '$ai_input_cost_usd', expected: 3 },
+ {
+ properties: { $ai_model: 'model-with-request-only', $ai_provider: 'custom' },
+ costProperty: '$ai_request_cost_usd',
+ expected: 0.002,
+ },
+ {
+ properties: { $ai_model: 'perplexity/sonar-pro', $ai_provider: 'perplexity', $ai_web_search_count: 3 },
+ costProperty: '$ai_web_search_cost_usd',
+ expected: 0.015,
+ },
+ ])('preserves known $costProperty alongside orphan cached images', ({ properties, costProperty, expected }) => {
+ delete event.properties!.$ai_input_tokens
+ delete event.properties!.$ai_output_tokens
+ Object.assign(event.properties!, properties, { $ai_cache_read_image_tokens: 100 })
+
+ const result = processAiEvent(event)
+
+ expect(result.properties![costProperty]).toBeCloseTo(expected)
+ expect(result.properties!.$ai_total_cost_usd).toBeGreaterThan(0)
+ if (costProperty !== '$ai_input_cost_usd') {
+ expect(result.properties!.$ai_input_cost_usd).toBeUndefined()
+ }
+ })
+
// Each side is a rate times its own counts, so one reported side must
// not fabricate a $0 for the other. Interrupted streams commonly report
// input only: Anthropic sends input tokens on message_start and the
diff --git a/packages/llm-normalizer/src/types.ts b/packages/llm-normalizer/src/types.ts
index b27f009a2159..1c373639715d 100644
--- a/packages/llm-normalizer/src/types.ts
+++ b/packages/llm-normalizer/src/types.ts
@@ -95,6 +95,10 @@ export interface GeminiAudioMessage {
export interface GeminiImageMessage {
type: 'image'
+ // Gemini Interactions API
+ data?: string
+ mime_type?: string
+ uri?: string
// snake_case (Python SDK)
inline_data?: {
data: string
@@ -109,6 +113,9 @@ export interface GeminiImageMessage {
export interface GeminiDocumentMessage {
type: 'document' | 'image' // 'image' when SDK misdetects PDF by MIME type
+ // Gemini Interactions API
+ data?: string
+ mime_type?: string
// snake_case (Python SDK)
inline_data?: {
data: string
diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.py
index 493c2c1df970..1b765b4f2c43 100644
--- a/posthog/taxonomy/taxonomy.py
+++ b/posthog/taxonomy/taxonomy.py
@@ -3441,6 +3441,12 @@ CORE_FILTER_DEFINITIONS_BY_GROUP: dict[str, dict[str, CoreFilterDefinition]] = {
"examples": [16],
"type": "Numeric",
},
+ "$ai_cache_read_image_tokens": {
+ "label": "AI cache read image tokens (LLM)",
+ "description": "The number of image tokens read from the prompt cache.",
+ "examples": [16],
+ "type": "Numeric",
+ },
"$ai_audio_cost_usd": {
"label": "AI audio cost USD (LLM)",
"description": "The cost in USD of the audio tokens for this LLM request.",
diff --git a/products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.test.tsx b/products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.test.tsx
index 8e03821a46c2..012bc3e6dd7a 100644
--- a/products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.test.tsx
+++ b/products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.test.tsx
@@ -149,9 +149,112 @@ describe('LLMMessageDisplay', () => {
)
expect(container.querySelector('img')).toBeNull()
+ expect(container.textContent).not.toContain('https://example.com/hidden.png')
expect(container.querySelector('[data-attr="ai-message-redacted-media"]')).not.toBeNull()
})
+ describe.each([
+ { minimal: false, encode: true },
+ { minimal: true, encode: true },
+ { minimal: false, encode: false },
+ { minimal: true, encode: false },
+ ])('redacted image content (minimal=$minimal, JSON-string=$encode)', ({ minimal, encode }) => {
+ it.each([
+ [
+ 'image field',
+ {
+ type: 'image',
+ image: 'https://example.com/hidden.png',
+ inlineData: { data: '[base64 image redacted]' },
+ },
+ ],
+ ['URI field', { type: 'image', uri: 'https://example.com/hidden.png', data: '[base64 image redacted]' }],
+ [
+ 'legacy content field',
+ {
+ type: 'image',
+ content: {
+ type: 'image',
+ image: 'https://example.com/hidden.png',
+ source: { data: '[base64 image redacted]' },
+ },
+ },
+ ],
+ ])('redacts %s before choosing a source', (_label, content) => {
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay
+ message={{
+ role: 'assistant',
+ content: encode
+ ? JSON.stringify(content)
+ : (content as unknown as CompatMessage['content']),
+ }}
+ show
+ minimal={minimal}
+ />
+ </Provider>
+ )
+ expect(container.querySelector('[data-attr="ai-message-redacted-media"]')).not.toBeNull()
+ expect(container.querySelector('img')).toBeNull()
+ expect(container.textContent).not.toContain('https://example.com/hidden.png')
+ })
+
+ it.each([
+ { type: 'image', uri: 'https://example.com/hidden.png', data: '[base64 image redacted]' },
+ {
+ type: 'image',
+ uri: 'https://example.com/hidden.png',
+ content: {
+ image: 'https://example.com/hidden.png',
+ source: { data: '[base64 image redacted]' },
+ },
+ },
+ {
+ type: 'image',
+ content: {
+ image: 'https://example.com/hidden.png',
+ inlineData: { data: '[base64 image redacted]' },
+ },
+ },
+ { type: 'image', content: { image: '[base64 image redacted]', uri: 'https://example.com/hidden.png' } },
+ ])('keeps adjacent text while hiding redacted image sources in content arrays: %j', async (image) => {
+ const content = [{ type: 'text', text: 'a readable neighboring sentence' }, image]
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay
+ message={{
+ role: 'assistant',
+ content: encode ? JSON.stringify(content) : (content as MultiModalContentItem[]),
+ }}
+ show
+ minimal={minimal}
+ />
+ </Provider>
+ )
+ await waitFor(() => expect(container.textContent).toContain('a readable neighboring sentence'), {
+ timeout: JSON_VIEWER_TIMEOUT_MS,
+ })
+ expect(container.textContent).not.toContain('https://example.com/hidden.png')
+ expect(container.querySelector('img')).toBeNull()
+ if (!minimal) {
+ expect(container.querySelector('[data-attr="ai-message-redacted-media"]')).not.toBeNull()
+ }
+ })
+ })
+
+ it.each(['[{"status":"complete"}]', '["a readable JSON value"]'])(
+ 'preserves an ordinary JSON array in the minimal preview: %s',
+ (content) => {
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay message={{ role: 'assistant', content }} show minimal />
+ </Provider>
+ )
+ expect(container.textContent).toContain(content)
+ }
+ )
+
it('keeps the transcript when redacted audio replaces the player', () => {
const message: CompatMessage = {
role: 'user',
@@ -732,9 +835,204 @@ describe('ImageMessageDisplay', () => {
expect(image).toHaveAttribute('data-attr', 'ai-message-image')
})
+ it('renders a Gemini Interactions image with top-level data and mime_type', () => {
+ const imageData = 'iVBORw0KGgo='
+ const message: CompatMessage = {
+ role: 'assistant',
+ content: [{ type: 'image', data: imageData, mime_type: 'image/png' } as MultiModalContentItem],
+ }
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay message={message} show />
+ </Provider>
+ )
+
+ expect(container.querySelector('[data-attr="ai-message-image"]')).toHaveAttribute(
+ 'src',
+ `data:image/png;base64,${imageData}`
+ )
+ expect(container.textContent).not.toContain(imageData)
+ })
+
+ it('shows a placeholder for a redacted Gemini Interactions image', () => {
+ const message: CompatMessage = {
+ role: 'assistant',
+ content: [
+ { type: 'image', data: '[base64 image/png redacted]', mime_type: 'image/png' } as MultiModalContentItem,
+ ],
+ }
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay message={message} show />
+ </Provider>
+ )
+
+ expect(container.querySelector('img')).toBeNull()
+ expect(screen.getByText('Image not captured.')).toBeInTheDocument()
+ expect(container.textContent).not.toContain('[base64 image/png redacted]')
+ })
+
+ it('renders a Gemini Interactions image from an HTTPS URI', () => {
+ const uri = 'https://example.com/image.png'
+ const message: CompatMessage = {
+ role: 'assistant',
+ content: [{ type: 'image', uri } as MultiModalContentItem],
+ }
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay message={message} show />
+ </Provider>
+ )
+
+ expect(container.querySelector('[data-attr="ai-message-image"]')).toHaveAttribute('src', uri)
+ })
+
+ it('renders a Gemini Interactions image from an offloaded blob URI', () => {
+ const uri = `phaiblob://v1/sha256/${'a'.repeat(64)}?mime=image%2Fpng&size=131072`
+ const message: CompatMessage = {
+ role: 'assistant',
+ content: [{ type: 'image', uri } as MultiModalContentItem],
+ }
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay message={message} show />
+ </Provider>
+ )
+
+ expect(container.querySelector('[data-attr="ai-message-image"]')).not.toBeNull()
+ expect(container.textContent).not.toContain(uri)
+ })
+
+ it.each<[string, { type: string } & Record<string, unknown>, string]>([
+ ['unsupported Gemini URI', { type: 'image', uri: 'javascript:alert(1)' }, 'javascript:alert(1)'],
+ ['MIME-less Gemini data', { type: 'image', data: 'iVBORw0KGgo=' }, 'iVBORw0KGgo='],
+ ['MIME-less nested Gemini data', { type: 'image', inline_data: { data: 'iVBORw0KGgo=' } }, 'iVBORw0KGgo='],
+ [
+ 'unsupported OpenAI URL',
+ { type: 'image_url', image_url: { url: 'javascript:alert(1)' } },
+ 'javascript:alert(1)',
+ ],
+ ['unsupported Vercel URL', { type: 'image', image: 'gs://example/image.png' }, 'gs://example/image.png'],
+ [
+ 'unsupported input_image URL',
+ { type: 'input_image', image_url: 'gs://example/image.png' },
+ 'gs://example/image.png',
+ ],
+ ])('keeps %s inspectable without labeling it redacted', async (_label, part, payload) => {
+ const message: CompatMessage = { role: 'assistant', content: [part as MultiModalContentItem] }
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay message={message} show />
+ </Provider>
+ )
+ expect(container.querySelector('img')).toBeNull()
+ expect(screen.queryByText('Image not captured.')).not.toBeInTheDocument()
+ await waitFor(() => expect(container.textContent).toContain(payload), { timeout: JSON_VIEWER_TIMEOUT_MS })
+ expect(container.querySelector('a[href^="javascript:"]')).toBeNull()
+ expect(container.querySelector('a[href^="gs:"]')).toBeNull()
+ })
+
+ it.each<[string, { type: string } & Record<string, unknown>]>([
+ ['Gemini URI', { type: 'image', uri: 'https://example.com/image.png' }],
+ ['OpenAI URL', { type: 'image_url', image_url: { url: 'https://example.com/image.png' } }],
+ ['Vercel image', { type: 'image', image: 'https://example.com/image.png' }],
+ ['input_image', { type: 'input_image', image_url: 'https://example.com/image.png' }],
+ [
+ 'Anthropic data',
+ { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0KGgo=' } },
+ ],
+ ['Gemini inline data', { type: 'image', data: 'iVBORw0KGgo=', mime_type: 'image/png' }],
+ ])('shows captured content when the %s image fails to load', async (_label, part) => {
+ const message: CompatMessage = { role: 'assistant', content: [part as MultiModalContentItem] }
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay message={message} show />
+ </Provider>
+ )
+ const image = container.querySelector('img')!
+ expect(image).not.toBeNull()
+ fireEvent.error(image)
+ expect(container.querySelector('img')).toBeNull()
+ expect(screen.queryByText('Image not captured.')).not.toBeInTheDocument()
+ await waitFor(() => expect(container.querySelector('.react-json-view')).not.toBeNull(), {
+ timeout: JSON_VIEWER_TIMEOUT_MS,
+ })
+ expect(container.textContent).toContain('type')
+ expect(container.textContent).toContain(
+ _label.includes('data') ? 'iVBORw0KGgo=' : 'https://example.com/image.png'
+ )
+ })
+
+ it('retries the image preview when a mounted message receives a new source', () => {
+ const message = (uri: string): CompatMessage => ({
+ role: 'assistant',
+ content: [{ type: 'image', uri } as MultiModalContentItem],
+ })
+ const { container, rerender } = render(
+ <Provider>
+ <LLMMessageDisplay message={message('https://example.com/first.png')} show />
+ </Provider>
+ )
+ fireEvent.error(container.querySelector('img')!)
+ expect(container.querySelector('img')).toBeNull()
+ rerender(
+ <Provider>
+ <LLMMessageDisplay message={message('https://example.com/second.png')} show />
+ </Provider>
+ )
+ expect(container.querySelector('img')).toHaveAttribute('src', 'https://example.com/second.png')
+ rerender(
+ <Provider>
+ <LLMMessageDisplay message={message('https://example.com/first.png')} show />
+ </Provider>
+ )
+ expect(container.querySelector('img')).toHaveAttribute('src', 'https://example.com/first.png')
+ })
+
+ it('does not reveal a URI when a Gemini Interactions image has redacted data', () => {
+ const message: CompatMessage = {
+ role: 'assistant',
+ content: [
+ {
+ type: 'image',
+ data: '[base64 image/png redacted]',
+ mime_type: 'image/png',
+ uri: 'https://example.com/image.png',
+ } as MultiModalContentItem,
+ ],
+ }
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay message={message} show />
+ </Provider>
+ )
+
+ expect(container.querySelector('img')).toBeNull()
+ expect(screen.getByText('Image not captured.')).toBeInTheDocument()
+ expect(container.textContent).not.toContain('https://example.com/image.png')
+ })
+
+ it('does not use a Gemini Interactions image URI when MIME-less data was redacted', () => {
+ const uri = 'https://example.com/image.png'
+ const message: CompatMessage = {
+ role: 'assistant',
+ content: [{ type: 'image', data: '[base64 image redacted]', uri } as MultiModalContentItem],
+ }
+ const { container } = render(
+ <Provider>
+ <LLMMessageDisplay message={message} show />
+ </Provider>
+ )
+
+ expect(container.querySelector('img')).toBeNull()
+ expect(screen.getByText('Image not captured.')).toBeInTheDocument()
+ expect(container.textContent).not.toContain(uri)
+ })
+
const REDACTED = '[base64 image redacted]'
const redactedParts: [string, MultiModalContentItem, string][] = [
['python image sentinel', { type: 'image_url', image_url: { url: REDACTED } }, 'Image'],
+ ['Gemini URI sentinel', { type: 'image', uri: REDACTED } as MultiModalContentItem, 'Image'],
['node image sentinel', { type: 'image_url', image_url: { url: '[base64 image/png redacted]' } }, 'Image'],
['data-uri-wrapped sentinel', { type: 'image_url', image_url: { url: `data:;base64,${REDACTED}` } }, 'Image'],
['vercel image sentinel', { type: 'image', image: REDACTED }, 'Image'],
@@ -754,6 +1052,33 @@ describe('ImageMessageDisplay', () => {
{ type: 'image', inlineData: { mimeType: 'image/png', data: REDACTED } },
'Image',
],
+ [
+ 'MIME-less nested sentinel with URI',
+ {
+ type: 'image',
+ inline_data: { data: REDACTED },
+ uri: 'https://example.com/hidden.png',
+ } as MultiModalContentItem,
+ 'Image',
+ ],
+ [
+ 'MIME-less camelCase sentinel with URI',
+ {
+ type: 'image',
+ inlineData: { data: REDACTED },
+ uri: 'https://example.com/hidden.png',
+ } as MultiModalContentItem,
+ 'Image',
+ ],
+ [
+ 'MIME-less Anthropic sentinel with URI',
+ {
+ type: 'image',
+ source: { data: REDACTED },
+ uri: 'https://example.com/hidden.png',
+ } as MultiModalContentItem,
+ 'Image',
+ ],
['file sentinel', { type: 'file', file: { file_data: '[base64 file redacted]', filename: 'doc.pdf' } }, 'File'],
[
'anthropic document sentinel',
@@ -781,6 +1106,7 @@ describe('ImageMessageDisplay', () => {
)
expect(container.querySelector('img')).toBeNull()
+ expect(container.textContent).not.toContain('https://example.com/hidden.png')
expect(container.querySelector('[data-attr="ai-message-redacted-media"]')).not.toBeNull()
expect(screen.getByText(`${label} not captured.`)).toBeInTheDocument()
expect(screen.getByRole('link')).toHaveAttribute(
diff --git a/products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.tsx b/products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.tsx
index 56597765645d..621490911e13 100644
--- a/products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.tsx
+++ b/products/ai_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.tsx
@@ -14,10 +14,11 @@ import { humanFriendlyNumber } from 'lib/utils/numbers'
import { teamLogic } from 'scenes/teamLogic'
import { aiBlobRenderHandlers, resolveAiBlobUrl, resolveDataUri } from '../aiBlob'
+import { AIImage } from '../components/AIImage'
import { getJsonContainerForDisplay, JSONValueDisplay } from '../components/JSONValueDisplay'
import { MessageSentimentBar } from '../components/SentimentTag'
import { LLMInputOutput } from '../LLMInputOutput'
-import { isRenderableMediaSource, redactedMediaKind } from '../mediaSource'
+import { isRedactedMediaSentinel, isRenderableMediaSource, redactedMediaKind } from '../mediaSource'
import { SearchHighlight } from '../SearchHighlight'
import { containsSearchQuery } from '../searchUtils'
import type { GenerationSentiment } from '../sentimentResults'
@@ -481,12 +482,19 @@ export const ImageMessageDisplay = ({ message }: { message: ImageDisplayMessage
if (typeof content === 'string') {
return <span>{content}</span>
} else if (content?.image) {
- if (!isRenderableMediaSource(content.image)) {
+ if (isRedactedMediaSentinel(content.image)) {
return <RedactedMediaPlaceholder kind="image" />
}
+ if (!isRenderableMediaSource(content.image)) {
+ return <HighlightedJSONViewer src={content} name={null} collapsed={5} />
+ }
const src = resolveAiBlobUrl(content.image, currentTeamId)
return (
- <img src={src} alt="User sent image" data-attr="ai-message-image" {...aiBlobRenderHandlers(src, 'image')} />
+ <AIImage
+ src={src}
+ alt="User sent image"
+ fallback={<HighlightedJSONViewer src={content} name={null} collapsed={5} />}
+ />
)
}
@@ -538,14 +546,16 @@ function renderContentItem(
}
if (isOpenAIImageURLMessage(item)) {
+ if (!isRenderableMediaSource(item.image_url.url)) {
+ return <HighlightedJSONViewer src={item} name={null} collapsed={5} searchQuery={searchQuery} />
+ }
const src = resolveAiBlobUrl(item.image_url.url, currentTeamId)
return (
- <img
+ <AIImage
src={src}
alt="Message content"
className="max-w-full max-h-[400px] rounded"
- data-attr="ai-message-image"
- {...aiBlobRenderHandlers(src, 'image')}
+ fallback={<HighlightedJSONViewer src={item} name={null} collapsed={5} searchQuery={searchQuery} />}
/>
)
}
@@ -553,29 +563,31 @@ function renderContentItem(
if (isAnthropicImageMessage(item)) {
const src = resolveDataUri(item.source.data, item.source.media_type, currentTeamId)
return (
- <img
+ <AIImage
src={src}
alt="Message content"
className="max-w-full max-h-[400px] rounded"
- data-attr="ai-message-image"
- {...aiBlobRenderHandlers(src, 'image')}
+ fallback={<HighlightedJSONViewer src={item} name={null} collapsed={5} searchQuery={searchQuery} />}
/>
)
}
if (isGeminiImageMessage(item)) {
const inlineData = getGeminiInlineData(item)
- if (!inlineData) {
- return null
+ const src = inlineData
+ ? resolveDataUri(inlineData.data, inlineData.mime_type, currentTeamId)
+ : typeof item.uri === 'string' && isRenderableMediaSource(item.uri)
+ ? resolveAiBlobUrl(item.uri, currentTeamId)
+ : null
+ if (!src) {
+ return <HighlightedJSONViewer src={item} name={null} collapsed={5} searchQuery={searchQuery} />
}
- const src = resolveDataUri(inlineData.data, inlineData.mime_type, currentTeamId)
return (
- <img
+ <AIImage
src={src}
alt="Message content"
className="max-w-full max-h-[400px] rounded"
- data-attr="ai-message-image"
- {...aiBlobRenderHandlers(src, 'image')}
+ fallback={<HighlightedJSONViewer src={item} name={null} collapsed={5} searchQuery={searchQuery} />}
/>
)
}
@@ -767,6 +779,16 @@ export const LLMMessageDisplay = React.memo(
}): JSX.Element => {
const { currentTeamId } = useValues(teamLogic)
const { role, content, ...additionalKwargs } = message
+ const parsedJsonContainer =
+ !show || Array.isArray(content)
+ ? null
+ : getJsonContainerForDisplay(content, { allowEmptyContainers: false })
+ const displayContent =
+ Array.isArray(parsedJsonContainer) &&
+ parsedJsonContainer.some((item) => redactedMediaKind(item) === 'image')
+ ? (parsedJsonContainer as MultiModalContentItem[])
+ : content
+ const isRedactedImage = redactedMediaKind(parsedJsonContainer) === 'image'
let resolvedIsRenderingMarkdown = isRenderingMarkdown
let resolvedIsRenderingXml = isRenderingXml
@@ -820,8 +842,6 @@ export const LLMMessageDisplay = React.memo(
</>
)
}
- const parsedJsonContainer = getJsonContainerForDisplay(content, { allowEmptyContainers: false })
-
if (parsedJsonContainer) {
if (isObject(parsedJsonContainer)) {
const imageMessage = getImageDisplayMessage(parsedJsonContainer)
@@ -997,12 +1017,14 @@ export const LLMMessageDisplay = React.memo(
)}
{show && !!content && (
<div className={!minimal ? 'p-2 border-t' : 'p-1'}>
- {minimal ? (
+ {isRedactedImage ? (
+ <RedactedMediaPlaceholder kind="image" />
+ ) : minimal ? (
<LemonMarkdown className="whitespace-pre-wrap">
- {extractMinimalPreview(content)}
+ {extractMinimalPreview(displayContent)}
</LemonMarkdown>
) : (
- renderMessageContent(content, searchQuery)
+ renderMessageContent(displayContent, searchQuery)
)}
</div>
)}
diff --git a/products/ai_observability/frontend/components/AIImage.stories.tsx b/products/ai_observability/frontend/components/AIImage.stories.tsx
new file mode 100644
index 000000000000..440eae4eb53e
--- /dev/null
+++ b/products/ai_observability/frontend/components/AIImage.stories.tsx
@@ -0,0 +1,44 @@
+import type { Meta, StoryObj } from '@storybook/react'
+import { waitFor } from '@testing-library/dom'
+
+import { HighlightedJSONViewer } from 'lib/components/HighlightedJSONViewer'
+
+import { AIImage } from './AIImage'
+
+const meta: Meta<typeof AIImage> = {
+ title: 'Scenes-App/AI observability/Image preview',
+ component: AIImage,
+ args: {
+ alt: 'Message content',
+ className: 'max-w-full max-h-40 rounded',
+ fallback: <HighlightedJSONViewer src={{ type: 'image', uri: 'https://example.com/image.png' }} name={null} />,
+ },
+}
+export default meta
+
+type Story = StoryObj<typeof AIImage>
+
+export const LoadedImage: Story = {
+ args: { src: '/static/posthog-logo.png' },
+ play: async ({ canvasElement }) => {
+ await waitFor(() => {
+ if (!canvasElement.querySelector('img')?.naturalWidth) {
+ throw new Error('The image must finish loading')
+ }
+ })
+ },
+}
+
+export const FailedImage: Story = {
+ args: { src: 'data:image/png;base64,invalid' },
+ play: async ({ canvasElement }) => {
+ await waitFor(() => {
+ if (
+ canvasElement.querySelector('img') ||
+ !canvasElement.textContent?.includes('https://example.com/image.png')
+ ) {
+ throw new Error('The failed image must show its captured URI')
+ }
+ })
+ },
+}
diff --git a/products/ai_observability/frontend/components/AIImage.tsx b/products/ai_observability/frontend/components/AIImage.tsx
new file mode 100644
index 000000000000..41c2a6941ed1
--- /dev/null
+++ b/products/ai_observability/frontend/components/AIImage.tsx
@@ -0,0 +1,33 @@
+import { ReactNode, useEffect, useState } from 'react'
+
+import { aiBlobRenderHandlers } from '../aiBlob'
+
+interface AIImageProps {
+ src: string
+ alt: string
+ className?: string
+ fallback: ReactNode
+}
+
+export function AIImage({ src, alt, className, fallback }: AIImageProps): JSX.Element {
+ const [failedSrc, setFailedSrc] = useState<string | null>(null)
+ useEffect(() => setFailedSrc(null), [src])
+ const handlers = aiBlobRenderHandlers(src, 'image')
+
+ if (failedSrc === src) {
+ return <>{fallback}</>
+ }
+ return (
+ <img
+ src={src}
+ alt={alt}
+ className={className}
+ data-attr="ai-message-image"
+ onLoad={handlers.onLoad}
+ onError={() => {
+ handlers.onError?.()
+ setFailedSrc(src)
+ }}
+ />
+ )
+}
diff --git a/products/ai_observability/frontend/mediaSource.test.ts b/products/ai_observability/frontend/mediaSource.test.ts
index 249f7e25b98f..c3b04541b2ae 100644
--- a/products/ai_observability/frontend/mediaSource.test.ts
+++ b/products/ai_observability/frontend/mediaSource.test.ts
@@ -26,6 +26,21 @@ describe('mediaSource', () => {
{ type: 'image', inline_data: { mime_type: 'image/png', data: '[base64 image redacted]' } },
'image',
],
+ ['Gemini URI sentinel', { type: 'image', uri: '[base64 image redacted]' }, 'image'],
+ [
+ 'legacy image with a nested sentinel and competing sources',
+ {
+ type: 'image',
+ uri: REMOTE_URL,
+ content: { image: REMOTE_URL, source: { data: '[base64 image redacted]' } },
+ },
+ 'image',
+ ],
+ [
+ 'legacy image with a sentinel image source',
+ { type: 'image', content: { image: '[base64 image redacted]', uri: REMOTE_URL } },
+ 'image',
+ ],
['audio sentinel', { type: 'audio', mime_type: 'audio/wav', data: '[base64 audio redacted]' }, 'audio'],
['vercel image sentinel', { type: 'image', image: '[base64 image redacted]' }, 'image'],
['input_image sentinel', { type: 'input_image', image_url: '[base64 image redacted]' }, 'image'],
@@ -34,7 +49,26 @@ describe('mediaSource', () => {
{ type: 'image_url', image_url: { url: 'data:image/png;base64,[base64 image redacted]' } },
'image',
],
- ['url that is not a source at all', { type: 'image_url', image_url: { url: 'nonsense' } }, 'image'],
+ [
+ 'MIME-less Gemini image sentinel',
+ { type: 'image', data: '[base64 image redacted]', uri: REMOTE_URL },
+ 'image',
+ ],
+ [
+ 'MIME-less nested Gemini sentinel',
+ { type: 'image', inline_data: { data: '[base64 image redacted]' }, uri: REMOTE_URL },
+ 'image',
+ ],
+ [
+ 'MIME-less camelCase sentinel',
+ { type: 'image', inlineData: { data: '[base64 image redacted]' }, uri: REMOTE_URL },
+ 'image',
+ ],
+ [
+ 'MIME-less Anthropic sentinel',
+ { type: 'image', source: { data: '[base64 image redacted]' }, uri: REMOTE_URL },
+ 'image',
+ ],
])('flags %s as redacted', (_name, item, expected) => {
expect(redactedMediaKind(item)).toBe(expected)
})
@@ -57,6 +91,19 @@ describe('mediaSource', () => {
['audio raw base64', { type: 'audio', mime_type: 'audio/wav', data: BASE64 }],
['file as data uri', { type: 'file', file: { file_data: DATA_URI, filename: 'doc.pdf' } }],
['file as raw base64', { type: 'file', file: { file_data: BASE64, filename: 'doc.pdf' } }],
+ ['unsupported Gemini URI', { type: 'image', uri: 'gs://example/image.png' }],
+ ['MIME-less Gemini data', { type: 'image', data: BASE64 }],
+ ['unsupported OpenAI URL', { type: 'image_url', image_url: { url: 'javascript:alert(1)' } }],
+ ['unsupported input_image URL', { type: 'input_image', image_url: 'gs://example/image.png' }],
+ ['unsupported Vercel URL', { type: 'image', image: 'nonsense' }],
+ ['captured legacy image', { type: 'image', content: { image: REMOTE_URL, inlineData: { data: BASE64 } } }],
+ [
+ 'legacy non-image MIME payload',
+ {
+ type: 'image',
+ content: { mime_type: 'application/pdf', data: '[base64 file redacted]', uri: REMOTE_URL },
+ },
+ ],
['text part', { type: 'text', text: 'hello' }],
['unrelated part', { type: 'function', function: { name: 'get_weather' } }],
])('leaves %s renderable', (_name, item) => {
@@ -64,6 +111,20 @@ describe('mediaSource', () => {
})
})
+ it('bounds legacy content inspection even when objects are cyclic', () => {
+ const item: Record<string, unknown> = { type: 'image', image: REMOTE_URL }
+ item.content = item
+ expect(redactedMediaKind(item)).toBeNull()
+ item.inlineData = { data: '[base64 image redacted]' }
+ expect(redactedMediaKind(item)).toBe('image')
+ })
+
+ it('retains file classification for redacted non-image Gemini MIME payloads', () => {
+ expect(redactedMediaKind({ type: 'image', mime_type: 'application/pdf', data: '[base64 file redacted]' })).toBe(
+ 'file'
+ )
+ })
+
describe('isRedactedMediaSentinel', () => {
it.each([
'[base64 image redacted]',
diff --git a/products/ai_observability/frontend/mediaSource.ts b/products/ai_observability/frontend/mediaSource.ts
index 4cf10549759b..a9540d4ff7c0 100644
--- a/products/ai_observability/frontend/mediaSource.ts
+++ b/products/ai_observability/frontend/mediaSource.ts
@@ -1,11 +1,11 @@
+import { isObject } from 'lib/utils/guards'
+
import { parseAiBlobPointer } from './aiBlob'
import {
- getGeminiInlineData,
isAnthropicDocumentMessage,
isAnthropicImageMessage,
isGeminiAudioMessage,
isGeminiDocumentMessage,
- isGeminiImageMessage,
isOpenAIAudioMessage,
isOpenAIFileMessage,
isOpenAIImageURLMessage,
@@ -31,31 +31,45 @@ export function isRenderableMediaSource(value: string): boolean {
return /^data:/i.test(value) || /^https?:\/\//i.test(value) || parseAiBlobPointer(value) !== null
}
-function hasRedactedGeminiInlineData(item: unknown): boolean {
- const inlineData = getGeminiInlineData(item)
- return inlineData !== null && isRedactedMediaSentinel(inlineData.data)
+function hasRedactedMediaData(item: unknown): boolean {
+ if (!isObject(item)) {
+ return false
+ }
+ return [item, item.inline_data, item.inlineData, item.source].some(
+ (payload) => isObject(payload) && typeof payload.data === 'string' && isRedactedMediaSentinel(payload.data)
+ )
}
-/**
- * Classifies a content item that cannot be rendered as media, so the caller can show a placeholder
- * instead of handing the value to the browser as a src. Branches are ordered to match the render
- * ladder in ConversationMessagesDisplay, so the shape judged here is the shape that would render.
- */
+/** Only explicit SDK redaction markers imply that media was not captured. */
export function redactedMediaKind(item: unknown): RedactedMediaKind | null {
if (!item || typeof item !== 'object' || !('type' in item)) {
return null
}
- // A url field has to carry a real source. Anything else, sentinel or not, resolves against the
- // page and 404s, so the whitelist is the gate.
- if (item.type === 'image' && 'image' in item && typeof item.image === 'string') {
- return isRenderableMediaSource(item.image) ? null : 'image'
+ if (item.type === 'image') {
+ // Legacy messages wrap one image payload in content. Inspect that level only, so cycles
+ // cannot recurse; non-image MIME payloads retain Gemini's document classification.
+ const legacyContent = 'content' in item && isObject(item.content) ? { ...item.content, type: 'image' } : null
+ for (const image of [item, legacyContent]) {
+ if (!image) {
+ continue
+ }
+ if (
+ (!isGeminiDocumentMessage(image) &&
+ (hasRedactedMediaData(image) ||
+ ('uri' in image && typeof image.uri === 'string' && isRedactedMediaSentinel(image.uri)))) ||
+ ('image' in image && typeof image.image === 'string' && isRedactedMediaSentinel(image.image))
+ ) {
+ return 'image'
+ }
+ }
}
+
if (item.type === 'input_image' && 'image_url' in item && typeof item.image_url === 'string') {
- return isRenderableMediaSource(item.image_url) ? null : 'image'
+ return isRedactedMediaSentinel(item.image_url) ? 'image' : null
}
if (isOpenAIImageURLMessage(item)) {
- return isRenderableMediaSource(item.image_url.url) ? null : 'image'
+ return isRedactedMediaSentinel(item.image_url.url) ? 'image' : null
}
// Payload fields legitimately hold bare base64, which no whitelist can accept, so only the
@@ -63,9 +77,6 @@ export function redactedMediaKind(item: unknown): RedactedMediaKind | null {
if (isAnthropicImageMessage(item)) {
return isRedactedMediaSentinel(item.source.data) ? 'image' : null
}
- if (isGeminiImageMessage(item)) {
- return hasRedactedGeminiInlineData(item) ? 'image' : null
- }
if (isOpenAIFileMessage(item)) {
return isRedactedMediaSentinel(item.file.file_data) ? 'file' : null
}
@@ -73,7 +84,7 @@ export function redactedMediaKind(item: unknown): RedactedMediaKind | null {
return isRedactedMediaSentinel(item.source.data) ? 'file' : null
}
if (isGeminiDocumentMessage(item)) {
- return hasRedactedGeminiInlineData(item) ? 'file' : null
+ return hasRedactedMediaData(item) ? 'file' : null
}
if (isOpenAIAudioMessage(item) || isGeminiAudioMessage(item)) {
return isRedactedMediaSentinel(item.data) ? 'audio' : null
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toAttachment.test.ts b/products/ai_observability/frontend/redesign/trace/adapter/toAttachment.test.ts
index 1d7eaca9338c..d896b49eee90 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/toAttachment.test.ts
+++ b/products/ai_observability/frontend/redesign/trace/adapter/toAttachment.test.ts
@@ -74,6 +74,55 @@ describe('toAttachment', () => {
url: 'data:application/pdf;base64,JVBERi0=',
},
],
+ [
+ 'a Gemini Interactions image with top-level data and MIME type',
+ { type: 'image', data: 'iVBORw0=', mime_type: 'image/png' },
+ { mediaType: 'image', name: null, mimeType: 'image/png', url: 'data:image/png;base64,iVBORw0=' },
+ ],
+ [
+ 'a Gemini Interactions image with an HTTPS URI',
+ { type: 'image', uri: 'https://cdn.example.com/image.png' },
+ { mediaType: 'image', name: null, mimeType: null, url: 'https://cdn.example.com/image.png' },
+ ],
+ [
+ 'a Gemini Interactions image with an offloaded blob URI',
+ { type: 'image', uri: `phaiblob://v1/sha256/${BLOB_HASH}?mime=image/png` },
+ {
+ mediaType: 'image',
+ name: null,
+ mimeType: 'image/png',
+ url: `/api/projects/7/ai_blob/v1/sha256/${BLOB_HASH}`,
+ },
+ ],
+ [
+ 'a Gemini Interactions image with redacted data and a URI',
+ {
+ type: 'image',
+ data: '[base64 image/png redacted]',
+ mime_type: 'image/png',
+ uri: 'https://cdn.example.com/image.png',
+ },
+ { mediaType: 'image', name: null, mimeType: 'image/png', url: null },
+ ],
+ [
+ 'a Gemini Interactions image with redacted data, no MIME type, and a URI',
+ { type: 'image', data: '[base64 image redacted]', uri: 'https://cdn.example.com/image.png' },
+ { mediaType: 'image', name: null, mimeType: null, url: null },
+ ],
+ [
+ 'an image with a redacted payload and a sibling URL',
+ { type: 'image', inlineData: { data: '[base64 image redacted]', url: 'https://example.com/hidden.png' } },
+ { mediaType: 'image', name: null, mimeType: null, url: null },
+ ],
+ [
+ 'a legacy image whose nested redaction suppresses a competing URI',
+ {
+ type: 'image',
+ uri: 'https://example.com/hidden.png',
+ content: { image: 'https://example.com/hidden.png', inlineData: { data: '[base64 image redacted]' } },
+ },
+ { mediaType: 'image', name: null, mimeType: null, url: null },
+ ],
[
'an image whose payload the SDK redacted',
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: '[base64 image redacted]' } },
diff --git a/products/ai_observability/frontend/redesign/trace/adapter/toAttachment.ts b/products/ai_observability/frontend/redesign/trace/adapter/toAttachment.ts
index 820e87cc6438..f2613693de29 100644
--- a/products/ai_observability/frontend/redesign/trace/adapter/toAttachment.ts
+++ b/products/ai_observability/frontend/redesign/trace/adapter/toAttachment.ts
@@ -1,7 +1,7 @@
import { isObject } from 'lib/utils/guards'
import { parseAiBlobPointer, resolveAiBlobUrl, resolveDataUri } from '../../../aiBlob'
-import { isRenderableMediaSource } from '../../../mediaSource'
+import { isRenderableMediaSource, redactedMediaKind } from '../../../mediaSource'
import { AttachmentMediaType, MessagePart } from '../types'
export type AttachmentPart = Extract<MessagePart, { kind: 'attachment' }>
@@ -22,7 +22,7 @@ const MEDIA_TYPE_BY_ITEM_TYPE: Record<string, AttachmentMediaType> = {
// Providers nest the payload under one of these keys, each with its own field names.
const NESTED_PAYLOAD_KEYS = ['source', 'file', 'input_audio', 'image_url', 'inline_data', 'inlineData']
-const SOURCE_KEYS = ['url', 'file_url', 'image_url', 'file_data', 'image', 'data', 'file']
+const SOURCE_KEYS = ['url', 'file_url', 'image_url', 'file_data', 'image', 'data', 'file', 'uri']
const MIME_KEYS = ['mediaType', 'mimeType', 'mime_type', 'media_type']
const NAME_KEYS = ['filename', 'name']
@@ -60,8 +60,8 @@ function mediaTypeOf(itemType: string, mimeType: string | null): AttachmentMedia
return topLevel === 'image' || topLevel === 'audio' || topLevel === 'video' ? topLevel : 'file'
}
-// Raw base64 payloads become data: URIs. The app's CSP allows media only from its own origin, data: and blob:,
-// so stored blob pointers must resolve to the same-origin blob endpoint.
+// Raw base64 payloads become data: URIs. Image CSP also permits HTTPS sources, while stored blob
+// pointers must resolve to the same-origin blob endpoint.
function resolveUrl(source: string | null, mimeType: string | null, teamId: TeamId): string | null {
if (source === null) {
return null
@@ -99,6 +99,6 @@ export function toAttachment(item: Record<string, unknown> & { type: string }, t
mediaType,
name: readString(containers, NAME_KEYS) ?? (isKnownMedia ? null : item.type),
mimeType,
- url: isKnownMedia ? resolveUrl(source, payloadMime, teamId) : null,
+ url: isKnownMedia && redactedMediaKind(item) === null ? resolveUrl(source, payloadMime, teamId) : null,
}
}
diff --git a/products/ai_observability/frontend/redesign/trace/components/AttachmentImage.test.tsx b/products/ai_observability/frontend/redesign/trace/components/AttachmentImage.test.tsx
new file mode 100644
index 000000000000..776bc142dd30
--- /dev/null
+++ b/products/ai_observability/frontend/redesign/trace/components/AttachmentImage.test.tsx
@@ -0,0 +1,45 @@
+import '@testing-library/jest-dom'
+
+import { fireEvent, render, waitFor } from '@testing-library/react'
+import posthog from 'posthog-js'
+
+import { initKeaTests } from '~/test/init'
+
+import { AttachmentImage } from './AttachmentImage'
+
+describe('AttachmentImage', () => {
+ beforeEach(() => initKeaTests())
+
+ it.each(['load', 'error'] as const)('reports blob image %s telemetry', async (event) => {
+ const capture = jest.spyOn(posthog, 'capture').mockImplementation(() => undefined)
+ try {
+ const hash = (event === 'load' ? 'c' : 'd').repeat(64)
+ const { container } = render(
+ <AttachmentImage url={`/api/projects/7/ai_blob/v1/sha256/${hash}`} alt="Attached image" />
+ )
+ fireEvent[event](container.querySelector('img')!)
+ await waitFor(() =>
+ expect(capture).toHaveBeenCalledWith(
+ 'llma ai blob render',
+ expect.objectContaining({
+ outcome: event === 'load' ? 'success' : 'error',
+ media_kind: 'image',
+ })
+ )
+ )
+ } finally {
+ capture.mockRestore()
+ }
+ })
+
+ it('preserves the source link when an image cannot load, and retries new sources', () => {
+ const url = 'https://example.com/image.png'
+ const { container, rerender } = render(<AttachmentImage url={url} alt="Attached image" />)
+ fireEvent.error(container.querySelector('img')!)
+ expect(container.querySelector('img')).toBeNull()
+ expect(container.textContent).toContain(url)
+ expect(container.querySelector('a')).toHaveAttribute('href', url)
+ rerender(<AttachmentImage url="https://example.com/second.png" alt="Attached image" />)
+ expect(container.querySelector('img')).toHaveAttribute('src', 'https://example.com/second.png')
+ })
+})
diff --git a/products/ai_observability/frontend/redesign/trace/components/AttachmentImage.tsx b/products/ai_observability/frontend/redesign/trace/components/AttachmentImage.tsx
index 2ad17a79294b..d97631f333ad 100644
--- a/products/ai_observability/frontend/redesign/trace/components/AttachmentImage.tsx
+++ b/products/ai_observability/frontend/redesign/trace/components/AttachmentImage.tsx
@@ -1,3 +1,4 @@
+import { AIImage } from '../../../components/AIImage'
import { AttachmentLink } from './AttachmentLink'
import { classifyAttachmentUrl } from './attachmentUrl'
@@ -8,7 +9,12 @@ export interface AttachmentImageProps {
export function AttachmentImage({ url, alt }: AttachmentImageProps): JSX.Element {
const image = (
- <img src={url} alt={alt} className="block max-h-48 max-w-full rounded border border-primary object-contain" />
+ <AIImage
+ src={url}
+ alt={alt}
+ className="block max-h-48 max-w-full rounded border border-primary object-contain"
+ fallback={<span className="break-all">{url}</span>}
+ />
)
const urlKind = classifyAttachmentUrl(url)
if (urlKind === null) {
diff --git a/products/ai_observability/frontend/utils.ts b/products/ai_observability/frontend/utils.ts
index 8946a6761475..18df0ef1f384 100644
--- a/products/ai_observability/frontend/utils.ts
+++ b/products/ai_observability/frontend/utils.ts
@@ -862,6 +862,13 @@ export function getGeminiInlineData(input: unknown): { data: string; mime_type:
}
}
+ // Gemini Interactions API uses top-level data and mime_type.
+ const d = input as Record<string, unknown>
+ const mimeType = d.mime_type ?? d.mimeType
+ if (typeof d.data === 'string' && typeof mimeType === 'string') {
+ return { data: d.data, mime_type: mimeType }
+ }
+
return null
}
@@ -870,7 +877,15 @@ export function isGeminiImageMessage(input: unknown): input is GeminiImageMessag
return false
}
const inlineData = getGeminiInlineData(input)
- return inlineData !== null && inlineData.mime_type.startsWith('image/')
+ if (inlineData) {
+ return inlineData.mime_type.startsWith('image/')
+ }
+ if ('data' in input && typeof input.data === 'string') {
+ return true
+ }
+ const uri = 'uri' in input ? input.uri : null
+ const mimeType = 'mime_type' in input ? input.mime_type : null
+ return typeof uri === 'string' && (typeof mimeType !== 'string' || mimeType.startsWith('image/'))
}
export function isGeminiDocumentMessage(input: unknown): input is GeminiDocumentMessage {
diff --git a/services/mcp/src/lib/trace-compaction.ts b/services/mcp/src/lib/trace-compaction.ts
index 8b89987fb4da..2757c1b9d106 100644
--- a/services/mcp/src/lib/trace-compaction.ts
+++ b/services/mcp/src/lib/trace-compaction.ts
@@ -141,6 +141,7 @@ export const SUMMARY_METADATA_PROPERTIES = new Set([
'$ai_cache_creation_1h_input_tokens',
'$ai_cache_creation_5m_input_tokens',
'$ai_cache_read_audio_tokens',
+ '$ai_cache_read_image_tokens',
'$ai_audio_cost_usd',
'$ai_image_cost_usd',
'$ai_video_cost_usd',
diff --git a/services/mcp/src/lib/trace-property-allowlist.generated.ts b/services/mcp/src/lib/trace-property-allowlist.generated.ts
index c67a9d7edb37..ebf7d192d457 100644
--- a/services/mcp/src/lib/trace-property-allowlist.generated.ts
+++ b/services/mcp/src/lib/trace-property-allowlist.generated.ts
@@ -17,6 +17,7 @@ export const AI_TAXONOMY_EVENT_PROPERTIES = [
'$ai_cache_creation_5m_input_tokens',
'$ai_cache_creation_input_tokens',
'$ai_cache_read_audio_tokens',
+ '$ai_cache_read_image_tokens',
'$ai_cache_read_input_tokens',
'$ai_cache_read_token_price',
'$ai_cache_reporting_exclusive',
diff --git a/services/mcp/tests/unit/trace-redaction.test.ts b/services/mcp/tests/unit/trace-redaction.test.ts
index bec14516ea92..7f1472e93159 100644
--- a/services/mcp/tests/unit/trace-redaction.test.ts
+++ b/services/mcp/tests/unit/trace-redaction.test.ts
@@ -84,6 +84,18 @@ describe('trace redaction', () => {
expect(properties.$session_id).toBe('session-1')
})
+ it.each(['full', 'summary'] as const)('retains cached-image spend metadata at %s detail', (detail) => {
+ const trace = {
+ id: 'trace-1',
+ events: [{ id: 'e1', properties: { $ai_cache_read_image_tokens: 16, custom_secret: SECRETS.credential } }],
+ }
+
+ const { results } = compactTraceResponse({ results: redactTraceResults([trace]) }, detail) as any
+
+ expect(results[0].events[0].properties.$ai_cache_read_image_tokens).toBe(16)
+ expect(secretsIn(results)).toEqual([])
+ })
+
it('withholds every non-AI property value and reports its name instead', () => {
const event = (redactTrace(traceWithSecrets()) as any).events[0]