Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(aio): move normalizer engine cases to a shared json file (#124896)

Carlos Marchalcd51377aa16d
diff --git a/products/ai_observability/frontend/messageNormalization.test.ts b/products/ai_observability/frontend/messageNormalization.test.ts
index d40eb41f2425..d2c6c525f993 100644
--- a/products/ai_observability/frontend/messageNormalization.test.ts
+++ b/products/ai_observability/frontend/messageNormalization.test.ts
@@ -1,7 +1,7 @@
import posthog from 'posthog-js'
import { resolveAiBlobUrl } from './aiBlob'
-import { captureNormalizationFailure, normalizeMessage, normalizeMessages } from './messageNormalization'
+import { captureNormalizationFailure, normalizeMessages } from './messageNormalization'
jest.mock('posthog-js', () => ({ __esModule: true, default: { capture: jest.fn() } }))
@@ -10,82 +10,13 @@ const capture = jest.mocked(posthog.capture)
describe('messageNormalization', () => {
beforeEach(() => jest.clearAllMocks())
- describe('delegates to the recipe normalizer', () => {
- it('normalizeMessage normalizes a single message', () => {
- expect(normalizeMessage({ role: 'assistant', content: 'hi' }, 'user')).toEqual([
- { role: 'assistant', content: 'hi' },
- ])
- })
-
- it('normalizeMessages prepends an available-tools pseudo-message when tools are passed', () => {
- const result = normalizeMessages({ role: 'user', content: 'hi' }, 'user', [{ name: 'search' }])
- expect(result.messages[0]).toEqual({ role: 'available tools', content: '', tools: [{ name: 'search' }] })
- })
-
- it('normalizeMessages carries no message for nullish/scalar input', () => {
- expect(normalizeMessages(null, 'user').messages).toEqual([])
- expect(normalizeMessages(42, 'user').messages).toEqual([])
- })
- })
-
describe('normalizeMessages recognition', () => {
- it('reports a recognized conversation alongside its messages', () => {
- expect(normalizeMessages({ role: 'assistant', content: 'hi' }, 'user')).toEqual({
- messages: [{ role: 'assistant', content: 'hi' }],
- recognized: true,
- })
- })
-
- it('reports opaque state as unrecognized and salvages it', () => {
- const result = normalizeMessages({ file_path: 'src/index.ts' }, 'user')
- expect(result.recognized).toBe(false)
- expect(result.messages).toHaveLength(1)
- })
-
it("never captures — reporting a failure is the caller's decision", () => {
normalizeMessages({ file_path: 'src/index.ts' }, 'user')
expect(capture).not.toHaveBeenCalled()
})
})
- describe('thinking blocks alongside typed function blocks', () => {
- // The Gemini SDK emits a thought summary block next to a tool call. The thinking
- // block knocks the message off compat_array's envelope rule, so the function block
- // must survive per-block delegation as a typed tool call, not stringified JSON.
- it('renders the thought as thinking and keeps the sibling tool call typed', () => {
- const { messages, recognized } = normalizeMessages(
- [
- {
- role: 'assistant',
- content: [
- { type: 'thinking', thinking: 'Two rapid clicks, check the events.' },
- {
- type: 'function',
- function: { name: 'get_events_around', arguments: { rec_t: 320 } },
- },
- ],
- },
- ],
- 'assistant'
- )
- expect(recognized).toBe(true)
- expect(messages).toEqual([
- { role: 'assistant (thinking)', content: 'Two rapid clicks, check the events.' },
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: undefined,
- function: { name: 'get_events_around', arguments: { rec_t: 320 } },
- },
- ],
- },
- ])
- })
- })
-
describe('offloaded media', () => {
const HASH = 'a'.repeat(64)
const POINTER = `phaiblob://v1/sha256/${HASH}?mime=image%2Fpng&size=332378`
diff --git a/products/ai_observability/frontend/normalizerCases.test.ts b/products/ai_observability/frontend/normalizerCases.test.ts
new file mode 100644
index 000000000000..b6052d0b9645
--- /dev/null
+++ b/products/ai_observability/frontend/normalizerCases.test.ts
@@ -0,0 +1,49 @@
+import { RecipeNormalizer } from '@posthog/llm-normalizer'
+
+import normalizerCases from '../shared/normalizer_cases.json'
+
+// Whole-engine cases live in a shared data file so another implementation of the engine can run the same cases,
+// so add or change them there rather than inline. The file is imported rather than read, which lets Jest's
+// related-test selection run this test when only the cases change.
+interface NormalizerCase {
+ name: string
+ call: string
+ input: unknown
+ default_role: string
+ tools?: unknown
+ expected: { messages: unknown[]; recognized: boolean }
+}
+
+const cases: NormalizerCase[] = normalizerCases.cases
+
+describe('normalizer cases', () => {
+ const normalizer = new RecipeNormalizer()
+
+ it('holds uniquely named cases', () => {
+ expect(cases.length).toBeGreaterThan(0)
+ expect(new Set(cases.map((c) => c.name)).size).toBe(cases.length)
+ })
+
+ it.each(cases.map((c) => [c.name, c] as const))('%s', (_, c) => {
+ const outcome =
+ c.call === 'normalizeMessage'
+ ? normalizer.normalizeMessage(c.input, c.default_role)
+ : normalizer.normalizeMessages(c.input, c.default_role, c.tools)
+ expect(outcome).toEqual(c.expected)
+ })
+
+ describe('inputs JSON cannot encode', () => {
+ it('returns no messages for undefined input', () => {
+ expect(normalizer.normalizeMessages(undefined, 'assistant')).toEqual({ messages: [], recognized: true })
+ })
+
+ it('handles Vercel AI SDK tool-result with non-serializable output', () => {
+ const circular: Record<string, unknown> = { key: 'value' }
+ circular.self = circular
+ const message = { type: 'tool-result', toolCallId: 'tc_circ', toolName: 'bad_tool', output: circular }
+ expect(normalizer.normalizeMessage(message, 'assistant').messages).toEqual([
+ { role: 'assistant (tool result)', content: '[object Object]', tool_call_id: 'tc_circ' },
+ ])
+ })
+ })
+})
diff --git a/products/ai_observability/frontend/utils.test.ts b/products/ai_observability/frontend/utils.test.ts
index a406eca4e6dc..62c48f393632 100644
--- a/products/ai_observability/frontend/utils.test.ts
+++ b/products/ai_observability/frontend/utils.test.ts
@@ -1,10 +1,8 @@
-import { RecipeNormalizer } from '@posthog/llm-normalizer'
-
import api from 'lib/api'
import { LLMTrace, LLMTraceEvent } from '~/queries/schema/schema-general'
-import { AnthropicInputMessage, CompatMessage, OpenAICompletionMessage } from './types'
+import { CompatMessage } from './types'
import {
asString,
costContextFromProperties,
@@ -227,17 +225,7 @@ describe('getSummarizationLookupDateRange', () => {
})
})
-// The recipe-based `RecipeNormalizer` in `./normalizer` is the production code path.
-const recipe = new RecipeNormalizer()
-const IMPLS = [
- {
- name: 'recipe',
- normalizeMessage: (r: unknown, d: string) => recipe.normalizeMessage(r, d).messages,
- normalizeMessages: (m: unknown, d: string, t?: unknown) => recipe.normalizeMessages(m, d, t).messages,
- },
-] as const
-
-describe.each(IMPLS)('AI observability utils [$name]', ({ normalizeMessage, normalizeMessages }) => {
+describe('AI observability utils', () => {
beforeEach(() => {
console.warn = jest.fn()
})
@@ -288,2253 +276,605 @@ describe.each(IMPLS)('AI observability utils [$name]', ({ normalizeMessage, norm
})
})
- describe('normalizeOutputMessage', () => {
- it('parses OpenAI message', () => {
- const message: OpenAICompletionMessage = {
- role: 'assistant',
- content: 'Hello, world!',
- }
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: 'Hello, world!',
- },
- ])
+ describe('parsePartialJSON', () => {
+ it.each([
+ ['full object', '{"key": "value", "n": 42}', { key: 'value', n: 42 }],
+ ['full array', '[1, 2, 3]', [1, 2, 3]],
+ ['truncated object', '{"key": "value", "long_field": "some te', { key: 'value', long_field: 'some te' }],
+ ['truncated nested object', '{"a": {"b": "c", "d": "efg', { a: { b: 'c', d: 'efg' } }],
+ ['truncated array', '[1, 2, "hel', [1, 2, 'hel']],
+ ['truncated string value', '"hello wor', 'hello wor'],
+ ])('%s', (_label, input, expected) => {
+ expect(parsePartialJSON(input)).toEqual(expected)
})
- it('stringifies incomplete OpenAI message', () => {
- const message = {
- role: 'assistant',
- }
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: JSON.stringify(message),
- },
- ])
+ it('throws on completely invalid input', () => {
+ expect(() => parsePartialJSON('not json at all')).toThrow()
+ })
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: JSON.stringify(message),
- },
- ])
+ it.each([
+ ['bracket-prefixed text', '[Thinking: The user wants to build a todo app.]I will build it'],
+ ['bracket-prefixed with tool call', '[Tool Call: lov-write, Input: {"file_path":"src/index.tsx"}]'],
+ ])('returns empty array for %s (not actual JSON)', (_label, input) => {
+ // These strings start with "[" so they look like JSON arrays,
+ // but the partial parser can't extract any valid elements
+ const result = parsePartialJSON(input)
+ expect(result).toEqual([])
})
+ })
- it('parses OpenAI tool calls', () => {
- const message = {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: '123',
- function: {
- name: 'test',
- arguments: '{"foo": "bar"}',
- },
- },
- {
- type: 'function',
- id: '456',
- function: {
- name: 'test2',
- arguments: '{"bar": "baz"}',
- },
- },
- ],
- }
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: '123',
- function: {
- name: 'test',
- arguments: { foo: 'bar' },
- },
- },
- {
- type: 'function',
- id: '456',
- function: {
- name: 'test2',
- arguments: { bar: 'baz' },
- },
- },
- ],
- },
- ])
+ describe('looksLikeXml', () => {
+ it('detects basic XML structures', () => {
+ expect(looksLikeXml('<root><child/></root>')).toBe(true)
+ expect(looksLikeXml('<?xml version="1.0"?><root></root>')).toBe(true)
+ expect(looksLikeXml('<root attr="1">text</root>')).toBe(true)
+ expect(looksLikeXml('<self-closing/>')).toBe(true)
})
- it('parses OpenAI tool use', () => {
- const message = {
- role: 'tool',
- content: 'response',
- tool_call_id: '456',
- }
+ it('returns true for typical HTML snippets (by design)', () => {
+ expect(looksLikeXml('<!DOCTYPE html><html><body></body></html>')).toBe(true)
+ expect(looksLikeXml('<div class="x">hi</div>')).toBe(true)
+ })
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'tool',
- content: 'response',
- tool_call_id: '456',
- },
- ])
+ it('returns false for non-XML-like strings', () => {
+ expect(looksLikeXml('a < b and c > d')).toBe(false)
+ expect(looksLikeXml('plain text')).toBe(false)
+ expect(looksLikeXml('{"foo":"bar"}')).toBe(false)
})
+ })
- it('parses a string message', () => {
- expect(normalizeMessage('foo', 'user')).toEqual([
- {
- role: 'user',
- content: 'foo',
- },
- ])
+ describe('getInternalTagName / isInternalTagMessage', () => {
+ // The shape these tests pin: typed-parts content with a single text item
+ // whose entire body is a balanced internal tag wrapper.
+ const typedParts = (text: string): CompatMessage['content'] =>
+ [{ type: 'text', text }] as unknown as CompatMessage['content']
- expect(normalizeMessage('foo', 'assistant')).toEqual([
- {
- role: 'assistant',
- content: 'foo',
- },
- ])
+ it.each<[name: string, body: string, expected: string]>([
+ // String content
+ ['flat string with system-reminder (kebab)', '<system-reminder>foo</system-reminder>', 'system-reminder'],
+ ['flat string with system_reminder (snake)', '<system_reminder>foo</system_reminder>', 'system_reminder'],
+ [
+ 'flat string with system_reminder_message',
+ '<system_reminder_message>foo</system_reminder_message>',
+ 'system_reminder_message',
+ ],
+ [
+ 'flat string with attached_context',
+ '<attached_context>foo bar baz</attached_context>',
+ 'attached_context',
+ ],
+ ['flat string with voice_mode', '<voice_mode>off</voice_mode>', 'voice_mode'],
+ [
+ 'multi-line internal tag body',
+ '<system_reminder>\nyou are an agent\nmode: foo\n</system_reminder>',
+ 'system_reminder',
+ ],
+ ['leading/trailing whitespace tolerated', ' \n<voice_mode>off</voice_mode>\n ', 'voice_mode'],
+ ])('returns the tag name for: %s', (_, body, expected) => {
+ const message: CompatMessage = { role: 'user', content: body }
+ expect(getInternalTagName(message)).toBe(expected)
+ expect(isInternalTagMessage(message)).toBe(true)
+ })
+
+ it('matches the typed-parts shape (single text item)', () => {
+ const message: CompatMessage = {
+ role: 'user',
+ content: typedParts('<system_reminder>be concise</system_reminder>'),
+ }
+ expect(getInternalTagName(message)).toBe('system_reminder')
})
- it('parses an Anthropic tool call message', () => {
- let message: any = {
- type: 'tool_use',
- id: 'toolu_01D7FLrfh4GYq7yT1ULFeyMV',
- name: 'get_stock_price',
- input: { ticker: '^GSPC' },
+ it('matches the {type, content: string} wrapper shape (Vercel SDK legacy)', () => {
+ const message: CompatMessage = {
+ role: 'user',
+ content: {
+ type: 'text',
+ content: '<system-reminder>foo</system-reminder>',
+ } as unknown as CompatMessage['content'],
}
+ expect(getInternalTagName(message)).toBe('system-reminder')
+ })
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'toolu_01D7FLrfh4GYq7yT1ULFeyMV',
- function: {
- name: 'get_stock_price',
- arguments: { ticker: '^GSPC' },
- },
- },
- ],
- },
- ])
+ // ---- negative cases: role gate ----
- message = {
+ it('returns undefined for an assistant-role message even with an internal tag wrapper', () => {
+ // Models can legitimately emit `<system_reminder>` in their reply. Don't hide.
+ const message: CompatMessage = {
role: 'assistant',
- content: [
- {
- type: 'tool_use',
- id: 'toolu_01D7FLrfh4GYq7yT1ULFeyMV',
- name: 'get_stock_price',
- input: { ticker: '^GSPC' },
- },
- {
- type: 'tool_use',
- id: 'toolu_01D7FLrfh4GYq7yT1ULFeyMV2',
- name: 'get_stock_price',
- input: { ticker: '^GSPC' },
- },
- ],
+ content: '<system_reminder>foo</system_reminder>',
}
-
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'toolu_01D7FLrfh4GYq7yT1ULFeyMV',
- function: {
- name: 'get_stock_price',
- arguments: { ticker: '^GSPC' },
- },
- },
- ],
- },
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'toolu_01D7FLrfh4GYq7yT1ULFeyMV2',
- function: {
- name: 'get_stock_price',
- arguments: { ticker: '^GSPC' },
- },
- },
- ],
- },
- ])
+ expect(getInternalTagName(message)).toBeUndefined()
})
- it('prefers top-level tool_calls over content tool_use blocks', () => {
- // This is the format produced by LangChain's Anthropic callback
- // content has raw Anthropic format with empty input (streaming artifact)
- // tool_calls has the normalized OpenAI format with correct arguments
- const message = {
- role: 'assistant',
- content: [
- { type: 'text', text: 'Let me check that.' },
- {
- type: 'tool_use',
- id: 'toolu_123',
- name: 'get_weather',
- input: {}, // Empty - streaming artifact
- },
- ],
- tool_calls: [
- {
- type: 'function',
- id: 'toolu_123',
- function: {
- name: 'get_weather',
- arguments: '{"location": "San Francisco"}', // Correct arguments
- },
- },
- ],
+ it('returns undefined for a system-role message (system messages are filtered upstream anyway)', () => {
+ const message: CompatMessage = {
+ role: 'system',
+ content: '<system_reminder>foo</system_reminder>',
}
+ expect(getInternalTagName(message)).toBeUndefined()
+ })
- const result = normalizeMessage(message, 'assistant')
+ // ---- negative cases: allowlist gate ----
- // Should use the tool_calls array, not extract from content
- expect(result).toHaveLength(2)
- expect(result[0]).toEqual({
- role: 'assistant',
- content: 'Let me check that.',
- })
- expect(result[1]).toEqual({
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'toolu_123',
- function: {
- name: 'get_weather',
- arguments: { location: 'San Francisco' },
- },
- },
- ],
- })
+ it.each<[name: string, body: string]>([
+ ['unrelated tag <thinking> — model may emit as visible content', '<thinking>let me think</thinking>'],
+ ['unrelated tag <summary>', '<summary>the user wants X</summary>'],
+ ['unrelated tag <analysis>', '<analysis>foo</analysis>'],
+ ['unrelated tag <reasoning>', '<reasoning>foo</reasoning>'],
+ ['unrelated tag <answer>', '<answer>The capital of France is Paris.</answer>'],
+ ['unrelated tag not in allowlist', '<useful-context>foo</useful-context>'],
+ ['unrelated tag <relevance-scores>', '<relevance-scores>0.9</relevance-scores>'],
+ ['generic single-word <task>', '<task>foo</task>'],
+ ['HTML structural <pre>', '<pre>code block</pre>'],
+ ['HTML structural <code>', '<code>print("hi")</code>'],
+ ])('returns undefined for non-allowlisted tag: %s', (_, body) => {
+ const message: CompatMessage = { role: 'user', content: body }
+ expect(getInternalTagName(message)).toBeUndefined()
+ expect(isInternalTagMessage(message)).toBe(false)
})
- it('parses an Anthropic tool result message', () => {
- let message: AnthropicInputMessage = {
+ // ---- negative cases: shape gate ----
+
+ it('returns undefined when text content has leading text before the wrapper', () => {
+ // User wrote actual text alongside a wrapper-shaped substring. Don't hide.
+ const message: CompatMessage = {
role: 'user',
- content: [
- {
- type: 'tool_result',
- tool_use_id: '1',
- content: 'foo',
- },
- ],
+ content: 'foo <system_reminder>bar</system_reminder>',
}
+ expect(getInternalTagName(message)).toBeUndefined()
+ })
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant (tool result)',
- content: 'foo',
- tool_call_id: '1',
- },
- ])
-
- message = {
+ it('returns undefined when text content has trailing text after the wrapper', () => {
+ const message: CompatMessage = {
role: 'user',
- content: [
- {
- type: 'tool_result',
- tool_use_id: '1',
- content: [
- {
- type: 'text',
- text: 'foo',
- },
- ],
- },
- ],
+ content: '<system_reminder>foo</system_reminder> bar',
}
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant (tool result)',
- content: 'foo',
- tool_call_id: '1',
- },
- ])
+ expect(getInternalTagName(message)).toBeUndefined()
})
- })
- describe('OpenAI Responses API', () => {
- it('parses a function_call item as an assistant tool call', () => {
- const message = {
- type: 'function_call',
- call_id: 'call_tLji37A7lp7Buy7hcqUPMUXE',
- name: 'search_category',
- arguments: '{"dimensions":[],"list_all":true}',
+ it('returns undefined when content has two sibling wrappers (multi-block, not single)', () => {
+ const message: CompatMessage = {
+ role: 'user',
+ content: '<system_reminder>foo</system_reminder>\n<voice_mode>off</voice_mode>',
}
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'call_tLji37A7lp7Buy7hcqUPMUXE',
- function: {
- name: 'search_category',
- arguments: { dimensions: [], list_all: true },
- },
- },
- ],
- },
- ])
+ // Conservative — coalescing multi-wrapper bodies is out of scope.
+ expect(getInternalTagName(message)).toBeUndefined()
})
- it('parses a function_call_output item as a tool result', () => {
- const message = {
- type: 'function_call_output',
- call_id: 'call_tLji37A7lp7Buy7hcqUPMUXE',
- output: '{"categories":[{"id":"149288","name":"Asheville Cloud Shaker"}]}',
+ it('returns undefined for typed-parts with more than one item', () => {
+ // Two text items, even if each is itself an internal tag wrapper. The renderer
+ // would lose information if we collapsed this; keep it visible.
+ const message: CompatMessage = {
+ role: 'user',
+ content: [
+ { type: 'text', text: '<system_reminder>foo</system_reminder>' },
+ { type: 'text', text: '<voice_mode>off</voice_mode>' },
+ ] as unknown as CompatMessage['content'],
}
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant (tool result)',
- content: '{"categories":[{"id":"149288","name":"Asheville Cloud Shaker"}]}',
- tool_call_id: 'call_tLji37A7lp7Buy7hcqUPMUXE',
- },
- ])
+ expect(getInternalTagName(message)).toBeUndefined()
})
- it('parses a reasoning item as assistant thinking', () => {
- const message = {
- type: 'reasoning',
- id: 'rs_123',
- summary: [
- { type: 'summary_text', text: 'Thinking about the query...' },
- { type: 'summary_text', text: 'Deciding on a response.' },
- ],
+ it('returns undefined for typed-parts with a non-text item (image, tool_use, …)', () => {
+ const message: CompatMessage = {
+ role: 'user',
+ content: [{ type: 'image_url', image_url: { url: 'x' } }] as unknown as CompatMessage['content'],
}
+ expect(getInternalTagName(message)).toBeUndefined()
+ })
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant (thinking)',
- content: 'Thinking about the query...\nDeciding on a response.',
- },
- ])
+ // ---- negative cases: case-sensitivity + tag-mismatch guards ----
+
+ it('returns undefined for uppercase tag names (internal tags are lowercase by convention)', () => {
+ const message: CompatMessage = { role: 'user', content: '<System_Reminder>foo</System_Reminder>' }
+ expect(getInternalTagName(message)).toBeUndefined()
})
- it('handles reasoning with no summary', () => {
- const message = {
- type: 'reasoning',
- id: 'rs_456',
+ it('returns undefined when open and close tag names differ', () => {
+ // Malformed XML — the backref enforces consistency.
+ const message: CompatMessage = {
+ role: 'user',
+ content: '<system_reminder>foo</voice_mode>',
}
+ expect(getInternalTagName(message)).toBeUndefined()
+ })
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant (thinking)',
- content: '',
- },
- ])
+ it('returns undefined for empty content', () => {
+ expect(getInternalTagName({ role: 'user', content: '' })).toBeUndefined()
+ expect(getInternalTagName({ role: 'user', content: [] })).toBeUndefined()
+ expect(getInternalTagName({ role: 'user', content: null as unknown as string })).toBeUndefined()
})
- it('handles Vercel AI SDK reasoning with text field', () => {
- const message = {
- type: 'reasoning',
- text: 'Let me think about this step by step.',
+ it('handles typed-parts content with a multi-line attached_context wrapper', () => {
+ const message: CompatMessage = {
+ role: 'user',
+ content: typedParts('<attached_context>\nfoo\n\nbar: baz\nqux: 123\n</attached_context>'),
}
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant (thinking)',
- content: 'Let me think about this step by step.',
- },
- ])
+ expect(getInternalTagName(message)).toBe('attached_context')
})
+ })
- it.each([
+ describe('isToolResult', () => {
+ it.each<[name: string, item: unknown]>([
+ ['Anthropic typed tool_result', { type: 'tool_result', tool_use_id: 'toolu_1', content: 'ok' }],
[
- 'string arguments (posthog-node format)',
- {
- type: 'tool-call',
- id: 'toolu_vrtx_01CTHfH7tfd2vNWSWJhKzfov',
- function: { name: 'queryModel', arguments: '{"query": "relevant data"}' },
- },
- 'toolu_vrtx_01CTHfH7tfd2vNWSWJhKzfov',
- 'queryModel',
- { query: 'relevant data' },
+ 'Vercel SDK tool-result',
+ { type: 'tool-result', toolCallId: 'a', toolName: 'search_docs', output: { ok: true } },
],
[
- 'object arguments (posthog-node format)',
- {
- type: 'tool-call',
- id: 'call_123',
- function: { name: 'get_weather', arguments: { location: 'NYC' } },
- },
- 'call_123',
- 'get_weather',
- { location: 'NYC' },
+ 'OpenAI Responses function_call_output',
+ { type: 'function_call_output', call_id: 'c1', output: 'opaque' },
],
[
- 'native Vercel SDK format (toolName/input/toolCallId)',
- { type: 'tool-call', toolCallId: 'tc_native_123', toolName: 'get_weather', input: { location: 'NYC' } },
- 'tc_native_123',
- 'get_weather',
- { location: 'NYC' },
+ 'custom {type:"function", tool_name, content}',
+ { type: 'function', tool_name: 'lookup', content: 'opaque' },
],
- ])('handles Vercel AI SDK tool-call with %s', (_label, message, expectedId, expectedName, expectedArgs) => {
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: expectedId,
- function: { name: expectedName, arguments: expectedArgs },
- },
- ],
- },
- ])
- })
-
- it.each([
- [
- 'object output',
- {
- type: 'tool-result',
- toolCallId: 'tc_123',
- toolName: 'get_weather',
- output: { temperature: 72, unit: 'F' },
- },
- '{"temperature":72,"unit":"F"}',
- 'tc_123',
- ],
- [
- 'string output',
- { type: 'tool-result', toolCallId: 'tc_456', toolName: 'search', output: 'Found 3 results' },
- 'Found 3 results',
- 'tc_456',
- ],
- ])('handles Vercel AI SDK tool-result with %s', (_label, message, expectedContent, expectedToolCallId) => {
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant (tool result)',
- content: expectedContent,
- tool_call_id: expectedToolCallId,
- },
- ])
- })
-
- it('handles Vercel AI SDK toolName variant with string input', () => {
- const message = { type: 'tool-call', toolCallId: 'tc_1', toolName: 'run_code', input: 'print("hi")' }
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'tc_1',
- function: { name: 'run_code', arguments: 'print("hi")' },
- },
- ],
- },
- ])
- })
-
- it('handles Vercel AI SDK toolName variant with missing input', () => {
- const message = { type: 'tool-call', toolCallId: 'tc_2', toolName: 'no_args_tool' }
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'tc_2',
- function: { name: 'no_args_tool', arguments: {} },
- },
- ],
- },
- ])
- })
-
- it('prefers function variant when both function and toolName are present', () => {
- const message = {
- type: 'tool-call',
- id: 'func_id',
- function: { name: 'from_function', arguments: '{"a":1}' },
- toolCallId: 'tool_id',
- toolName: 'from_toolName',
- input: { b: 2 },
- }
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'func_id',
- function: { name: 'from_function', arguments: { a: 1 } },
- },
- ],
- },
- ])
- })
-
- it('handles Vercel AI SDK tool-result with undefined output', () => {
- const message = { type: 'tool-result', toolCallId: 'tc_empty', toolName: 'void_tool' }
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant (tool result)',
- content: '',
- tool_call_id: 'tc_empty',
- },
- ])
- })
-
- it('handles Vercel AI SDK tool-result with non-serializable output', () => {
- const circular: Record<string, unknown> = { key: 'value' }
- circular.self = circular
- const message = { type: 'tool-result', toolCallId: 'tc_circ', toolName: 'bad_tool', output: circular }
- const result = normalizeMessage(message, 'assistant')
- expect(result).toEqual([
- {
- role: 'assistant (tool result)',
- content: '[object Object]',
- tool_call_id: 'tc_circ',
- },
- ])
- })
-
- it('handles Vercel AI SDK mixed content array with reasoning + text + tool-call', () => {
- const message = {
- role: 'assistant',
- content: [
- { type: 'reasoning', text: 'I should use a tool to look up the data.' },
- { type: 'text', text: 'Let me check that for you.' },
- {
- type: 'tool-call',
- id: 'toolu_vrtx_abc',
- function: { name: 'queryModel', arguments: '{"query":"data"}' },
- },
- ],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant (thinking)',
- content: 'I should use a tool to look up the data.',
- },
- {
- role: 'assistant',
- content: 'Let me check that for you.',
- },
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'toolu_vrtx_abc',
- function: { name: 'queryModel', arguments: { query: 'data' } },
- },
- ],
- },
- ])
+ ])('returns true for: %s', (_, item) => {
+ expect(isToolResult(item)).toBe(true)
})
- it.each([
- [
- 'web_search_call',
- 'ws_123',
- 'web_search_call',
- { action: { type: 'search', query: 'weather today' } },
- {
- type: 'web_search_call',
- id: 'ws_123',
- status: 'completed',
- action: { type: 'search', query: 'weather today' },
- },
- ],
- [
- 'code_interpreter_call',
- 'ci_123',
- 'code_interpreter_call',
- { code: 'print("hello")' },
- { type: 'code_interpreter_call', id: 'ci_123', status: 'completed', code: 'print("hello")' },
- ],
- [
- 'image_generation_call',
- 'ig_123',
- 'image_generation_call',
- { prompt: 'a red cat' },
- { type: 'image_generation_call', id: 'ig_123', status: 'completed', prompt: 'a red cat' },
- ],
- [
- 'mcp_call',
- 'mcp_123',
- 'search_docs',
- { query: 'pricing' },
- {
- type: 'mcp_call',
- id: 'mcp_123',
- status: 'completed',
- name: 'search_docs',
- server_label: 'docs',
- arguments: '{"query":"pricing"}',
- },
- ],
- [
- 'file_search_call',
- 'fs_123',
- 'file_search_call',
- { queries: ['q1', 'q2'] },
- { type: 'file_search_call', id: 'fs_123', status: 'completed', queries: ['q1', 'q2'] },
- ],
+ it.each<[name: string, item: unknown]>([
+ ['plain text part', { type: 'text', text: 'hi' }],
+ ['Anthropic tool_use (a tool CALL, not a result)', { type: 'tool_use', id: 't1', name: 'x', input: {} }],
[
- 'computer_call',
- 'cc_123',
- 'computer_call',
- { action: { type: 'click', x: 10, y: 20 } },
- {
- type: 'computer_call',
- id: 'cc_123',
- status: 'completed',
- action: { type: 'click', x: 10, y: 20 },
- },
+ 'OpenAI tool CALL with nested function object',
+ { type: 'function', function: { name: 'get_weather', arguments: '{}' } },
],
- ])(
- 'parses %s as an assistant tool call preserving metadata',
- (_toolType, toolId, expectedName, expectedArguments, message) => {
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: toolId,
- function: { name: expectedName, arguments: expectedArguments },
- },
- ],
- },
- ])
- }
- )
-
- it('handles function_call with unparseable arguments', () => {
- const message = {
- type: 'function_call',
- call_id: 'call_abc',
- name: 'my_func',
- arguments: '{invalid json',
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'call_abc',
- function: {
- name: 'my_func',
- arguments: '{invalid json',
- },
- },
- ],
- },
- ])
+ ['Vercel SDK tool-call', { type: 'tool-call', toolCallId: 'a', toolName: 'x', input: {} }],
+ ['image part', { type: 'image_url', image_url: { url: 'x' } }],
+ ['null', null],
+ ['plain string', 'hello'],
+ ['function-typed item without tool_name', { type: 'function' }],
+ ])('returns false for: %s', (_, item) => {
+ expect(isToolResult(item)).toBe(false)
})
})
- it('normalizeMessage: lifts function items out of array-based content into tool_calls', () => {
- const message = {
- role: 'assistant',
- content: [
- {
- type: 'text',
- text: "I'll check the weather for you.",
- },
- {
- type: 'function',
- id: 'call_123',
- function: {
- name: 'get_weather',
- arguments: { location: 'New York City' },
- },
- },
- ],
- }
-
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: [
- {
- type: 'text',
- text: "I'll check the weather for you.",
- },
- ],
- tool_calls: [
- {
- type: 'function',
- id: 'call_123',
- function: {
- name: 'get_weather',
- arguments: { location: 'New York City' },
- },
- },
- ],
- },
- ])
- })
-
- describe('normalizeMessage: array-based content with function items', () => {
- it('leaves content empty when every item is a function', () => {
- const message = {
- role: 'assistant',
+ describe('isInternalToolResultUserMessage', () => {
+ it('returns true for a user message whose content is only typed tool_result parts', () => {
+ const message: CompatMessage = {
+ role: 'user',
content: [
- {
- type: 'function',
- id: 'call_1',
- function: { name: 'get_weather', arguments: { location: 'Berlin' } },
- },
- {
- type: 'function',
- id: 'call_2',
- function: { name: 'search_docs', arguments: { query: 'foo' } },
- },
- ],
+ { type: 'tool_result', tool_use_id: 't1', content: 'value-1' },
+ { type: 'tool_result', tool_use_id: 't2', content: 'value-2' },
+ ] as unknown as CompatMessage['content'],
}
-
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'call_1',
- function: { name: 'get_weather', arguments: { location: 'Berlin' } },
- },
- {
- type: 'function',
- id: 'call_2',
- function: { name: 'search_docs', arguments: { query: 'foo' } },
- },
- ],
- },
- ])
+ expect(isInternalToolResultUserMessage(message)).toBe(true)
})
- it.each([
- ['pre-parsed object', { location: 'Berlin', unit: 'celsius' }, { location: 'Berlin', unit: 'celsius' }],
- ['JSON-encoded string', '{"location":"Berlin","unit":"celsius"}', { location: 'Berlin', unit: 'celsius' }],
- ])('parses function arguments provided as %s', (_label, rawArguments, expectedArguments) => {
- const message = {
- role: 'assistant',
+ it('returns true for the custom `{type:"function", tool_name, content}` shape', () => {
+ const message: CompatMessage = {
+ role: 'user',
content: [
- {
- type: 'function',
- id: 'call_1',
- function: { name: 'get_weather', arguments: rawArguments },
- },
- ],
+ { type: 'function', tool_name: 'lookup', content: 'opaque' },
+ ] as unknown as CompatMessage['content'],
}
-
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'call_1',
- function: { name: 'get_weather', arguments: expectedArguments },
- },
- ],
- },
- ])
+ expect(isInternalToolResultUserMessage(message)).toBe(true)
})
- it('lifts function items into tool_calls and keeps non-function items as content', () => {
- const message = {
- role: 'assistant',
+ it('returns false when a tool-result part sits alongside real user text', () => {
+ // Hiding would drop the user's prose.
+ const message: CompatMessage = {
+ role: 'user',
content: [
- {
- type: 'function',
- id: 'fs_1',
- function: { name: 'file_search', arguments: { queries: ['q'] } },
- },
- { type: 'text', text: 'hi' },
- ],
+ { type: 'tool_result', tool_use_id: 't1', content: 'value' },
+ { type: 'text', text: 'and please summarize the above' },
+ ] as unknown as CompatMessage['content'],
}
-
- expect(normalizeMessage(message, 'assistant')).toEqual([
- {
- role: 'assistant',
- content: [{ type: 'text', text: 'hi' }],
- tool_calls: [
- {
- type: 'function',
- id: 'fs_1',
- function: { name: 'file_search', arguments: { queries: ['q'] } },
- },
- ],
- },
- ])
+ expect(isInternalToolResultUserMessage(message)).toBe(false)
})
- })
-
- describe('normalizeMessage: built-in tool call with explicit name', () => {
- it('uses rawMessage.name for mcp_call, not the "mcp_call" type string', () => {
- const message = {
- type: 'mcp_call',
- id: 'mcp_123',
- name: 'search_docs',
- server_label: 'docs',
- arguments: '{"query":"pricing"}',
- }
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'mcp_123',
- function: { name: 'search_docs', arguments: { query: 'pricing' } },
- },
- ],
- },
- ])
+ it('returns false for an empty content array', () => {
+ expect(isInternalToolResultUserMessage({ role: 'user', content: [] })).toBe(false)
})
- })
- describe('role preservation in nested content', () => {
- it('preserves assistant role when recursing into nested array content with output_text', () => {
- // This is the bug we fixed - messages with role + array content should preserve the role
- // Now output_text is properly recognized and content is preserved as array
- const message = {
+ it('returns false for an assistant-role message even if its content is tool-result-shaped', () => {
+ const message: CompatMessage = {
role: 'assistant',
content: [
- {
- type: 'output_text',
- text: 'Hello! How can I help you?',
- },
- ],
+ { type: 'tool_result', tool_use_id: 't1', content: 'value' },
+ ] as unknown as CompatMessage['content'],
}
-
- const result = normalizeMessage(message, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('assistant') // Role is preserved!
- expect(result[0].content).toEqual([
- {
- type: 'output_text',
- text: 'Hello! How can I help you?',
- },
- ])
+ expect(isInternalToolResultUserMessage(message)).toBe(false)
})
- it('preserves system role when recursing into nested array content with output_text', () => {
- const message = {
- role: 'system',
- content: [
- {
- type: 'output_text',
- text: 'You are a helpful assistant.',
- },
- ],
- }
-
- const result = normalizeMessage(message, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('system') // Role is preserved!
- expect(result[0].content).toEqual([
- {
- type: 'output_text',
- text: 'You are a helpful assistant.',
- },
- ])
+ it('returns false for a flat-string user message (text body, not a parts list)', () => {
+ expect(isInternalToolResultUserMessage({ role: 'user', content: 'follow-up question' })).toBe(false)
})
+ })
- it('preserves role even when defaultRole is different', () => {
- const assistantMessage = {
- role: 'assistant',
- content: [{ type: 'output_text', text: 'Response' }],
+ describe('formatLLMEventTitle', () => {
+ it('formats LLMTrace with traceName', () => {
+ const trace: LLMTrace = {
+ id: 'trace-1',
+ createdAt: '2024-01-01T00:00:00Z',
+ distinctId: 'user-1',
+ traceName: 'My Custom Trace',
+ person: {
+ uuid: 'person-1',
+ created_at: '2024-01-01T00:00:00Z',
+ properties: {},
+ distinct_id: 'user-1',
+ },
+ events: [],
}
-
- // Even though defaultRole is 'user', the actual role 'assistant' should be preserved
- const result = normalizeMessage(assistantMessage, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('assistant') // Key test: role is preserved despite different defaultRole
- expect(result[0].content).toEqual([{ type: 'output_text', text: 'Response' }])
+ expect(formatLLMEventTitle(trace)).toBe('My Custom Trace')
})
- it('preserves input_text content type', () => {
- const message = {
- role: 'user',
- content: [{ type: 'input_text', text: 'What is the weather?' }],
+ it('formats LLMTrace without traceName', () => {
+ const trace: LLMTrace = {
+ id: 'trace-1',
+ createdAt: '2024-01-01T00:00:00Z',
+ distinctId: 'user-1',
+ person: {
+ uuid: 'person-1',
+ created_at: '2024-01-01T00:00:00Z',
+ properties: {},
+ distinct_id: 'user-1',
+ },
+ events: [],
}
-
- const result = normalizeMessage(message, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('user')
- expect(result[0].content).toEqual([{ type: 'input_text', text: 'What is the weather?' }])
+ expect(formatLLMEventTitle(trace)).toBe('Trace')
})
- it('uses defaultRole when message has no role property', () => {
- const messageWithoutRole = {
- type: 'output_text',
- text: 'Some text',
- }
-
- const result = normalizeMessage(messageWithoutRole, 'assistant')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('assistant')
- // Without a role wrapper, output_text falls through to unsupported and gets stringified
- expect(result[0].content).toBe('{"type":"output_text","text":"Some text"}')
- })
+ describe('$ai_generation events', () => {
+ it('formats generation event with span name', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_generation',
+ properties: {
+ $ai_span_name: 'Chat Completion',
+ $ai_model: 'gpt-4',
+ $ai_provider: 'openai',
+ },
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('Chat Completion')
+ })
- it('handles Anthropic tool result with nested content and overrides role to assistant (tool result)', () => {
- const toolResultMessage = {
- type: 'tool_result',
- tool_use_id: 'tool_123',
- content: [
- {
- type: 'text',
- text: 'Weather is sunny',
+ it('formats generation event with model and provider', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_generation',
+ properties: {
+ $ai_model: 'gpt-4',
+ $ai_provider: 'openai',
},
- ],
- }
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('gpt-4 (openai)')
+ })
- const result = normalizeMessage(toolResultMessage, 'tool')
+ it('formats generation event with model only', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_generation',
+ properties: {
+ $ai_model: 'gpt-4',
+ },
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('gpt-4')
+ })
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('assistant (tool result)')
- expect(result[0].content).toBe('Weather is sunny')
- expect(result[0].tool_call_id).toBe('tool_123')
- })
-
- it('preserves custom/unknown roles', () => {
- const customRoleMessage = {
- role: 'custom_agent',
- content: [{ type: 'text', text: 'Custom response' }],
- }
-
- const result = normalizeMessage(customRoleMessage, 'user')
-
- // Unknown roles should be preserved as-is (lowercased)
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('custom_agent')
- expect(result[0].content).toEqual([{ type: 'text', text: 'Custom response' }])
- })
-
- it.each([
- [
- 'single text block',
- {
- role: 'user',
- content: JSON.stringify([
- { type: 'text', text: 'lets respond inline to each bot to explain that its fine' },
- ]),
- },
- [{ type: 'text', text: 'lets respond inline to each bot to explain that its fine' }],
- ],
- ['plain string stays plain', { role: 'user', content: 'plain text' }, 'plain text'],
- ['invalid JSON falls back to raw string', { role: 'user', content: '[not json' }, '[not json'],
- [
- 'unknown block types stay as raw string',
- { role: 'user', content: JSON.stringify([{ type: 'schema', definition: { foo: 'bar' } }]) },
- '[{"type":"schema","definition":{"foo":"bar"}}]',
- ],
- ['empty structured array parses as empty array', { role: 'user', content: '[]' }, []],
- ])(
- 'handles stringified structured content in OpenAI-compatible messages: %s',
- (_label, message, expectedContent) => {
- const result = normalizeMessage(message, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('user')
- expect(result[0].content).toEqual(expectedContent)
- }
- )
-
- it('handles LiteLLM choice wrapper and preserves nested role', () => {
- const liteLLMChoice = {
- finish_reason: 'stop',
- index: 0,
- message: {
- role: 'assistant',
- content: [{ type: 'text', text: 'Response from LiteLLM' }],
- },
- }
-
- const result = normalizeMessage(liteLLMChoice, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('assistant')
- expect(result[0].content).toEqual([{ type: 'text', text: 'Response from LiteLLM' }])
- })
- })
-
- describe('parsePartialJSON', () => {
- it.each([
- ['full object', '{"key": "value", "n": 42}', { key: 'value', n: 42 }],
- ['full array', '[1, 2, 3]', [1, 2, 3]],
- ['truncated object', '{"key": "value", "long_field": "some te', { key: 'value', long_field: 'some te' }],
- ['truncated nested object', '{"a": {"b": "c", "d": "efg', { a: { b: 'c', d: 'efg' } }],
- ['truncated array', '[1, 2, "hel', [1, 2, 'hel']],
- ['truncated string value', '"hello wor', 'hello wor'],
- ])('%s', (_label, input, expected) => {
- expect(parsePartialJSON(input)).toEqual(expected)
- })
-
- it('throws on completely invalid input', () => {
- expect(() => parsePartialJSON('not json at all')).toThrow()
- })
-
- it.each([
- ['bracket-prefixed text', '[Thinking: The user wants to build a todo app.]I will build it'],
- ['bracket-prefixed with tool call', '[Tool Call: lov-write, Input: {"file_path":"src/index.tsx"}]'],
- ])('returns empty array for %s (not actual JSON)', (_label, input) => {
- // These strings start with "[" so they look like JSON arrays,
- // but the partial parser can't extract any valid elements
- const result = parsePartialJSON(input)
- expect(result).toEqual([])
- })
- })
-
- describe('looksLikeXml', () => {
- it('detects basic XML structures', () => {
- expect(looksLikeXml('<root><child/></root>')).toBe(true)
- expect(looksLikeXml('<?xml version="1.0"?><root></root>')).toBe(true)
- expect(looksLikeXml('<root attr="1">text</root>')).toBe(true)
- expect(looksLikeXml('<self-closing/>')).toBe(true)
- })
-
- it('returns true for typical HTML snippets (by design)', () => {
- expect(looksLikeXml('<!DOCTYPE html><html><body></body></html>')).toBe(true)
- expect(looksLikeXml('<div class="x">hi</div>')).toBe(true)
- })
-
- it('returns false for non-XML-like strings', () => {
- expect(looksLikeXml('a < b and c > d')).toBe(false)
- expect(looksLikeXml('plain text')).toBe(false)
- expect(looksLikeXml('{"foo":"bar"}')).toBe(false)
- })
- })
-
- describe('getInternalTagName / isInternalTagMessage', () => {
- // The shape these tests pin: typed-parts content with a single text item
- // whose entire body is a balanced internal tag wrapper.
- const typedParts = (text: string): CompatMessage['content'] =>
- [{ type: 'text', text }] as unknown as CompatMessage['content']
-
- it.each<[name: string, body: string, expected: string]>([
- // String content
- ['flat string with system-reminder (kebab)', '<system-reminder>foo</system-reminder>', 'system-reminder'],
- ['flat string with system_reminder (snake)', '<system_reminder>foo</system_reminder>', 'system_reminder'],
- [
- 'flat string with system_reminder_message',
- '<system_reminder_message>foo</system_reminder_message>',
- 'system_reminder_message',
- ],
- [
- 'flat string with attached_context',
- '<attached_context>foo bar baz</attached_context>',
- 'attached_context',
- ],
- ['flat string with voice_mode', '<voice_mode>off</voice_mode>', 'voice_mode'],
- [
- 'multi-line internal tag body',
- '<system_reminder>\nyou are an agent\nmode: foo\n</system_reminder>',
- 'system_reminder',
- ],
- ['leading/trailing whitespace tolerated', ' \n<voice_mode>off</voice_mode>\n ', 'voice_mode'],
- ])('returns the tag name for: %s', (_, body, expected) => {
- const message: CompatMessage = { role: 'user', content: body }
- expect(getInternalTagName(message)).toBe(expected)
- expect(isInternalTagMessage(message)).toBe(true)
- })
-
- it('matches the typed-parts shape (single text item)', () => {
- const message: CompatMessage = {
- role: 'user',
- content: typedParts('<system_reminder>be concise</system_reminder>'),
- }
- expect(getInternalTagName(message)).toBe('system_reminder')
- })
-
- it('matches the {type, content: string} wrapper shape (Vercel SDK legacy)', () => {
- const message: CompatMessage = {
- role: 'user',
- content: {
- type: 'text',
- content: '<system-reminder>foo</system-reminder>',
- } as unknown as CompatMessage['content'],
- }
- expect(getInternalTagName(message)).toBe('system-reminder')
- })
-
- // ---- negative cases: role gate ----
-
- it('returns undefined for an assistant-role message even with an internal tag wrapper', () => {
- // Models can legitimately emit `<system_reminder>` in their reply. Don't hide.
- const message: CompatMessage = {
- role: 'assistant',
- content: '<system_reminder>foo</system_reminder>',
- }
- expect(getInternalTagName(message)).toBeUndefined()
- })
-
- it('returns undefined for a system-role message (system messages are filtered upstream anyway)', () => {
- const message: CompatMessage = {
- role: 'system',
- content: '<system_reminder>foo</system_reminder>',
- }
- expect(getInternalTagName(message)).toBeUndefined()
- })
-
- // ---- negative cases: allowlist gate ----
-
- it.each<[name: string, body: string]>([
- ['unrelated tag <thinking> — model may emit as visible content', '<thinking>let me think</thinking>'],
- ['unrelated tag <summary>', '<summary>the user wants X</summary>'],
- ['unrelated tag <analysis>', '<analysis>foo</analysis>'],
- ['unrelated tag <reasoning>', '<reasoning>foo</reasoning>'],
- ['unrelated tag <answer>', '<answer>The capital of France is Paris.</answer>'],
- ['unrelated tag not in allowlist', '<useful-context>foo</useful-context>'],
- ['unrelated tag <relevance-scores>', '<relevance-scores>0.9</relevance-scores>'],
- ['generic single-word <task>', '<task>foo</task>'],
- ['HTML structural <pre>', '<pre>code block</pre>'],
- ['HTML structural <code>', '<code>print("hi")</code>'],
- ])('returns undefined for non-allowlisted tag: %s', (_, body) => {
- const message: CompatMessage = { role: 'user', content: body }
- expect(getInternalTagName(message)).toBeUndefined()
- expect(isInternalTagMessage(message)).toBe(false)
- })
-
- // ---- negative cases: shape gate ----
-
- it('returns undefined when text content has leading text before the wrapper', () => {
- // User wrote actual text alongside a wrapper-shaped substring. Don't hide.
- const message: CompatMessage = {
- role: 'user',
- content: 'foo <system_reminder>bar</system_reminder>',
- }
- expect(getInternalTagName(message)).toBeUndefined()
- })
-
- it('returns undefined when text content has trailing text after the wrapper', () => {
- const message: CompatMessage = {
- role: 'user',
- content: '<system_reminder>foo</system_reminder> bar',
- }
- expect(getInternalTagName(message)).toBeUndefined()
- })
-
- it('returns undefined when content has two sibling wrappers (multi-block, not single)', () => {
- const message: CompatMessage = {
- role: 'user',
- content: '<system_reminder>foo</system_reminder>\n<voice_mode>off</voice_mode>',
- }
- // Conservative — coalescing multi-wrapper bodies is out of scope.
- expect(getInternalTagName(message)).toBeUndefined()
- })
-
- it('returns undefined for typed-parts with more than one item', () => {
- // Two text items, even if each is itself an internal tag wrapper. The renderer
- // would lose information if we collapsed this; keep it visible.
- const message: CompatMessage = {
- role: 'user',
- content: [
- { type: 'text', text: '<system_reminder>foo</system_reminder>' },
- { type: 'text', text: '<voice_mode>off</voice_mode>' },
- ] as unknown as CompatMessage['content'],
- }
- expect(getInternalTagName(message)).toBeUndefined()
- })
-
- it('returns undefined for typed-parts with a non-text item (image, tool_use, …)', () => {
- const message: CompatMessage = {
- role: 'user',
- content: [{ type: 'image_url', image_url: { url: 'x' } }] as unknown as CompatMessage['content'],
- }
- expect(getInternalTagName(message)).toBeUndefined()
- })
-
- // ---- negative cases: case-sensitivity + tag-mismatch guards ----
-
- it('returns undefined for uppercase tag names (internal tags are lowercase by convention)', () => {
- const message: CompatMessage = { role: 'user', content: '<System_Reminder>foo</System_Reminder>' }
- expect(getInternalTagName(message)).toBeUndefined()
- })
-
- it('returns undefined when open and close tag names differ', () => {
- // Malformed XML — the backref enforces consistency.
- const message: CompatMessage = {
- role: 'user',
- content: '<system_reminder>foo</voice_mode>',
- }
- expect(getInternalTagName(message)).toBeUndefined()
- })
-
- it('returns undefined for empty content', () => {
- expect(getInternalTagName({ role: 'user', content: '' })).toBeUndefined()
- expect(getInternalTagName({ role: 'user', content: [] })).toBeUndefined()
- expect(getInternalTagName({ role: 'user', content: null as unknown as string })).toBeUndefined()
- })
-
- it('handles typed-parts content with a multi-line attached_context wrapper', () => {
- const message: CompatMessage = {
- role: 'user',
- content: typedParts('<attached_context>\nfoo\n\nbar: baz\nqux: 123\n</attached_context>'),
- }
- expect(getInternalTagName(message)).toBe('attached_context')
- })
- })
-
- describe('isToolResult', () => {
- it.each<[name: string, item: unknown]>([
- ['Anthropic typed tool_result', { type: 'tool_result', tool_use_id: 'toolu_1', content: 'ok' }],
- [
- 'Vercel SDK tool-result',
- { type: 'tool-result', toolCallId: 'a', toolName: 'search_docs', output: { ok: true } },
- ],
- [
- 'OpenAI Responses function_call_output',
- { type: 'function_call_output', call_id: 'c1', output: 'opaque' },
- ],
- [
- 'custom {type:"function", tool_name, content}',
- { type: 'function', tool_name: 'lookup', content: 'opaque' },
- ],
- ])('returns true for: %s', (_, item) => {
- expect(isToolResult(item)).toBe(true)
- })
-
- it.each<[name: string, item: unknown]>([
- ['plain text part', { type: 'text', text: 'hi' }],
- ['Anthropic tool_use (a tool CALL, not a result)', { type: 'tool_use', id: 't1', name: 'x', input: {} }],
- [
- 'OpenAI tool CALL with nested function object',
- { type: 'function', function: { name: 'get_weather', arguments: '{}' } },
- ],
- ['Vercel SDK tool-call', { type: 'tool-call', toolCallId: 'a', toolName: 'x', input: {} }],
- ['image part', { type: 'image_url', image_url: { url: 'x' } }],
- ['null', null],
- ['plain string', 'hello'],
- ['function-typed item without tool_name', { type: 'function' }],
- ])('returns false for: %s', (_, item) => {
- expect(isToolResult(item)).toBe(false)
- })
- })
-
- describe('isInternalToolResultUserMessage', () => {
- it('returns true for a user message whose content is only typed tool_result parts', () => {
- const message: CompatMessage = {
- role: 'user',
- content: [
- { type: 'tool_result', tool_use_id: 't1', content: 'value-1' },
- { type: 'tool_result', tool_use_id: 't2', content: 'value-2' },
- ] as unknown as CompatMessage['content'],
- }
- expect(isInternalToolResultUserMessage(message)).toBe(true)
- })
-
- it('returns true for the custom `{type:"function", tool_name, content}` shape', () => {
- const message: CompatMessage = {
- role: 'user',
- content: [
- { type: 'function', tool_name: 'lookup', content: 'opaque' },
- ] as unknown as CompatMessage['content'],
- }
- expect(isInternalToolResultUserMessage(message)).toBe(true)
- })
-
- it('returns false when a tool-result part sits alongside real user text', () => {
- // Hiding would drop the user's prose.
- const message: CompatMessage = {
- role: 'user',
- content: [
- { type: 'tool_result', tool_use_id: 't1', content: 'value' },
- { type: 'text', text: 'and please summarize the above' },
- ] as unknown as CompatMessage['content'],
- }
- expect(isInternalToolResultUserMessage(message)).toBe(false)
- })
-
- it('returns false for an empty content array', () => {
- expect(isInternalToolResultUserMessage({ role: 'user', content: [] })).toBe(false)
- })
-
- it('returns false for an assistant-role message even if its content is tool-result-shaped', () => {
- const message: CompatMessage = {
- role: 'assistant',
- content: [
- { type: 'tool_result', tool_use_id: 't1', content: 'value' },
- ] as unknown as CompatMessage['content'],
- }
- expect(isInternalToolResultUserMessage(message)).toBe(false)
- })
-
- it('returns false for a flat-string user message (text body, not a parts list)', () => {
- expect(isInternalToolResultUserMessage({ role: 'user', content: 'follow-up question' })).toBe(false)
- })
- })
-
- describe('formatLLMEventTitle', () => {
- it('formats LLMTrace with traceName', () => {
- const trace: LLMTrace = {
- id: 'trace-1',
- createdAt: '2024-01-01T00:00:00Z',
- distinctId: 'user-1',
- traceName: 'My Custom Trace',
- person: {
- uuid: 'person-1',
- created_at: '2024-01-01T00:00:00Z',
- properties: {},
- distinct_id: 'user-1',
- },
- events: [],
- }
- expect(formatLLMEventTitle(trace)).toBe('My Custom Trace')
- })
-
- it('formats LLMTrace without traceName', () => {
- const trace: LLMTrace = {
- id: 'trace-1',
- createdAt: '2024-01-01T00:00:00Z',
- distinctId: 'user-1',
- person: {
- uuid: 'person-1',
- created_at: '2024-01-01T00:00:00Z',
- properties: {},
- distinct_id: 'user-1',
- },
- events: [],
- }
- expect(formatLLMEventTitle(trace)).toBe('Trace')
- })
-
- describe('$ai_generation events', () => {
- it('formats generation event with span name', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_generation',
- properties: {
- $ai_span_name: 'Chat Completion',
- $ai_model: 'gpt-4',
- $ai_provider: 'openai',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('Chat Completion')
- })
-
- it('formats generation event with model and provider', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_generation',
- properties: {
- $ai_model: 'gpt-4',
- $ai_provider: 'openai',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('gpt-4 (openai)')
- })
-
- it('formats generation event with model only', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_generation',
- properties: {
- $ai_model: 'gpt-4',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('gpt-4')
- })
-
- it('formats generation event without model or provider', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_generation',
- properties: {},
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('Generation')
- })
- })
-
- describe('$ai_embedding events', () => {
- it('formats embedding event with span name', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_embedding',
- properties: {
- $ai_span_name: 'Document Embedding',
- $ai_model: 'text-embedding-3-small',
- $ai_provider: 'openai',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('Document Embedding')
- })
-
- it('formats embedding event with model and provider', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_embedding',
- properties: {
- $ai_model: 'text-embedding-3-small',
- $ai_provider: 'openai',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('text-embedding-3-small (openai)')
- })
-
- it('formats embedding event with model only', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_embedding',
- properties: {
- $ai_model: 'text-embedding-3-small',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('text-embedding-3-small')
- })
-
- it('formats embedding event without model or provider', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_embedding',
- properties: {},
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('Embedding')
- })
- })
-
- describe('$ai_span and other events', () => {
- it('formats span event with span name', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_span',
- properties: {
- $ai_span_name: 'Database Query',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('Database Query')
- })
-
- it('formats span event without span name', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_span',
- properties: {},
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('Span')
- })
-
- it('formats metric event', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_metric',
- properties: {
- $ai_span_name: 'Performance Metric',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('Performance Metric')
- })
-
- it('formats feedback event', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: '$ai_feedback',
- properties: {
- $ai_span_name: 'User Feedback',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('User Feedback')
- })
-
- it('formats unknown event type', () => {
- const event: LLMTraceEvent = {
- id: 'event-1',
- event: 'custom_event',
- properties: {
- $ai_span_name: 'Custom Action',
- },
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatLLMEventTitle(event)).toBe('Custom Action')
- })
- })
-
- describe('non-string properties (untyped property bag)', () => {
- // `$ai_*` title properties are strings by convention, but the event property bag is
- // untyped. A structured value (object/array) or number must never be returned verbatim,
- // or it would be rendered as a raw React child and crash the page (React error #31).
- const cases: Array<[string, string, Record<string, unknown>, string]> = [
- [
- 'generation, object span name, with model',
- '$ai_generation',
- { $ai_span_name: {}, $ai_model: 'gpt-4' },
- 'gpt-4',
- ],
- ['generation, object span name, no model', '$ai_generation', { $ai_span_name: {} }, 'Generation'],
- ['generation, object model', '$ai_generation', { $ai_model: {} }, 'Generation'],
- [
- 'generation, object model, with provider',
- '$ai_generation',
- { $ai_model: {}, $ai_provider: 'openai' },
- 'Generation (openai)',
- ],
- ['generation, object provider', '$ai_generation', { $ai_model: 'gpt-4', $ai_provider: {} }, 'gpt-4'],
- ['embedding, object span name', '$ai_embedding', { $ai_span_name: {} }, 'Embedding'],
- ['embedding, object model', '$ai_embedding', { $ai_model: {} }, 'Embedding'],
- ['span, object span name', '$ai_span', { $ai_span_name: {} }, 'Span'],
- ['span, array span name', '$ai_span', { $ai_span_name: [] }, 'Span'],
- ['span, numeric span name', '$ai_span', { $ai_span_name: 123 }, 'Span'],
- ]
-
- it.each(cases)('%s falls back to a string label', (_label, event, properties, expected) => {
- const traceEvent: LLMTraceEvent = {
+ it('formats generation event without model or provider', () => {
+ const event: LLMTraceEvent = {
id: 'event-1',
- event,
- properties,
- createdAt: '2024-01-01T00:00:00Z',
- }
- const title = formatLLMEventTitle(traceEvent)
- expect(typeof title).toBe('string')
- expect(title).toBe(expected)
- })
- })
- })
-
- describe('formatModelRowLabel', () => {
- it('returns null for a trace', () => {
- const trace: LLMTrace = {
- id: 'trace-1',
- createdAt: '2024-01-01T00:00:00Z',
- distinctId: 'user-1',
- person: {
- uuid: 'person-1',
- created_at: '2024-01-01T00:00:00Z',
- properties: {},
- distinct_id: 'user-1',
- },
- events: [],
- }
- expect(formatModelRowLabel(trace)).toBeNull()
- })
-
- const cases: Array<[string, string, Record<string, unknown>, string | null]> = [
- // Only generations with a span name get a model row; otherwise the event title covers it.
- ['span event', '$ai_span', { $ai_span_name: 'My Span', $ai_model: 'gpt-4' }, null],
- ['embedding event', '$ai_embedding', { $ai_span_name: 'Embed', $ai_model: 'gpt-4' }, null],
- ['generation without span name', '$ai_generation', { $ai_model: 'gpt-4' }, null],
- ['generation with object span name', '$ai_generation', { $ai_span_name: {}, $ai_model: 'gpt-4' }, null],
- [
- 'generation with model and provider',
- '$ai_generation',
- { $ai_span_name: 'Chat', $ai_model: 'gpt-4', $ai_provider: 'openai' },
- 'gpt-4 (openai)',
- ],
- ['generation with model only', '$ai_generation', { $ai_span_name: 'Chat', $ai_model: 'gpt-4' }, 'gpt-4'],
- [
- 'generation with object model and string provider',
- '$ai_generation',
- { $ai_span_name: 'Chat', $ai_model: {}, $ai_provider: 'openai' },
- 'openai',
- ],
- [
- 'generation with string model and object provider',
- '$ai_generation',
- { $ai_span_name: 'Chat', $ai_model: 'gpt-4', $ai_provider: {} },
- 'gpt-4',
- ],
- ['generation with object model only', '$ai_generation', { $ai_span_name: 'Chat', $ai_model: {} }, null],
- ['generation with array model only', '$ai_generation', { $ai_span_name: 'Chat', $ai_model: [] }, null],
- ['generation with numeric model only', '$ai_generation', { $ai_span_name: 'Chat', $ai_model: 4 }, null],
- ['generation without model or provider', '$ai_generation', { $ai_span_name: 'Chat' }, null],
- ]
-
- it.each(cases)('%s', (_label, event, properties, expected) => {
- const traceEvent: LLMTraceEvent = {
- id: 'event-1',
- event,
- properties,
- createdAt: '2024-01-01T00:00:00Z',
- }
- expect(formatModelRowLabel(traceEvent)).toBe(expected)
- })
- })
-
- describe('LiteLLM support', () => {
- const litellmChoice = {
- finish_reason: 'stop',
- index: 0,
- message: {
- annotations: [],
- content:
- "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!",
- function_call: null,
- role: 'assistant',
- tool_calls: null,
- },
- provider_specific_fields: {},
- }
-
- const litellmResponse = {
- choices: [litellmChoice],
- model: 'gpt-3.5-turbo',
- usage: { prompt_tokens: 10, completion_tokens: 25, total_tokens: 35 },
- }
-
- describe('normalizeMessage', () => {
- it('should handle LiteLLM choice format', () => {
- const result = normalizeMessage(litellmChoice, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0]).toMatchObject({
- role: 'assistant',
- content:
- "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!",
- function_call: null,
- })
- expect(result[0].tool_calls).toBeUndefined()
- })
-
- it('should handle LiteLLM choice with tool calls', () => {
- const choiceWithTools = {
- ...litellmChoice,
- message: {
- ...litellmChoice.message,
- content: null,
- tool_calls: [
- {
- type: 'function',
- id: 'call_123',
- function: {
- name: 'get_weather',
- arguments: '{"location": "San Francisco"}',
- },
- },
- ],
- },
- }
-
- const result = normalizeMessage(choiceWithTools, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('assistant')
- expect(result[0].tool_calls).toHaveLength(1)
- expect(result[0].tool_calls![0].function.name).toBe('get_weather')
- })
- })
-
- describe('normalizeMessages', () => {
- it('should handle LiteLLM response format', () => {
- const result = normalizeMessages(litellmResponse, 'assistant')
-
- expect(result).toHaveLength(1)
- expect(result[0]).toMatchObject({
- role: 'assistant',
- content:
- "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!",
- function_call: null,
- })
- expect(result[0].tool_calls).toBeUndefined()
- })
-
- it('should handle LiteLLM response with multiple choices', () => {
- const multiChoiceResponse = {
- choices: [
- litellmChoice,
- {
- ...litellmChoice,
- index: 1,
- message: {
- ...litellmChoice.message,
- content: 'Alternative response',
- },
- },
- ],
- }
-
- const result = normalizeMessages(multiChoiceResponse, 'assistant')
-
- expect(result).toHaveLength(2)
- expect(result[0].content).toBe(
- "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!"
- )
- expect(result[1].content).toBe('Alternative response')
- })
-
- it('should handle empty LiteLLM response gracefully', () => {
- const emptyResponse = { choices: [] }
- const result = normalizeMessages(emptyResponse, 'assistant')
-
- expect(result).toHaveLength(0)
- })
-
- it('returns no messages for undefined input', () => {
- expect(normalizeMessages(undefined, 'assistant')).toEqual([])
- })
- })
-
- describe('Role normalization', () => {
- it('should preserve system role in array-based content', () => {
- const systemMessage = {
- role: 'system',
- content: [{ type: 'text', text: 'You are a helpful assistant.' }],
- }
-
- const result = normalizeMessage(systemMessage, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('system')
- expect(result[0].content).toEqual([{ type: 'text', text: 'You are a helpful assistant.' }])
- })
-
- it('should preserve system role in string content', () => {
- const systemMessage = {
- role: 'system',
- content: 'You are a helpful assistant.',
- }
-
- const result = normalizeMessage(systemMessage, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('system')
- expect(result[0].content).toBe('You are a helpful assistant.')
- })
-
- it('should map known provider roles', () => {
- const humanMessage = {
- role: 'human',
- content: 'Hello',
- }
-
- const result = normalizeMessage(humanMessage, 'user')
-
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('user')
- expect(result[0].content).toBe('Hello')
- })
- })
- })
-
- describe('LangChain/LangGraph format', () => {
- it('normalizeMessage maps human type to user role', () => {
- const message = { type: 'human', content: 'Hello', id: 'msg-1' }
- const result = normalizeMessage(message, 'assistant')
- expect(result).toEqual([{ role: 'user', content: 'Hello' }])
- })
-
- it('normalizeMessage maps ai type to assistant role', () => {
- const message = { type: 'ai', content: 'Hi there', id: 'msg-2', meta: {} }
- const result = normalizeMessage(message, 'user')
- expect(result).toEqual([{ role: 'assistant', content: 'Hi there' }])
- })
-
- it('normalizeMessage handles ai type with tool_calls', () => {
- const message = {
- type: 'ai',
- content: '',
- tool_calls: [
- {
- type: 'function' as const,
- id: 'toolu_123',
- function: { name: 'search', arguments: '{"q":"test"}' },
- },
- ],
- }
- const result = normalizeMessage(message, 'user')
- expect(result).toHaveLength(1)
- expect(result[0].role).toBe('assistant')
- expect(result[0].tool_calls).toHaveLength(1)
- expect(result[0].tool_calls![0].function.name).toBe('search')
- })
-
- it('normalizeMessage handles tool type with tool_call_id', () => {
- const message = {
- type: 'tool',
- content: 'Search results: ...',
- tool_call_id: 'toolu_123',
- }
- const result = normalizeMessage(message, 'assistant')
- expect(result).toEqual([{ role: 'assistant', content: 'Search results: ...', tool_call_id: 'toolu_123' }])
- })
-
- it('normalizeMessage maps context type to system role', () => {
- const message = { type: 'context', content: '<system_reminder>Your initial mode is sql.</system_reminder>' }
- const result = normalizeMessage(message, 'user')
- expect(result).toEqual([
- { role: 'system', content: '<system_reminder>Your initial mode is sql.</system_reminder>' },
- ])
- })
- })
-
- describe('OTel parts format', () => {
- it('normalizes a single text part into a string content', () => {
- const message = {
- role: 'assistant',
- parts: [{ type: 'text', content: 'Hello from OTel!' }],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([{ role: 'assistant', content: 'Hello from OTel!' }])
- })
-
- it('normalizes multiple text parts into an array content', () => {
- const message = {
- role: 'assistant',
- parts: [
- { type: 'text', content: 'First part.' },
- { type: 'text', content: 'Second part.' },
- ],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: [
- { type: 'text', text: 'First part.' },
- { type: 'text', text: 'Second part.' },
- ],
- },
- ])
- })
-
- it('normalizes tool_call parts into CompatToolCall format', () => {
- const message = {
- role: 'assistant',
- parts: [
- {
- type: 'tool_call',
- id: 'call_1',
- name: 'get_weather',
- arguments: { location: 'London' },
- },
- ],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'call_1',
- function: {
- name: 'get_weather',
- arguments: { location: 'London' },
- },
- },
- ],
- },
- ])
- })
-
- it('handles mixed text and tool_call parts', () => {
- const message = {
- role: 'assistant',
- parts: [
- { type: 'text', content: 'Let me check that.' },
- {
- type: 'tool_call',
- id: 'call_1',
- name: 'get_weather',
- arguments: { location: 'London' },
- },
- ],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: 'Let me check that.',
- tool_calls: [
- {
- type: 'function',
- id: 'call_1',
- function: {
- name: 'get_weather',
- arguments: { location: 'London' },
- },
- },
- ],
- },
- ])
- })
-
- it('preserves top-level fields via spread', () => {
- const message = {
- role: 'assistant',
- finish_reason: 'stop',
- parts: [{ type: 'text', content: 'Done.' }],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- { role: 'assistant', finish_reason: 'stop', content: 'Done.' },
- ])
- })
-
- it('handles tool_call without id', () => {
- const message = {
- role: 'assistant',
- parts: [
- {
- type: 'tool_call',
- name: 'do_thing',
- arguments: { x: 1 },
- },
- ],
- }
-
- const result = normalizeMessage(message, 'user')
- expect(result[0].tool_calls![0].id).toBeUndefined()
- expect(result[0].tool_calls![0].function.name).toBe('do_thing')
- })
-
- it('handles tool_call without arguments', () => {
- const message = {
- role: 'assistant',
- parts: [{ type: 'tool_call', id: 'call_1', name: 'no_args' }],
- }
-
- const result = normalizeMessage(message, 'user')
- expect(result[0].tool_calls![0].function.arguments).toEqual({})
- })
-
- it('normalizes role aliases', () => {
- const message = {
- role: 'human',
- parts: [{ type: 'text', content: 'Hi' }],
- }
-
- expect(normalizeMessage(message, 'assistant')).toEqual([{ role: 'user', content: 'Hi' }])
- })
-
- it.each([
- ['the schema response key', { response: 'Sunny, 25°C' }],
- ['the spec example result key', { result: 'Sunny, 25°C' }],
- ['both keys, preferring response', { response: 'Sunny, 25°C', result: 'stale' }],
- ['a null response falling back to result', { response: null, result: 'Sunny, 25°C' }],
- ])('normalizes tool_call_response parts into tool messages from %s', (_label, resultFields) => {
- const message = {
- role: 'user',
- parts: [
- {
- type: 'tool_call_response',
- id: 'call_abc',
- name: 'get_weather',
- ...resultFields,
- },
- ],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- { role: 'tool', content: 'Sunny, 25°C', tool_call_id: 'call_abc' },
- ])
- })
-
- it('keeps an empty string response over a stale result', () => {
- const message = {
- role: 'user',
- parts: [
- {
- type: 'tool_call_response',
- id: 'call_abc',
- name: 'get_weather',
- response: '',
- result: 'stale',
- },
- ],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([{ role: 'tool', content: '', tool_call_id: 'call_abc' }])
- })
-
- it.each(['response', 'result'])('stringifies non-string tool_call_response %s values', (field) => {
- const message = {
- role: 'user',
- parts: [
- {
- type: 'tool_call_response',
- id: 'call_1',
- name: 'get_data',
- [field]: { temperature: 25, unit: 'C' },
- },
- ],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'tool',
- content: '{"temperature":25,"unit":"C"}',
- tool_call_id: 'call_1',
- },
- ])
- })
-
- it('parses stringified JSON tool_call arguments', () => {
- const message = {
- role: 'assistant',
- parts: [
- {
- type: 'tool_call',
- id: 'call_1',
- name: 'get_weather',
- arguments: '{"latitude":45.5,"longitude":-73.5}',
- },
- ],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'call_1',
- function: {
- name: 'get_weather',
- arguments: { latitude: 45.5, longitude: -73.5 },
- },
- },
- ],
- },
- ])
- })
-
- it('handles a full tool-use conversation', () => {
- const messages = [
- {
- role: 'assistant',
- parts: [
- {
- type: 'tool_call',
- id: 'call_1',
- name: 'get_weather',
- arguments: '{"location":"Montreal"}',
- },
- ],
- finish_reason: 'tool_call',
- },
- {
- role: 'user',
- parts: [
- {
- type: 'tool_call_response',
- id: 'call_1',
- name: 'get_weather',
- result: '-10°C, partly cloudy',
- },
- ],
- },
- ]
-
- const result = normalizeMessages(messages, 'user')
-
- expect(result).toEqual([
- {
- role: 'assistant',
- finish_reason: 'tool_call',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'call_1',
- function: { name: 'get_weather', arguments: { location: 'Montreal' } },
- },
- ],
- },
- {
- role: 'tool',
- content: '-10°C, partly cloudy',
- tool_call_id: 'call_1',
- },
- ])
+ event: '$ai_generation',
+ properties: {},
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('Generation')
+ })
})
- it('handles empty parts array', () => {
- const message = {
- role: 'assistant',
- parts: [],
- }
+ describe('$ai_embedding events', () => {
+ it('formats embedding event with span name', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_embedding',
+ properties: {
+ $ai_span_name: 'Document Embedding',
+ $ai_model: 'text-embedding-3-small',
+ $ai_provider: 'openai',
+ },
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('Document Embedding')
+ })
- expect(normalizeMessage(message, 'user')).toEqual([{ role: 'assistant', content: '' }])
- })
+ it('formats embedding event with model and provider', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_embedding',
+ properties: {
+ $ai_model: 'text-embedding-3-small',
+ $ai_provider: 'openai',
+ },
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('text-embedding-3-small (openai)')
+ })
- it('normalizes reasoning parts into thinking messages without an empty primary', () => {
- const message = {
- role: 'assistant',
- parts: [
- { type: 'reasoning', content: 'The user wants the forecast.' },
- { type: 'text', content: 'Here is the forecast.' },
- ],
- }
+ it('formats embedding event with model only', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_embedding',
+ properties: {
+ $ai_model: 'text-embedding-3-small',
+ },
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('text-embedding-3-small')
+ })
- expect(normalizeMessage(message, 'user')).toEqual([
- { role: 'assistant', content: 'Here is the forecast.' },
- { role: 'assistant (thinking)', content: 'The user wants the forecast.' },
- ])
+ it('formats embedding event without model or provider', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_embedding',
+ properties: {},
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('Embedding')
+ })
})
- it('normalizes a reasoning-only message into just a thinking message', () => {
- const message = {
- role: 'assistant',
- parts: [{ type: 'reasoning', content: 'Considering options.' }],
- }
+ describe('$ai_span and other events', () => {
+ it('formats span event with span name', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_span',
+ properties: {
+ $ai_span_name: 'Database Query',
+ },
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('Database Query')
+ })
- expect(normalizeMessage(message, 'user')).toEqual([
- { role: 'assistant (thinking)', content: 'Considering options.' },
- ])
- })
+ it('formats span event without span name', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_span',
+ properties: {},
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('Span')
+ })
- it('normalizes server_tool_call parts into tool calls', () => {
- const message = {
- role: 'assistant',
- parts: [
- {
- type: 'server_tool_call',
- id: 'st_1',
- name: 'web_search',
- server_tool_call: { type: 'web_search', query: 'weather in Montreal' },
+ it('formats metric event', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_metric',
+ properties: {
+ $ai_span_name: 'Performance Metric',
},
- ],
- }
-
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'st_1',
- function: {
- name: 'web_search',
- arguments: { type: 'web_search', query: 'weather in Montreal' },
- },
- },
- ],
- },
- ])
- })
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('Performance Metric')
+ })
- it('normalizes server_tool_call_response parts into tool messages', () => {
- const message = {
- role: 'assistant',
- parts: [
- {
- type: 'server_tool_call_response',
- id: 'st_1',
- server_tool_call_response: { type: 'web_search', status: 'completed' },
+ it('formats feedback event', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: '$ai_feedback',
+ properties: {
+ $ai_span_name: 'User Feedback',
},
- ],
- }
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('User Feedback')
+ })
- expect(normalizeMessage(message, 'user')).toEqual([
- {
- role: 'tool',
- content: '{"type":"web_search","status":"completed"}',
- tool_call_id: 'st_1',
- },
- ])
+ it('formats unknown event type', () => {
+ const event: LLMTraceEvent = {
+ id: 'event-1',
+ event: 'custom_event',
+ properties: {
+ $ai_span_name: 'Custom Action',
+ },
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatLLMEventTitle(event)).toBe('Custom Action')
+ })
})
- it('normalizes image blob parts into data-URI image items', () => {
- const message = {
- role: 'user',
- parts: [
- { type: 'text', content: 'What is in this picture?' },
- { type: 'blob', modality: 'image', mime_type: 'image/png', content: 'aGVsbG8=' },
+ describe('non-string properties (untyped property bag)', () => {
+ // `$ai_*` title properties are strings by convention, but the event property bag is
+ // untyped. A structured value (object/array) or number must never be returned verbatim,
+ // or it would be rendered as a raw React child and crash the page (React error #31).
+ const cases: Array<[string, string, Record<string, unknown>, string]> = [
+ [
+ 'generation, object span name, with model',
+ '$ai_generation',
+ { $ai_span_name: {}, $ai_model: 'gpt-4' },
+ 'gpt-4',
],
- }
+ ['generation, object span name, no model', '$ai_generation', { $ai_span_name: {} }, 'Generation'],
+ ['generation, object model', '$ai_generation', { $ai_model: {} }, 'Generation'],
+ [
+ 'generation, object model, with provider',
+ '$ai_generation',
+ { $ai_model: {}, $ai_provider: 'openai' },
+ 'Generation (openai)',
+ ],
+ ['generation, object provider', '$ai_generation', { $ai_model: 'gpt-4', $ai_provider: {} }, 'gpt-4'],
+ ['embedding, object span name', '$ai_embedding', { $ai_span_name: {} }, 'Embedding'],
+ ['embedding, object model', '$ai_embedding', { $ai_model: {} }, 'Embedding'],
+ ['span, object span name', '$ai_span', { $ai_span_name: {} }, 'Span'],
+ ['span, array span name', '$ai_span', { $ai_span_name: [] }, 'Span'],
+ ['span, numeric span name', '$ai_span', { $ai_span_name: 123 }, 'Span'],
+ ]
- expect(normalizeMessage(message, 'user')).toEqual([
- { role: 'user', content: 'What is in this picture?' },
- { role: 'user', content: [{ type: 'image', image: 'data:image/png;base64,aGVsbG8=' }] },
- ])
+ it.each(cases)('%s falls back to a string label', (_label, event, properties, expected) => {
+ const traceEvent: LLMTraceEvent = {
+ id: 'event-1',
+ event,
+ properties,
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ const title = formatLLMEventTitle(traceEvent)
+ expect(typeof title).toBe('string')
+ expect(title).toBe(expected)
+ })
})
+ })
- it('normalizes image uri parts into image items', () => {
- const message = {
- role: 'user',
- parts: [
- { type: 'uri', modality: 'image', mime_type: 'image/jpeg', uri: 'https://example.com/cat.jpg' },
- ],
+ describe('formatModelRowLabel', () => {
+ it('returns null for a trace', () => {
+ const trace: LLMTrace = {
+ id: 'trace-1',
+ createdAt: '2024-01-01T00:00:00Z',
+ distinctId: 'user-1',
+ person: {
+ uuid: 'person-1',
+ created_at: '2024-01-01T00:00:00Z',
+ properties: {},
+ distinct_id: 'user-1',
+ },
+ events: [],
}
-
- expect(normalizeMessage(message, 'user')).toEqual([
- { role: 'user', content: [{ type: 'image', image: 'https://example.com/cat.jpg' }] },
- ])
+ expect(formatModelRowLabel(trace)).toBeNull()
})
- it.each([
+ const cases: Array<[string, string, Record<string, unknown>, string | null]> = [
+ // Only generations with a span name get a model row; otherwise the event title covers it.
+ ['span event', '$ai_span', { $ai_span_name: 'My Span', $ai_model: 'gpt-4' }, null],
+ ['embedding event', '$ai_embedding', { $ai_span_name: 'Embed', $ai_model: 'gpt-4' }, null],
+ ['generation without span name', '$ai_generation', { $ai_model: 'gpt-4' }, null],
+ ['generation with object span name', '$ai_generation', { $ai_span_name: {}, $ai_model: 'gpt-4' }, null],
[
- 'a non-image blob',
- { type: 'blob', modality: 'audio', mime_type: 'audio/mp3', content: 'aGVsbG8=' },
- '[audio]',
+ 'generation with model and provider',
+ '$ai_generation',
+ { $ai_span_name: 'Chat', $ai_model: 'gpt-4', $ai_provider: 'openai' },
+ 'gpt-4 (openai)',
],
+ ['generation with model only', '$ai_generation', { $ai_span_name: 'Chat', $ai_model: 'gpt-4' }, 'gpt-4'],
[
- 'a non-image uri',
- { type: 'uri', modality: 'document', mime_type: 'application/pdf', uri: 'https://example.com/doc.pdf' },
- '[document: https://example.com/doc.pdf]',
+ 'generation with object model and string provider',
+ '$ai_generation',
+ { $ai_span_name: 'Chat', $ai_model: {}, $ai_provider: 'openai' },
+ 'openai',
],
- ['a file reference', { type: 'file', modality: 'document', file_id: 'file_123' }, '[file: file_123]'],
- ])('normalizes %s into a text marker', (_label, part, expectedContent) => {
- const message = { role: 'user', parts: [part] }
-
- expect(normalizeMessage(message, 'user')).toEqual([{ role: 'user', content: expectedContent }])
- })
-
- it.each([
[
- 'its summary when present',
- { type: 'compaction', id: 'c1', content: 'Earlier turns covered pricing.' },
- 'Earlier turns covered pricing.',
+ 'generation with string model and object provider',
+ '$ai_generation',
+ { $ai_span_name: 'Chat', $ai_model: 'gpt-4', $ai_provider: {} },
+ 'gpt-4',
],
- ['a marker when the summary is absent', { type: 'compaction', id: 'c1' }, '[conversation compacted]'],
- ])('normalizes compaction parts into %s', (_label, part, expectedContent) => {
- const message = { role: 'user', parts: [part] }
+ ['generation with object model only', '$ai_generation', { $ai_span_name: 'Chat', $ai_model: {} }, null],
+ ['generation with array model only', '$ai_generation', { $ai_span_name: 'Chat', $ai_model: [] }, null],
+ ['generation with numeric model only', '$ai_generation', { $ai_span_name: 'Chat', $ai_model: 4 }, null],
+ ['generation without model or provider', '$ai_generation', { $ai_span_name: 'Chat' }, null],
+ ]
- expect(normalizeMessage(message, 'user')).toEqual([{ role: 'user', content: expectedContent }])
+ it.each(cases)('%s', (_label, event, properties, expected) => {
+ const traceEvent: LLMTraceEvent = {
+ id: 'event-1',
+ event,
+ properties,
+ createdAt: '2024-01-01T00:00:00Z',
+ }
+ expect(formatModelRowLabel(traceEvent)).toBe(expected)
})
})
@@ -2905,129 +1245,7 @@ describe.each(IMPLS)('AI observability utils [$name]', ({ normalizeMessage, norm
// Regressions found by sampling real production payloads across teams. These
// shapes were mishandled at some point by the recipe pipeline.
- describe('production payload regressions', () => {
- it('null input carries no message', () => {
- // Recipe used to salvage `null` into a spurious empty user message.
- expect(normalizeMessages(null, 'user')).toEqual([])
- })
-
- it('number/boolean input carries no message', () => {
- expect(normalizeMessages(42, 'user')).toEqual([])
- expect(normalizeMessages(true, 'user')).toEqual([])
- })
-
- it('a single-field {content} object inherits the default role', () => {
- // Recipe used to force role:user even on the output (assistant) side.
- expect(normalizeMessages({ content: 'hi' }, 'assistant')).toEqual([{ role: 'assistant', content: 'hi' }])
- expect(normalizeMessages({ content: 'hi' }, 'user')).toEqual([{ role: 'user', content: 'hi' }])
- })
-
- it('an empty top-level tool_calls array adds no synthetic message', () => {
- // Recipe used to append an empty assistant message for `tool_calls: []`.
- // (thinking+text content so the Anthropic envelope path is exercised.)
- const message = {
- role: 'assistant',
- content: [
- { type: 'thinking', thinking: 'x' },
- { type: 'text', text: 'done' },
- ],
- tool_calls: [],
- }
- expect(normalizeMessage(message, 'assistant')).toEqual([
- { role: 'assistant (thinking)', content: 'x' },
- { role: 'assistant', content: 'done' },
- ])
- })
-
- it('OTel parts whose text lives under an unexpected key degrade to empty content, not [null]', () => {
- // `text`-keyed parts (some SDKs) don't match the `content` pluck; the
- // result must collapse to '' rather than a malformed [null] array.
- const message = { role: 'user', parts: [{ type: 'text', text: 'hi' }] }
- expect(normalizeMessage(message, 'user')).toEqual([{ role: 'user', content: '' }])
- })
- })
-
// Structured content the recipe pipeline surfaces as canonical messages.
- describe('recipe normalization improvements', () => {
- const expectRecipe = (input: unknown, role: string, expected: unknown): void => {
- expect(normalizeMessages(input, role)).toEqual(expected)
- }
-
- it('extracts single-field {text}/{message} wrappers instead of stringifying them', () => {
- expectRecipe({ text: 'hello' }, 'assistant', [{ role: 'assistant', content: 'hello' }])
- })
-
- it('empties null content and drops an empty tool_calls array', () => {
- // content is normalized to '' (renderers expect a string).
- expectRecipe({ role: 'assistant', content: null, tool_calls: [] }, 'assistant', [
- { role: 'assistant', content: '' },
- ])
- })
-
- it('preserves and canonicalizes a tool_call lacking the type marker', () => {
- // canonicalizes it to {type, id, function:{name, parsed args}}.
- const input = {
- role: 'assistant',
- content: null,
- tool_calls: [
- {
- id: 'call_1',
- caller: { type: 'direct' },
- index: 0,
- function: { name: 'send_email', arguments: '{"to":"[email protected]"}' },
- },
- ],
- }
- expectRecipe(input, 'assistant', [
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'call_1',
- function: { name: 'send_email', arguments: { to: '[email protected]' } },
- },
- ],
- },
- ])
- })
-
- it('flattens a doubly-nested message array instead of stringifying it', () => {
- const input = [
- [
- { role: 'system', content: 'a' },
- { role: 'user', content: 'b' },
- ],
- ]
- expectRecipe(input, 'user', [
- { role: 'system', content: 'a' },
- { role: 'user', content: 'b' },
- ])
- })
-
- it('parses typed agent items (tool_call/tool_result) into real tool messages', () => {
- // Flat type-discriminated agent stream (OpenAI Agents SDK and similar).
- const input = [
- { type: 'tool_call', callId: 'c1', name: 'getWeather', arguments: { city: 'NYC' } },
- { type: 'tool_result', callId: 'c1', name: 'getWeather', output: { tempF: 71 } },
- ]
- expectRecipe(input, 'user', [
- {
- role: 'assistant',
- content: '',
- tool_calls: [
- {
- type: 'function',
- id: 'c1',
- function: { name: 'getWeather', arguments: { city: 'NYC' } },
- },
- ],
- },
- { role: 'tool', content: '{"tempF":71}', tool_call_id: 'c1' },
- ])
- })
- })
})
describe('queryEvaluationRuns', () => {
diff --git a/products/ai_observability/shared/normalizer_cases.json b/products/ai_observability/shared/normalizer_cases.json
new file mode 100644
index 000000000000..8d9dbea1be60
--- /dev/null
+++ b/products/ai_observability/shared/normalizer_cases.json
@@ -0,0 +1,3300 @@
+{
+ "cases": [
+ {
+ "name": "normalizeOutputMessage parses OpenAI message",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": "Hello, world!"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "assistant",
+ "content": "Hello, world!"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeOutputMessage stringifies incomplete OpenAI message",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "assistant",
+ "content": "{\"role\":\"assistant\"}"
+ }
+ ],
+ "recognized": false
+ }
+ },
+ {
+ "name": "normalizeOutputMessage stringifies incomplete OpenAI message #2",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "role": "assistant",
+ "content": "{\"role\":\"assistant\"}"
+ }
+ ],
+ "recognized": false
+ }
+ },
+ {
+ "name": "normalizeOutputMessage parses OpenAI tool calls",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "123",
+ "function": {
+ "name": "test",
+ "arguments": "{\"foo\": \"bar\"}"
+ }
+ },
+ {
+ "type": "function",
+ "id": "456",
+ "function": {
+ "name": "test2",
+ "arguments": "{\"bar\": \"baz\"}"
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "123",
+ "function": {
+ "name": "test",
+ "arguments": {
+ "foo": "bar"
+ }
+ }
+ },
+ {
+ "type": "function",
+ "id": "456",
+ "function": {
+ "name": "test2",
+ "arguments": {
+ "bar": "baz"
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeOutputMessage parses OpenAI tool use",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "tool",
+ "content": "response",
+ "tool_call_id": "456"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "tool",
+ "content": "response",
+ "tool_call_id": "456"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeOutputMessage parses a string message",
+ "call": "normalizeMessage",
+ "input": "foo",
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "foo",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeOutputMessage parses a string message #2",
+ "call": "normalizeMessage",
+ "input": "foo",
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "foo",
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeOutputMessage parses an Anthropic tool call message",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool_use",
+ "id": "toolu_01D7FLrfh4GYq7yT1ULFeyMV",
+ "name": "get_stock_price",
+ "input": {
+ "ticker": "^GSPC"
+ }
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "toolu_01D7FLrfh4GYq7yT1ULFeyMV",
+ "function": {
+ "name": "get_stock_price",
+ "arguments": {
+ "ticker": "^GSPC"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeOutputMessage parses an Anthropic tool call message #2",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_01D7FLrfh4GYq7yT1ULFeyMV",
+ "name": "get_stock_price",
+ "input": {
+ "ticker": "^GSPC"
+ }
+ },
+ {
+ "type": "tool_use",
+ "id": "toolu_01D7FLrfh4GYq7yT1ULFeyMV2",
+ "name": "get_stock_price",
+ "input": {
+ "ticker": "^GSPC"
+ }
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "toolu_01D7FLrfh4GYq7yT1ULFeyMV",
+ "function": {
+ "name": "get_stock_price",
+ "arguments": {
+ "ticker": "^GSPC"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ },
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "toolu_01D7FLrfh4GYq7yT1ULFeyMV2",
+ "function": {
+ "name": "get_stock_price",
+ "arguments": {
+ "ticker": "^GSPC"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeOutputMessage prefers top-level tool_calls over content tool_use blocks",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Let me check that."
+ },
+ {
+ "type": "tool_use",
+ "id": "toolu_123",
+ "name": "get_weather",
+ "input": {}
+ }
+ ],
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "toolu_123",
+ "function": {
+ "name": "get_weather",
+ "arguments": "{\"location\": \"San Francisco\"}"
+ }
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "Let me check that.",
+ "role": "assistant"
+ },
+ {
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "toolu_123",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "San Francisco"
+ }
+ }
+ }
+ ],
+ "role": "assistant",
+ "content": ""
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeOutputMessage parses an Anthropic tool result message",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "1",
+ "content": "foo"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "foo",
+ "tool_call_id": "1",
+ "role": "assistant (tool result)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeOutputMessage parses an Anthropic tool result message #2",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "1",
+ "content": [
+ {
+ "type": "text",
+ "text": "foo"
+ }
+ ]
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "foo",
+ "role": "assistant (tool result)",
+ "tool_call_id": "1"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API parses a function_call item as an assistant tool call",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "function_call",
+ "call_id": "call_tLji37A7lp7Buy7hcqUPMUXE",
+ "name": "search_category",
+ "arguments": "{\"dimensions\":[],\"list_all\":true}"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_tLji37A7lp7Buy7hcqUPMUXE",
+ "function": {
+ "name": "search_category",
+ "arguments": {
+ "dimensions": [],
+ "list_all": true
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API parses a function_call_output item as a tool result",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "function_call_output",
+ "call_id": "call_tLji37A7lp7Buy7hcqUPMUXE",
+ "output": "{\"categories\":[{\"id\":\"149288\",\"name\":\"Asheville Cloud Shaker\"}]}"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "{\"categories\":[{\"id\":\"149288\",\"name\":\"Asheville Cloud Shaker\"}]}",
+ "tool_call_id": "call_tLji37A7lp7Buy7hcqUPMUXE",
+ "role": "assistant (tool result)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API parses a reasoning item as assistant thinking",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "reasoning",
+ "id": "rs_123",
+ "summary": [
+ {
+ "type": "summary_text",
+ "text": "Thinking about the query..."
+ },
+ {
+ "type": "summary_text",
+ "text": "Deciding on a response."
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Thinking about the query...\nDeciding on a response.",
+ "role": "assistant (thinking)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles reasoning with no summary",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "reasoning",
+ "id": "rs_456"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "role": "assistant (thinking)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK reasoning with text field",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "reasoning",
+ "text": "Let me think about this step by step."
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Let me think about this step by step.",
+ "role": "assistant (thinking)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK tool-call with string arguments (posthog-node format)",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool-call",
+ "id": "toolu_vrtx_01CTHfH7tfd2vNWSWJhKzfov",
+ "function": {
+ "name": "queryModel",
+ "arguments": "{\"query\": \"relevant data\"}"
+ }
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "toolu_vrtx_01CTHfH7tfd2vNWSWJhKzfov",
+ "function": {
+ "name": "queryModel",
+ "arguments": {
+ "query": "relevant data"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK tool-call with object arguments (posthog-node format)",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool-call",
+ "id": "call_123",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "NYC"
+ }
+ }
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_123",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "NYC"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK tool-call with native Vercel SDK format (toolName/input/toolCallId)",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool-call",
+ "toolCallId": "tc_native_123",
+ "toolName": "get_weather",
+ "input": {
+ "location": "NYC"
+ }
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "tc_native_123",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "NYC"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK tool-result with object output",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool-result",
+ "toolCallId": "tc_123",
+ "toolName": "get_weather",
+ "output": {
+ "temperature": 72,
+ "unit": "F"
+ }
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "{\"temperature\":72,\"unit\":\"F\"}",
+ "tool_call_id": "tc_123",
+ "role": "assistant (tool result)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK tool-result with string output",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool-result",
+ "toolCallId": "tc_456",
+ "toolName": "search",
+ "output": "Found 3 results"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "Found 3 results",
+ "tool_call_id": "tc_456",
+ "role": "assistant (tool result)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK toolName variant with string input",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool-call",
+ "toolCallId": "tc_1",
+ "toolName": "run_code",
+ "input": "print(\"hi\")"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "tc_1",
+ "function": {
+ "name": "run_code",
+ "arguments": "print(\"hi\")"
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK toolName variant with missing input",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool-call",
+ "toolCallId": "tc_2",
+ "toolName": "no_args_tool"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "tc_2",
+ "function": {
+ "name": "no_args_tool",
+ "arguments": {}
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API prefers function variant when both function and toolName are present",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool-call",
+ "id": "func_id",
+ "function": {
+ "name": "from_function",
+ "arguments": "{\"a\":1}"
+ },
+ "toolCallId": "tool_id",
+ "toolName": "from_toolName",
+ "input": {
+ "b": 2
+ }
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "func_id",
+ "function": {
+ "name": "from_function",
+ "arguments": {
+ "a": 1
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK tool-result with undefined output",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool-result",
+ "toolCallId": "tc_empty",
+ "toolName": "void_tool"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_call_id": "tc_empty",
+ "role": "assistant (tool result)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles Vercel AI SDK mixed content array with reasoning + text + tool-call",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "reasoning",
+ "text": "I should use a tool to look up the data."
+ },
+ {
+ "type": "text",
+ "text": "Let me check that for you."
+ },
+ {
+ "type": "tool-call",
+ "id": "toolu_vrtx_abc",
+ "function": {
+ "name": "queryModel",
+ "arguments": "{\"query\":\"data\"}"
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "I should use a tool to look up the data.",
+ "role": "assistant (thinking)"
+ },
+ {
+ "content": "Let me check that for you.",
+ "role": "assistant"
+ },
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "toolu_vrtx_abc",
+ "function": {
+ "name": "queryModel",
+ "arguments": {
+ "query": "data"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API parses web_search_call as an assistant tool call preserving metadata",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "web_search_call",
+ "id": "ws_123",
+ "status": "completed",
+ "action": {
+ "type": "search",
+ "query": "weather today"
+ }
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "ws_123",
+ "function": {
+ "name": "web_search_call",
+ "arguments": {
+ "action": {
+ "type": "search",
+ "query": "weather today"
+ }
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API parses code_interpreter_call as an assistant tool call preserving metadata",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "code_interpreter_call",
+ "id": "ci_123",
+ "status": "completed",
+ "code": "print(\"hello\")"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "ci_123",
+ "function": {
+ "name": "code_interpreter_call",
+ "arguments": {
+ "code": "print(\"hello\")"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API parses image_generation_call as an assistant tool call preserving metadata",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "image_generation_call",
+ "id": "ig_123",
+ "status": "completed",
+ "prompt": "a red cat"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "ig_123",
+ "function": {
+ "name": "image_generation_call",
+ "arguments": {
+ "prompt": "a red cat"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API parses mcp_call as an assistant tool call preserving metadata",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "mcp_call",
+ "id": "mcp_123",
+ "status": "completed",
+ "name": "search_docs",
+ "server_label": "docs",
+ "arguments": "{\"query\":\"pricing\"}"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "mcp_123",
+ "function": {
+ "name": "search_docs",
+ "arguments": {
+ "query": "pricing"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API parses file_search_call as an assistant tool call preserving metadata",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "file_search_call",
+ "id": "fs_123",
+ "status": "completed",
+ "queries": ["q1", "q2"]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "fs_123",
+ "function": {
+ "name": "file_search_call",
+ "arguments": {
+ "queries": ["q1", "q2"]
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API parses computer_call as an assistant tool call preserving metadata",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "computer_call",
+ "id": "cc_123",
+ "status": "completed",
+ "action": {
+ "type": "click",
+ "x": 10,
+ "y": 20
+ }
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "cc_123",
+ "function": {
+ "name": "computer_call",
+ "arguments": {
+ "action": {
+ "type": "click",
+ "x": 10,
+ "y": 20
+ }
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OpenAI Responses API handles function_call with unparseable arguments",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "function_call",
+ "call_id": "call_abc",
+ "name": "my_func",
+ "arguments": "{invalid json"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_abc",
+ "function": {
+ "name": "my_func",
+ "arguments": "{invalid json"
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeMessage: lifts function items out of array-based content into tool_calls",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll check the weather for you."
+ },
+ {
+ "type": "function",
+ "id": "call_123",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "New York City"
+ }
+ }
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": "I'll check the weather for you."
+ }
+ ],
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_123",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "New York City"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeMessage: array-based content with function items leaves content empty when every item is a function",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "Berlin"
+ }
+ }
+ },
+ {
+ "type": "function",
+ "id": "call_2",
+ "function": {
+ "name": "search_docs",
+ "arguments": {
+ "query": "foo"
+ }
+ }
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "Berlin"
+ }
+ }
+ },
+ {
+ "type": "function",
+ "id": "call_2",
+ "function": {
+ "name": "search_docs",
+ "arguments": {
+ "query": "foo"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeMessage: array-based content with function items parses function arguments provided as pre-parsed object",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "Berlin",
+ "unit": "celsius"
+ }
+ }
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "Berlin",
+ "unit": "celsius"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeMessage: array-based content with function items parses function arguments provided as JSON-encoded string",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": "{\"location\":\"Berlin\",\"unit\":\"celsius\"}"
+ }
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "Berlin",
+ "unit": "celsius"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeMessage: array-based content with function items lifts function items into tool_calls and keeps non-function items as content",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "function",
+ "id": "fs_1",
+ "function": {
+ "name": "file_search",
+ "arguments": {
+ "queries": ["q"]
+ }
+ }
+ },
+ {
+ "type": "text",
+ "text": "hi"
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": "hi"
+ }
+ ],
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "fs_1",
+ "function": {
+ "name": "file_search",
+ "arguments": {
+ "queries": ["q"]
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeMessage: built-in tool call with explicit name uses rawMessage.name for mcp_call, not the \"mcp_call\" type string",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "mcp_call",
+ "id": "mcp_123",
+ "name": "search_docs",
+ "server_label": "docs",
+ "arguments": "{\"query\":\"pricing\"}"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "mcp_123",
+ "function": {
+ "name": "search_docs",
+ "arguments": {
+ "query": "pricing"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content preserves assistant role when recursing into nested array content with output_text",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "output_text",
+ "text": "Hello! How can I help you?"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "output_text",
+ "text": "Hello! How can I help you?"
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content preserves system role when recursing into nested array content with output_text",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "system",
+ "content": [
+ {
+ "type": "output_text",
+ "text": "You are a helpful assistant."
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "output_text",
+ "text": "You are a helpful assistant."
+ }
+ ],
+ "role": "system"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content preserves role even when defaultRole is different",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "output_text",
+ "text": "Response"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "output_text",
+ "text": "Response"
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content preserves input_text content type",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "content": [
+ {
+ "type": "input_text",
+ "text": "What is the weather?"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "input_text",
+ "text": "What is the weather?"
+ }
+ ],
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content uses defaultRole when message has no role property",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "output_text",
+ "text": "Some text"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "role": "assistant",
+ "content": "{\"type\":\"output_text\",\"text\":\"Some text\"}"
+ }
+ ],
+ "recognized": false
+ }
+ },
+ {
+ "name": "role preservation in nested content handles Anthropic tool result with nested content and overrides role to assistant (tool result)",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool_result",
+ "tool_use_id": "tool_123",
+ "content": [
+ {
+ "type": "text",
+ "text": "Weather is sunny"
+ }
+ ]
+ },
+ "default_role": "tool",
+ "expected": {
+ "messages": [
+ {
+ "content": "Weather is sunny",
+ "role": "assistant (tool result)",
+ "tool_call_id": "tool_123"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content preserves custom/unknown roles",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "custom_agent",
+ "content": [
+ {
+ "type": "text",
+ "text": "Custom response"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": "Custom response"
+ }
+ ],
+ "role": "custom_agent"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content handles stringified structured content in OpenAI-compatible messages: single text block",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "content": "[{\"type\":\"text\",\"text\":\"lets respond inline to each bot to explain that its fine\"}]"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "lets respond inline to each bot to explain that its fine"
+ }
+ ]
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content handles stringified structured content in OpenAI-compatible messages: plain string stays plain",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "content": "plain text"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "plain text"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content handles stringified structured content in OpenAI-compatible messages: invalid JSON falls back to raw string",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "content": "[not json"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "[not json"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content handles stringified structured content in OpenAI-compatible messages: unknown block types stay as raw string",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "content": "[{\"type\":\"schema\",\"definition\":{\"foo\":\"bar\"}}]"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "[{\"type\":\"schema\",\"definition\":{\"foo\":\"bar\"}}]"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content handles stringified structured content in OpenAI-compatible messages: empty structured array parses as empty array",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "content": "[]"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "user",
+ "content": []
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "role preservation in nested content handles LiteLLM choice wrapper and preserves nested role",
+ "call": "normalizeMessage",
+ "input": {
+ "finish_reason": "stop",
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Response from LiteLLM"
+ }
+ ]
+ }
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": "Response from LiteLLM"
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LiteLLM support normalizeMessage should handle LiteLLM choice format",
+ "call": "normalizeMessage",
+ "input": {
+ "finish_reason": "stop",
+ "index": 0,
+ "message": {
+ "annotations": [],
+ "content": "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!",
+ "function_call": null,
+ "role": "assistant",
+ "tool_calls": null
+ },
+ "provider_specific_fields": {}
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "annotations": [],
+ "content": "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!",
+ "function_call": null,
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LiteLLM support normalizeMessage should handle LiteLLM choice with tool calls",
+ "call": "normalizeMessage",
+ "input": {
+ "finish_reason": "stop",
+ "index": 0,
+ "message": {
+ "annotations": [],
+ "content": null,
+ "function_call": null,
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_123",
+ "function": {
+ "name": "get_weather",
+ "arguments": "{\"location\": \"San Francisco\"}"
+ }
+ }
+ ]
+ },
+ "provider_specific_fields": {}
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "annotations": [],
+ "content": "",
+ "function_call": null,
+ "role": "assistant",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_123",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "San Francisco"
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LiteLLM support normalizeMessages should handle LiteLLM response format",
+ "call": "normalizeMessages",
+ "input": {
+ "choices": [
+ {
+ "finish_reason": "stop",
+ "index": 0,
+ "message": {
+ "annotations": [],
+ "content": "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!",
+ "function_call": null,
+ "role": "assistant",
+ "tool_calls": null
+ },
+ "provider_specific_fields": {}
+ }
+ ],
+ "model": "gpt-3.5-turbo",
+ "usage": {
+ "prompt_tokens": 10,
+ "completion_tokens": 25,
+ "total_tokens": 35
+ }
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "annotations": [],
+ "content": "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!",
+ "function_call": null,
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LiteLLM support normalizeMessages should handle LiteLLM response with multiple choices",
+ "call": "normalizeMessages",
+ "input": {
+ "choices": [
+ {
+ "finish_reason": "stop",
+ "index": 0,
+ "message": {
+ "annotations": [],
+ "content": "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!",
+ "function_call": null,
+ "role": "assistant",
+ "tool_calls": null
+ },
+ "provider_specific_fields": {}
+ },
+ {
+ "finish_reason": "stop",
+ "index": 1,
+ "message": {
+ "annotations": [],
+ "content": "Alternative response",
+ "function_call": null,
+ "role": "assistant",
+ "tool_calls": null
+ },
+ "provider_specific_fields": {}
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "annotations": [],
+ "content": "That's wonderful to hear! 😊 I'm here to spread positivity. If you have any questions or need anything, just let me know!",
+ "function_call": null,
+ "role": "assistant"
+ },
+ {
+ "annotations": [],
+ "content": "Alternative response",
+ "function_call": null,
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LiteLLM support normalizeMessages should handle empty LiteLLM response gracefully",
+ "call": "normalizeMessages",
+ "input": {
+ "choices": []
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LiteLLM support Role normalization should preserve system role in array-based content",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": "You are a helpful assistant."
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": "You are a helpful assistant."
+ }
+ ],
+ "role": "system"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LiteLLM support Role normalization should preserve system role in string content",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "system",
+ "content": "You are a helpful assistant."
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant."
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LiteLLM support Role normalization should map known provider roles",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "human",
+ "content": "Hello"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "Hello"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LangChain/LangGraph format normalizeMessage maps human type to user role",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "human",
+ "content": "Hello",
+ "id": "msg-1"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "Hello",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LangChain/LangGraph format normalizeMessage maps ai type to assistant role",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "ai",
+ "content": "Hi there",
+ "id": "msg-2",
+ "meta": {}
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Hi there",
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LangChain/LangGraph format normalizeMessage handles ai type with tool_calls",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "ai",
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "toolu_123",
+ "function": {
+ "name": "search",
+ "arguments": "{\"q\":\"test\"}"
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "toolu_123",
+ "function": {
+ "name": "search",
+ "arguments": {
+ "q": "test"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LangChain/LangGraph format normalizeMessage handles tool type with tool_call_id",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "tool",
+ "content": "Search results: ...",
+ "tool_call_id": "toolu_123"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "Search results: ...",
+ "tool_call_id": "toolu_123",
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "LangChain/LangGraph format normalizeMessage maps context type to system role",
+ "call": "normalizeMessage",
+ "input": {
+ "type": "context",
+ "content": "<system_reminder>Your initial mode is sql.</system_reminder>"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "<system_reminder>Your initial mode is sql.</system_reminder>",
+ "role": "system"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes a single text part into a string content",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "text",
+ "content": "Hello from OTel!"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Hello from OTel!",
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes multiple text parts into an array content",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "text",
+ "content": "First part."
+ },
+ {
+ "type": "text",
+ "content": "Second part."
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": "First part."
+ },
+ {
+ "type": "text",
+ "text": "Second part."
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes tool_call parts into CompatToolCall format",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "tool_call",
+ "id": "call_1",
+ "name": "get_weather",
+ "arguments": {
+ "location": "London"
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "London"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format handles mixed text and tool_call parts",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "text",
+ "content": "Let me check that."
+ },
+ {
+ "type": "tool_call",
+ "id": "call_1",
+ "name": "get_weather",
+ "arguments": {
+ "location": "London"
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Let me check that.",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "London"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format preserves top-level fields via spread",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "finish_reason": "stop",
+ "parts": [
+ {
+ "type": "text",
+ "content": "Done."
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "finish_reason": "stop",
+ "content": "Done.",
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format handles tool_call without id",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "tool_call",
+ "name": "do_thing",
+ "arguments": {
+ "x": 1
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "function": {
+ "name": "do_thing",
+ "arguments": {
+ "x": 1
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format handles tool_call without arguments",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "tool_call",
+ "id": "call_1",
+ "name": "no_args"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "no_args",
+ "arguments": {}
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes role aliases",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "human",
+ "parts": [
+ {
+ "type": "text",
+ "content": "Hi"
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "Hi",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes tool_call_response parts into tool messages from the schema response key",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "tool_call_response",
+ "id": "call_abc",
+ "name": "get_weather",
+ "response": "Sunny, 25°C"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Sunny, 25°C",
+ "tool_call_id": "call_abc",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes tool_call_response parts into tool messages from the spec example result key",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "tool_call_response",
+ "id": "call_abc",
+ "name": "get_weather",
+ "result": "Sunny, 25°C"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Sunny, 25°C",
+ "tool_call_id": "call_abc",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes tool_call_response parts into tool messages from both keys, preferring response",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "tool_call_response",
+ "id": "call_abc",
+ "name": "get_weather",
+ "response": "Sunny, 25°C",
+ "result": "stale"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Sunny, 25°C",
+ "tool_call_id": "call_abc",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes tool_call_response parts into tool messages from a null response falling back to result",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "tool_call_response",
+ "id": "call_abc",
+ "name": "get_weather",
+ "response": null,
+ "result": "Sunny, 25°C"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Sunny, 25°C",
+ "tool_call_id": "call_abc",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format keeps an empty string response over a stale result",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "tool_call_response",
+ "id": "call_abc",
+ "name": "get_weather",
+ "response": "",
+ "result": "stale"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_call_id": "call_abc",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format stringifies non-string tool_call_response response values",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "tool_call_response",
+ "id": "call_1",
+ "name": "get_data",
+ "response": {
+ "temperature": 25,
+ "unit": "C"
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "{\"temperature\":25,\"unit\":\"C\"}",
+ "tool_call_id": "call_1",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format stringifies non-string tool_call_response result values",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "tool_call_response",
+ "id": "call_1",
+ "name": "get_data",
+ "result": {
+ "temperature": 25,
+ "unit": "C"
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "{\"temperature\":25,\"unit\":\"C\"}",
+ "tool_call_id": "call_1",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format parses stringified JSON tool_call arguments",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "tool_call",
+ "id": "call_1",
+ "name": "get_weather",
+ "arguments": "{\"latitude\":45.5,\"longitude\":-73.5}"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "latitude": 45.5,
+ "longitude": -73.5
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format handles a full tool-use conversation",
+ "call": "normalizeMessages",
+ "input": [
+ {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "tool_call",
+ "id": "call_1",
+ "name": "get_weather",
+ "arguments": "{\"location\":\"Montreal\"}"
+ }
+ ],
+ "finish_reason": "tool_call"
+ },
+ {
+ "role": "user",
+ "parts": [
+ {
+ "type": "tool_call_response",
+ "id": "call_1",
+ "name": "get_weather",
+ "result": "-10°C, partly cloudy"
+ }
+ ]
+ }
+ ],
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "finish_reason": "tool_call",
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "get_weather",
+ "arguments": {
+ "location": "Montreal"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ },
+ {
+ "content": "-10°C, partly cloudy",
+ "tool_call_id": "call_1",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format handles empty parts array",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": []
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes reasoning parts into thinking messages without an empty primary",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "reasoning",
+ "content": "The user wants the forecast."
+ },
+ {
+ "type": "text",
+ "content": "Here is the forecast."
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Here is the forecast.",
+ "role": "assistant"
+ },
+ {
+ "content": "The user wants the forecast.",
+ "role": "assistant (thinking)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes a reasoning-only message into just a thinking message",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "reasoning",
+ "content": "Considering options."
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Considering options.",
+ "role": "assistant (thinking)"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes server_tool_call parts into tool calls",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "server_tool_call",
+ "id": "st_1",
+ "name": "web_search",
+ "server_tool_call": {
+ "type": "web_search",
+ "query": "weather in Montreal"
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "st_1",
+ "function": {
+ "name": "web_search",
+ "arguments": {
+ "type": "web_search",
+ "query": "weather in Montreal"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes server_tool_call_response parts into tool messages",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "parts": [
+ {
+ "type": "server_tool_call_response",
+ "id": "st_1",
+ "server_tool_call_response": {
+ "type": "web_search",
+ "status": "completed"
+ }
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "{\"type\":\"web_search\",\"status\":\"completed\"}",
+ "tool_call_id": "st_1",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes image blob parts into data-URI image items",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "text",
+ "content": "What is in this picture?"
+ },
+ {
+ "type": "blob",
+ "modality": "image",
+ "mime_type": "image/png",
+ "content": "aGVsbG8="
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "What is in this picture?",
+ "role": "user"
+ },
+ {
+ "content": [
+ {
+ "type": "image",
+ "image": "data:image/png;base64,aGVsbG8="
+ }
+ ],
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes image uri parts into image items",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "uri",
+ "modality": "image",
+ "mime_type": "image/jpeg",
+ "uri": "https://example.com/cat.jpg"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": [
+ {
+ "type": "image",
+ "image": "https://example.com/cat.jpg"
+ }
+ ],
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes a non-image blob into a text marker",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "blob",
+ "modality": "audio",
+ "mime_type": "audio/mp3",
+ "content": "aGVsbG8="
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "[audio]",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes a non-image uri into a text marker",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "uri",
+ "modality": "document",
+ "mime_type": "application/pdf",
+ "uri": "https://example.com/doc.pdf"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "[document: https://example.com/doc.pdf]",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes a file reference into a text marker",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "file",
+ "modality": "document",
+ "file_id": "file_123"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "[file: file_123]",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes compaction parts into its summary when present",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "compaction",
+ "id": "c1",
+ "content": "Earlier turns covered pricing."
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "Earlier turns covered pricing.",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "OTel parts format normalizes compaction parts into a marker when the summary is absent",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "compaction",
+ "id": "c1"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "[conversation compacted]",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "production payload regressions null input carries no message",
+ "call": "normalizeMessages",
+ "input": null,
+ "default_role": "user",
+ "expected": {
+ "messages": [],
+ "recognized": true
+ }
+ },
+ {
+ "name": "production payload regressions number/boolean input carries no message",
+ "call": "normalizeMessages",
+ "input": 42,
+ "default_role": "user",
+ "expected": {
+ "messages": [],
+ "recognized": true
+ }
+ },
+ {
+ "name": "production payload regressions number/boolean input carries no message #2",
+ "call": "normalizeMessages",
+ "input": true,
+ "default_role": "user",
+ "expected": {
+ "messages": [],
+ "recognized": true
+ }
+ },
+ {
+ "name": "production payload regressions a single-field {content} object inherits the default role",
+ "call": "normalizeMessages",
+ "input": {
+ "content": "hi"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "hi",
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "production payload regressions a single-field {content} object inherits the default role #2",
+ "call": "normalizeMessages",
+ "input": {
+ "content": "hi"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "hi",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "production payload regressions an empty top-level tool_calls array adds no synthetic message",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "thinking",
+ "thinking": "x"
+ },
+ {
+ "type": "text",
+ "text": "done"
+ }
+ ],
+ "tool_calls": []
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "x",
+ "role": "assistant (thinking)"
+ },
+ {
+ "content": "done",
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "production payload regressions OTel parts whose text lives under an unexpected key degrade to empty content, not [null]",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "user",
+ "parts": [
+ {
+ "type": "text",
+ "text": "hi"
+ }
+ ]
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "role": "user"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "recipe normalization improvements extracts single-field {text}/{message} wrappers instead of stringifying them",
+ "call": "normalizeMessages",
+ "input": {
+ "text": "hello"
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "hello",
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "recipe normalization improvements empties null content and drops an empty tool_calls array",
+ "call": "normalizeMessages",
+ "input": {
+ "role": "assistant",
+ "content": null,
+ "tool_calls": []
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "role": "assistant",
+ "content": ""
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "recipe normalization improvements preserves and canonicalizes a tool_call lacking the type marker",
+ "call": "normalizeMessages",
+ "input": {
+ "role": "assistant",
+ "content": null,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "caller": {
+ "type": "direct"
+ },
+ "index": 0,
+ "function": {
+ "name": "send_email",
+ "arguments": "{\"to\":\"[email protected]\"}"
+ }
+ }
+ ]
+ },
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "call_1",
+ "function": {
+ "name": "send_email",
+ "arguments": {
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "recipe normalization improvements flattens a doubly-nested message array instead of stringifying it",
+ "call": "normalizeMessages",
+ "input": [
+ [
+ {
+ "role": "system",
+ "content": "a"
+ },
+ {
+ "role": "user",
+ "content": "b"
+ }
+ ]
+ ],
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "system",
+ "content": "a"
+ },
+ {
+ "role": "user",
+ "content": "b"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "recipe normalization improvements parses typed agent items (tool_call/tool_result) into real tool messages",
+ "call": "normalizeMessages",
+ "input": [
+ {
+ "type": "tool_call",
+ "callId": "c1",
+ "name": "getWeather",
+ "arguments": {
+ "city": "NYC"
+ }
+ },
+ {
+ "type": "tool_result",
+ "callId": "c1",
+ "name": "getWeather",
+ "output": {
+ "tempF": 71
+ }
+ }
+ ],
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "id": "c1",
+ "function": {
+ "name": "getWeather",
+ "arguments": {
+ "city": "NYC"
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ },
+ {
+ "content": "{\"tempF\":71}",
+ "tool_call_id": "c1",
+ "role": "tool"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "delegates to the recipe normalizer normalizeMessage normalizes a single message",
+ "call": "normalizeMessage",
+ "input": {
+ "role": "assistant",
+ "content": "hi"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "assistant",
+ "content": "hi"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "delegates to the recipe normalizer normalizeMessages prepends an available-tools pseudo-message when tools are passed",
+ "call": "normalizeMessages",
+ "input": {
+ "role": "user",
+ "content": "hi"
+ },
+ "default_role": "user",
+ "tools": [
+ {
+ "name": "search"
+ }
+ ],
+ "expected": {
+ "messages": [
+ {
+ "role": "available tools",
+ "content": "",
+ "tools": [
+ {
+ "name": "search"
+ }
+ ]
+ },
+ {
+ "role": "user",
+ "content": "hi"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "delegates to the recipe normalizer normalizeMessages carries no message for nullish/scalar input",
+ "call": "normalizeMessages",
+ "input": null,
+ "default_role": "user",
+ "expected": {
+ "messages": [],
+ "recognized": true
+ }
+ },
+ {
+ "name": "delegates to the recipe normalizer normalizeMessages carries no message for nullish/scalar input #2",
+ "call": "normalizeMessages",
+ "input": 42,
+ "default_role": "user",
+ "expected": {
+ "messages": [],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeMessages recognition reports a recognized conversation alongside its messages",
+ "call": "normalizeMessages",
+ "input": {
+ "role": "assistant",
+ "content": "hi"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "assistant",
+ "content": "hi"
+ }
+ ],
+ "recognized": true
+ }
+ },
+ {
+ "name": "normalizeMessages recognition reports opaque state as unrecognized and salvages it",
+ "call": "normalizeMessages",
+ "input": {
+ "file_path": "src/index.ts"
+ },
+ "default_role": "user",
+ "expected": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "{\"file_path\":\"src/index.ts\"}"
+ }
+ ],
+ "recognized": false
+ }
+ },
+ {
+ "name": "thinking blocks alongside typed function blocks renders the thought as thinking and keeps the sibling tool call typed",
+ "call": "normalizeMessages",
+ "input": [
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "thinking",
+ "thinking": "Two rapid clicks, check the events."
+ },
+ {
+ "type": "function",
+ "function": {
+ "name": "get_events_around",
+ "arguments": {
+ "rec_t": 320
+ }
+ }
+ }
+ ]
+ }
+ ],
+ "default_role": "assistant",
+ "expected": {
+ "messages": [
+ {
+ "content": "Two rapid clicks, check the events.",
+ "role": "assistant (thinking)"
+ },
+ {
+ "content": "",
+ "tool_calls": [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_events_around",
+ "arguments": {
+ "rec_t": 320
+ }
+ }
+ }
+ ],
+ "role": "assistant"
+ }
+ ],
+ "recognized": true
+ }
+ }
+ ]
+}