🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import type { NormalizerContext, ProviderAdapter } from "../types";
import {
removeNullFields,
stringifyToolResultContent,
parseMetadata,
isRichToolResult,
normalizeToolDefinitionsForChatMl,
attachToolDefinitionsToMessages,
} from "../helpers";
import { z } from "zod";
/**
* AI SDK v5 Adapter
*
* Handles traces from Vercel AI SDK v5 with any model provider
*
* Key characteristics:
* - Observation names: ai.generateText.doGenerate, ai.generateObject.doGenerate
* - Metadata: ai.operationId, ai.model.provider, ai.model.id
* - Message structure: {role, content: [{type, text/toolCallId/...}]}
* - Tool calls: {type: "tool-call", toolCallId, toolName, input|args}
* - Tool results: {type: "tool-result", toolCallId, toolName, output|result}
*/
// Message with AI SDK v5 tool-call content structure
const AISDKToolCallMessageSchema = z.looseObject({
role: z.string(),
content: z.array(
z.looseObject({
type: z.literal("tool-call"),
toolCallId: z.string(),
toolName: z.string(),
}),
),
});
// Message with AI SDK v5 tool-result content structure
const AISDKToolResultMessageSchema = z.looseObject({
role: z.literal("tool"),
content: z.array(
z.looseObject({
type: z.literal("tool-result"),
toolCallId: z.string(),
toolName: z.string(),
}),
),
});
// Message array with AI SDK v5 patterns
const AISDKMessagesArraySchema = z
.array(
z.looseObject({
role: z.string(),
content: z
.union([z.string(), z.array(z.looseObject({ type: z.string() }))])
.optional(),
}),
)
.refine(
(messages) => {
// Must have at least one message with AI SDK v5 structured content
return messages.some(
(msg) =>
Array.isArray(msg.content) &&
msg.content.some(
(item: unknown) =>
(item &&
typeof item === "object" &&
"type" in item &&
(item as Record<string, unknown>).type === "tool-call") ||
(item as Record<string, unknown>).type === "tool-result",
),
);
},
{ message: "Must have AI SDK v5 tool-call or tool-result patterns" },
);
// Array of raw tool call objects, format as seen on OUTPUTs
// Example: [{toolCallId: "...", toolName: "...", input: {...}}, ...]
const AISDKRawToolCallArraySchema = z
.array(
z.looseObject({
toolCallId: z.string(),
toolName: z.string(),
// Can have either 'input' (OpenAI) or 'args' (Bedrock)
}),
)
.refine((arr) => arr.length > 0, {
message: "Must have at least one tool call",
});
// normalize a single AI SDK v5 message to ChatML format
// we don't want additional fields here to get clean rendering
function normalizeMessage(msg: unknown): Record<string, unknown> {
if (!msg || typeof msg !== "object") return {};
let working = msg as Record<string, unknown>;
// Strip provider-specific metadata (Bedrock, OpenAI, etc.)
const {
providerMetadata: _providerMetadata,
providerOptions: _providerOptions,
...withoutProviderFields
} = working;
working = withoutProviderFields;
let normalized = removeNullFields(working);
// Normalize content: [{type: "text", text: "..."}] → string
if (
normalized.content &&
Array.isArray(normalized.content) &&
normalized.content.length > 0
) {
// all text already or do we need to normalize further?
const allTextItems = normalized.content.every(
(item: unknown) =>
item &&
typeof item === "object" &&
(item as Record<string, unknown>).type === "text" &&
typeof (item as Record<string, unknown>).text === "string",
);
if (allTextItems && normalized.content.length === 1) {
// Single text item: extract the text
const firstItem = normalized.content[0] as Record<string, unknown>;
normalized.content = firstItem.text;
} else if (allTextItems && normalized.content.length > 1) {
// Multiple text items: concatenate
const texts = normalized.content.map(
(item: unknown) => (item as Record<string, unknown>).text,
);
normalized.content = texts.join("");
} else {
// Mixed content or tool-calls/tool-results: normalize each item
normalized.content = normalized.content.map((item: unknown) => {
if (!item || typeof item !== "object") return item;
const contentItem = item as Record<string, unknown>;
// Handle tool-call content item
// {type: "tool-call", toolCallId, toolName, input|args, providerOptions}
if (contentItem.type === "tool-call") {
const { providerOptions: _po, ...cleanedItem } = contentItem;
// Normalize args|input to arguments
const args = contentItem.args ?? contentItem.input;
return {
...cleanedItem,
arguments:
typeof args === "string" ? args : JSON.stringify(args ?? {}),
};
}
// Handle tool-result content item
// {type: "tool-result", toolCallId, toolName, output|result}
if (contentItem.type === "tool-result") {
// Normalize output|result to content
const resultValue = contentItem.output ?? contentItem.result;
// If result has nested {type: "text", value: "..."}, extract value
if (
resultValue &&
typeof resultValue === "object" &&
!Array.isArray(resultValue) &&
(resultValue as Record<string, unknown>).type === "text" &&
"value" in (resultValue as Record<string, unknown>)
) {
return {
...contentItem,
content: (resultValue as Record<string, unknown>).value,
};
}
// Otherwise stringify the result
return {
...contentItem,
content:
typeof resultValue === "string"
? resultValue
: JSON.stringify(resultValue ?? ""),
};
}
return contentItem;
});
// Convert tool-call content items to tool_calls array
if (Array.isArray(normalized.content)) {
const toolCallItems = normalized.content.filter(
(item: unknown) =>
item &&
typeof item === "object" &&
(item as Record<string, unknown>).type === "tool-call",
);
if (toolCallItems.length > 0) {
normalized.tool_calls = toolCallItems.map((item: unknown) => {
const tc = item as Record<string, unknown>;
return {
id: tc.toolCallId,
name: tc.toolName,
arguments: tc.arguments,
type: "function",
};
});
// Remove tool-call items from content, keep only text items
const textItems = normalized.content.filter(
(item: unknown) =>
item &&
typeof item === "object" &&
(item as Record<string, unknown>).type === "text",
);
if (textItems.length > 0) {
const texts = textItems.map(
(item: unknown) => (item as Record<string, unknown>).text ?? "",
);
normalized.content = texts.join("");
} else {
// No text content, remove content field
delete normalized.content;
}
}
}
// Handle tool-result content items for tool role messages
// Only process if content is still an array (wasn't converted to string above)
if (Array.isArray(normalized.content)) {
const toolResultItems = normalized.content.filter(
(item: unknown) =>
item &&
typeof item === "object" &&
(item as Record<string, unknown>).type === "tool-result",
);
if (toolResultItems.length > 0 && normalized.role === "tool") {
// For single tool result, extract tool_call_id and content
if (toolResultItems.length === 1) {
const result = toolResultItems[0] as Record<string, unknown>;
normalized.tool_call_id = result.toolCallId;
normalized.content = result.content;
} else {
// Multiple tool results: keep as array (less common)
normalized.content = toolResultItems;
}
}
}
}
}
// For tool messages with rich object content, spread into message
// so it goes to json passthrough field and renders as PrettyJsonView.
// Rich = nested structure OR 3+ keys. Simple <=2 scalar keys.
if (
normalized.role === "tool" &&
typeof normalized.content === "object" &&
normalized.content !== null &&
!Array.isArray(normalized.content)
) {
if (isRichToolResult(normalized.content)) {
// Rich object: spread for table rendering
const { content, ...rest } = normalized;
return { ...rest, ...content };
}
// Simple object: stringify for text rendering
normalized.content = stringifyToolResultContent(normalized.content);
}
return normalized;
}
/**
* Split tool result messages with multiple results into separate messages
* AI SDK can have: {role: "tool", content: [{type: "tool-result", ...}, {type: "tool-result", ...}]}
* ChatML expects: [{role: "tool", tool_call_id: "...", content: "..."}, ...]
*/
function splitToolResultMessages(messages: unknown[]): unknown[] {
const result: unknown[] = [];
for (const msg of messages) {
if (!msg || typeof msg !== "object") {
result.push(msg);
continue;
}
const message = msg as Record<string, unknown>;
// Check if this is a tool message with array content containing multiple tool-result items
if (
message.role === "tool" &&
Array.isArray(message.content) &&
message.content.length > 1 &&
message.content.every(
(item: unknown) =>
item &&
typeof item === "object" &&
(item as Record<string, unknown>).type === "tool-result",
)
) {
// Split into separate messages, one per tool result
for (const item of message.content) {
const toolResult = item as Record<string, unknown>;
result.push({
role: "tool",
tool_call_id: toolResult.toolCallId,
content: toolResult.content,
});
}
} else {
result.push(message);
}
}
return result;
}
function preprocessData(data: unknown, ctx?: NormalizerContext): unknown {
if (!data) return data;
// Extract tools from context metadata (observation.metadata.tools)
let toolsFromContext: Array<Record<string, unknown>> | undefined;
if (ctx?.metadata && typeof ctx.metadata === "object") {
const meta = ctx.metadata as Record<string, unknown>;
toolsFromContext = normalizeToolDefinitionsForChatMl(meta.tools);
}
// Handle wrapped format with messages and tools
// {messages: [...], tools: [...]} or {messages: [...], providerOptions: {...}}
if (typeof data === "object" && !Array.isArray(data) && "messages" in data) {
const obj = data as Record<string, unknown>;
const messagesArray = obj.messages as unknown[];
if (Array.isArray(messagesArray)) {
let tools: Array<Record<string, unknown>> | undefined;
if (Array.isArray(obj.tools)) {
tools = normalizeToolDefinitionsForChatMl(obj.tools);
}
const normalized = messagesArray.map(normalizeMessage);
const split = splitToolResultMessages(normalized);
// Attach tools to messages if present
if (tools && tools.length > 0) {
return attachToolDefinitionsToMessages(split, tools);
}
return split;
}
}
if (Array.isArray(data)) {
// is an array of raw tool call objects?
// Example: [{toolCallId: "...", toolName: "...", input: {...}}, ...]
const rawToolCallArrayResult = AISDKRawToolCallArraySchema.safeParse(data);
if (rawToolCallArrayResult.success) {
// Convert raw tool calls into a single assistant message with tool_calls
const toolCalls = data.map((item) => {
const tc = item as Record<string, unknown>;
const args = tc.args ?? tc.input;
return {
id: tc.toolCallId,
name: tc.toolName,
arguments:
typeof args === "string" ? args : JSON.stringify(args ?? {}),
type: "function",
};
});
const assistantMessage: Record<string, unknown> = {
role: "assistant",
content: "",
tool_calls: toolCalls,
};
// Attach tools from context if available
if (toolsFromContext && toolsFromContext.length > 0) {
assistantMessage.tools = toolsFromContext;
}
return [assistantMessage];
}
// otherwise it's an array of messages
const normalized = data.map(normalizeMessage);
const split = splitToolResultMessages(normalized);
// Attach tools from context metadata if present
if (toolsFromContext && toolsFromContext.length > 0) {
return attachToolDefinitionsToMessages(split, toolsFromContext);
}
return split;
}
// if it's not an array but just a single message
if (typeof data === "object" && "role" in data) {
return normalizeMessage(data);
}
return data;
}
export const aisdkAdapter: ProviderAdapter = {
id: "aisdk",
detect(ctx: NormalizerContext): boolean {
const meta = parseMetadata(ctx.metadata);
// EXPLICIT: Framework hint
if (ctx.framework === "aisdk" || ctx.framework === "aisdk-v5") return true;
// STRONG INDICATORS: AI SDK v5 telemetry markers
if (meta && typeof meta === "object") {
if (
"scope" in meta &&
meta.scope !== null &&
typeof meta.scope === "object"
) {
const scope = meta.scope as Record<string, unknown>;
if (scope.name === "ai") return true;
}
if (meta["scope.name"] === "ai") return true;
if ("attributes" in meta && typeof meta.attributes === "object") {
const attrs = meta.attributes as Record<string, unknown> | null;
if (
attrs &&
typeof attrs["operation.name"] === "string" &&
attrs["operation.name"].startsWith("ai.")
) {
return true;
}
}
const flatOperationName = meta["attributes.operation.name"];
if (
typeof flatOperationName === "string" &&
flatOperationName.startsWith("ai.")
) {
return true;
}
const flatAiOperationId = meta["attributes.ai.operationId"];
if (
typeof flatAiOperationId === "string" &&
flatAiOperationId.startsWith("ai.")
) {
return true;
}
}
// STRUCTURAL: Schema-based detection (for edge cases without metadata)
if (AISDKToolCallMessageSchema.safeParse(ctx.metadata).success) return true;
if (AISDKToolResultMessageSchema.safeParse(ctx.metadata).success)
return true;
if (AISDKMessagesArraySchema.safeParse(ctx.metadata).success) return true;
if (AISDKToolCallMessageSchema.safeParse(ctx.data).success) return true;
if (AISDKToolResultMessageSchema.safeParse(ctx.data).success) return true;
if (AISDKMessagesArraySchema.safeParse(ctx.data).success) return true;
// Raw tool call array detection (OUTPUT format)
if (AISDKRawToolCallArraySchema.safeParse(ctx.data).success) return true;
return false;
},
preprocess(
data: unknown,
_kind: "input" | "output",
ctx: NormalizerContext,
): unknown {
return preprocessData(data, ctx);
},
};