🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
// Mock the problematic @langfuse/shared import before importing our functions
vi.mock("@langfuse/shared", async () => {
const actual = await vi.importActual("@langfuse/shared");
const { z } = (await vi.importActual("zod")) as any;
const OpenAITextContentPart = z.object({
type: z.literal("text"),
text: z.string(),
});
const OpenAIImageContentPart = z.object({
type: z.literal("image_url"),
image_url: z.union([
z.string(),
z.object({
url: z.string(),
detail: z.enum(["auto", "low", "high"]).optional(),
}),
]),
});
return {
...actual,
ChatMessageRole: {
System: "system",
Developer: "developer",
User: "user",
Assistant: "assistant",
Tool: "tool",
Model: "model",
},
ChatMessageType: {
PublicAPICreated: "public-api-created",
AssistantToolCall: "assistant-tool-call",
ToolResult: "tool-result",
Placeholder: "placeholder",
System: "system",
},
OpenAIToolSchema: z.object({
type: z.literal("function"),
function: z.object({
name: z.string(),
description: z.string().optional(),
parameters: z.any().optional(),
}),
}),
BaseChatMlMessageSchema: z
.object({
role: z.string().optional(),
name: z.string().optional(),
content: z
.union([
z.record(z.string(), z.any()),
z.string(),
z.array(z.any()),
z.any(), // Simplified - was OpenAIContentSchema
])
.nullish(),
audio: z.any().optional(),
additional_kwargs: z.record(z.string(), z.any()).optional(),
tools: z.array(z.any()).optional(),
tool_calls: z.array(z.any()).optional(),
tool_call_id: z.string().optional(),
})
.loose(),
isOpenAITextContentPart: (content: any) => {
return OpenAITextContentPart.safeParse(content).success;
},
isOpenAIImageContentPart: (content: any) => {
return OpenAIImageContentPart.safeParse(content).success;
},
};
});
import { normalizeInput, normalizeOutput } from "@langfuse/shared";
import { convertChatMlToPlayground } from "./playgroundConverter";
import { extractTools } from "./extractTools";
describe("Playground Jump Full Pipeline", () => {
it("should convert full trace with tool calls through mapper and playground conversion", () => {
// Same data as integration test
const observationName = "OpenAI-generation";
const metadata = {
language: "en",
resourceAttributes: {
"telemetry.sdk.language": "python",
"telemetry.sdk.name": "opentelemetry",
"telemetry.sdk.version": "1.34.1",
"deployment.environment": "prod",
},
scope: {
name: "langfuse-sdk",
version: "3.2.3",
attributes: {
public_key: "pk-lf-1234567890",
},
},
};
const input = {
tools: [
{
type: "function",
function: {
name: "get_user_info",
description: "Retrieve user contact information",
parameters: {
type: "object",
properties: {},
required: [],
},
},
},
{
type: "function",
function: {
name: "query_database",
description: "Query the knowledge base for information",
parameters: {
type: "object",
properties: {
query: {
type: "string",
description: "The search query",
},
lang: {
type: "string",
description: "Language code",
},
},
required: ["query", "lang"],
},
},
},
],
messages: [
{
role: "system",
content:
"You are a helpful assistant. Use tools to gather information before responding.",
},
{
role: "assistant",
content: "",
tool_calls: [
{
id: "call_user_info_001",
type: "function",
function: {
name: "get_user_info",
arguments: {},
},
},
],
},
{
role: "tool",
content: JSON.stringify({
userInfo: {
displayName: "Test User",
email: "[email protected]",
},
}),
tool_call_id: "call_user_info_001",
},
{
role: "user",
content: "How do I configure the API settings?",
},
{
role: "assistant",
content: "",
tool_calls: [
{
id: "call_query_db_002",
type: "function",
function: {
name: "query_database",
arguments: {
query: "configure API settings",
lang: "en",
},
},
},
],
},
{
role: "tool",
content:
"To configure API settings, go to Settings > API Configuration. Enter your API key and select the appropriate region.",
name: "query_database",
tool_call_id: "call_query_db_002",
},
],
};
// const output = {
// role: "assistant",
// content:
// "To configure the API settings, follow these steps:\n\n1. Navigate to **Settings\n2. Enter your API key\n3. Select the appropriate region.",
// };
const ctx = { metadata, observationName };
const inResult = normalizeInput(input, ctx);
expect(inResult.success).toBe(true);
if (!inResult.data) throw new Error("Expected data to be defined");
// convert all messages to playground format
const playgroundMessages = inResult.data
.map(convertChatMlToPlayground)
.filter((msg) => msg !== null);
// Verify we got messages (if length is 0, button would be disabled!)
expect(playgroundMessages.length).toBeGreaterThan(0);
expect(playgroundMessages).toHaveLength(6);
// Verify first assistant message with tool call
const assistantWithToolCall1 = playgroundMessages[1];
expect(assistantWithToolCall1?.type).toBe("assistant-tool-call");
if (assistantWithToolCall1?.type === "assistant-tool-call") {
expect(assistantWithToolCall1.toolCalls).toHaveLength(1);
expect(assistantWithToolCall1.toolCalls[0].id).toBe("call_user_info_001");
expect(assistantWithToolCall1.toolCalls[0].name).toBe("get_user_info");
expect(assistantWithToolCall1.toolCalls[0].args).toEqual({});
}
// Verify first tool result
const toolResult1 = playgroundMessages[2];
expect(toolResult1?.type).toBe("tool-result");
if (toolResult1?.type === "tool-result") {
expect(toolResult1.toolCallId).toBe("call_user_info_001");
expect(typeof toolResult1.content).toBe("string");
expect(toolResult1.content).toContain("Test User");
}
// Verify second assistant message with tool call (object args)
const assistantWithToolCall2 = playgroundMessages[4];
expect(assistantWithToolCall2?.type).toBe("assistant-tool-call");
if (assistantWithToolCall2?.type === "assistant-tool-call") {
expect(assistantWithToolCall2.toolCalls).toHaveLength(1);
expect(assistantWithToolCall2.toolCalls[0].id).toBe("call_query_db_002");
expect(assistantWithToolCall2.toolCalls[0].name).toBe("query_database");
// Object arguments should be parsed back to object
expect(assistantWithToolCall2.toolCalls[0].args).toEqual({
query: "configure API settings",
lang: "en",
});
}
// Verify second tool result
const toolResult2 = playgroundMessages[5];
expect(toolResult2?.type).toBe("tool-result");
if (toolResult2?.type === "tool-result") {
expect(toolResult2.toolCallId).toBe("call_query_db_002");
expect(typeof toolResult2.content).toBe("string");
expect(toolResult2.content).toContain("Settings > API Configuration");
}
});
it("should handle Langchain Gemini-style tool definitions embedded in messages", () => {
// Langchain Gemini format embeds tool definitions as messages with role="tool"
// This test verifies that:
// 1. Tool definition messages are filtered out of the message list
// 2. Tool definitions are extracted and available for the playground
const observationName = "VertexGemini";
const metadata = {
ls_provider: "google_vertexai",
ls_model_name: "gemini-2.5-flash",
};
const input = [
{
role: "system",
content: "You are a helpful assistant with access to tools.",
},
{
role: "model",
content: [
{
type: "text",
text: "Hello! How can I help you today?",
},
],
},
{
role: "tool",
content: {
type: "function",
function: {
name: "transition_to_next_stage",
description:
"Use this function to transition to the next stage when conditions are met",
parameters: {
properties: {
reason: {
description: "Explanation of why transitioning",
type: "string",
},
},
required: ["reason"],
type: "object",
},
},
},
},
{
role: "tool",
content: {
type: "function",
function: {
name: "get_user_info",
description: "Retrieve user contact information",
parameters: {
type: "object",
properties: {},
required: [],
},
},
},
},
{
role: "user",
content: "What can you do?",
},
];
const ctx = { metadata, observationName };
const inResult = normalizeInput(input, ctx);
expect(inResult.success).toBe(true);
if (!inResult.data) throw new Error("Expected data to be defined");
// Convert all messages to playground format
const playgroundMessages = inResult.data
.map(convertChatMlToPlayground)
.filter((msg) => msg !== null);
// Should have 3 real messages (system, model, user)
// Tool definition messages should be filtered out by the converter
expect(playgroundMessages.length).toBe(3);
// Filter out placeholder messages for role testing
const regularMessages = playgroundMessages.filter(
(msg) => msg.type !== "placeholder",
);
// Verify message roles
expect(regularMessages[0]?.role).toBe("system");
expect(regularMessages[1]?.role).toBe("model");
expect(regularMessages[2]?.role).toBe("user");
// All should have type public-api-created (regular messages)
expect(playgroundMessages[0]?.type).toBe("public-api-created");
expect(playgroundMessages[1]?.type).toBe("public-api-created");
expect(playgroundMessages[2]?.type).toBe("public-api-created");
// IMPORTANT: Test that tools are extracted from the Langchain Gemini input format
// The parseTools function in JumpToPlaygroundButton.tsx needs to handle Langchain Gemini format
// where tool definitions are embedded as messages with role="tool"
// We need to test parseTools behavior directly
// For now, manually verify the structure exists
const toolMessages = input.filter(
(msg: any) =>
msg.role === "tool" &&
typeof msg.content === "object" &&
msg.content?.type === "function" &&
msg.content?.function,
);
// Verify the structure that parseTools should handle
expect(toolMessages.length).toBe(2);
expect((toolMessages[0].content as any).function.name).toBe(
"transition_to_next_stage",
);
expect((toolMessages[1].content as any).function.name).toBe(
"get_user_info",
);
});
it("should extract tools from Gemini format using extractTools utility", () => {
const geminiInput = [
{
role: "system",
content: "You are a helpful assistant.",
},
{
role: "tool",
content: {
type: "function",
function: {
name: "get_weather",
description: "Get the current weather in a location",
parameters: {
type: "object",
properties: {
location: {
type: "string",
description: "The city name",
},
},
required: ["location"],
},
},
},
},
{
role: "tool",
content: {
type: "function",
function: {
name: "search_web",
description: "Search the web for information",
parameters: {
type: "object",
properties: {
query: {
type: "string",
description: "Search query",
},
},
required: ["query"],
},
},
},
},
];
// Test the exported extractTools utility directly
const tools = extractTools(geminiInput);
expect(tools.length).toBe(2);
expect(tools[0]).toEqual({
id: expect.any(String),
name: "get_weather",
description: "Get the current weather in a location",
parameters: {
type: "object",
properties: {
location: {
type: "string",
description: "The city name",
},
},
required: ["location"],
},
});
expect(tools[1].name).toBe("search_web");
});
it("should stringify array content in tool messages for playground (eg as from langgraph)", () => {
// CodeMirror expects string but got array in tool result content
const input = [
{
role: "tool",
content: [{ url: "https://example.com", title: "Example" }],
tool_call_id: "call_123",
},
];
const inResult = normalizeInput(input, { framework: "langgraph" });
expect(inResult.success).toBe(true);
const playgroundMsg = convertChatMlToPlayground(inResult.data![0]);
expect(playgroundMsg?.type).toBe("tool-result");
// Array must be stringified, not left as object
if (playgroundMsg && "content" in playgroundMsg) {
expect(typeof playgroundMsg.content).toBe("string");
expect(playgroundMsg.content).toBe(
'[{"url":"https://example.com","title":"Example"}]',
);
}
});
it("should stringify multimodal array content in tool results", () => {
// This format appears in some OpenAI traces where tool results have
// complex nested structures with type/text fields
const input = {
tools: [
{
type: "function",
function: {
name: "get_user_contact_information",
description: "Get user contact info",
parameters: {},
},
},
],
messages: [
{
role: "assistant",
content: "",
tool_calls: [
{
id: "call_user_info",
type: "function",
function: {
name: "get_user_contact_information",
arguments: {},
},
},
],
},
{
role: "tool",
content: [
{
type: "text",
text: {
userInformation: {
firstName: "John",
lastName: "Doe",
email: "[email protected]",
},
},
},
],
tool_call_id: "call_user_info",
},
],
};
const ctx = { framework: "openai" };
const inResult = normalizeInput(input, ctx);
expect(inResult.success).toBe(true);
// Convert all messages to playground format
const playgroundMessages = inResult
.data!.map(convertChatMlToPlayground)
.filter((msg) => msg !== null);
// Find the tool result message
const toolResult = playgroundMessages.find(
(msg) => msg?.type === "tool-result",
);
expect(toolResult).toBeDefined();
if (toolResult && "content" in toolResult) {
// CRITICAL: Content must be stringified, not left as array/object
// Otherwise CodeMirror will throw "value must be typeof string but got object"
expect(typeof toolResult.content).toBe("string");
// Verify it's valid JSON string
const parsed = JSON.parse(toolResult.content);
expect(Array.isArray(parsed)).toBe(true);
expect(parsed[0].type).toBe("text");
expect(parsed[0].text.userInformation.firstName).toBe("John");
}
});
it("should extract text from Vercel AI SDK content array format", () => {
// Vercel AI SDK format: content as array with type/text structure
const input = {
messages: [
{
role: "assistant",
content: [{ type: "text", text: "Hello world" }],
},
],
};
const inResult = normalizeInput(input, {});
expect(inResult.success).toBe(true);
const playgroundMsg = convertChatMlToPlayground(inResult.data![0]);
// Should extract text, not stringify array
expect(
playgroundMsg && "content" in playgroundMsg && playgroundMsg.content,
).toBe("Hello world");
expect(
playgroundMsg && "content" in playgroundMsg && playgroundMsg.content,
).not.toContain("[{");
});
it("should extract tools from Microsoft Agent Framework metadata", () => {
// Microsoft Agent Framework stores tools in metadata.attributes["gen_ai.tool.definitions"]
const input = [
{
role: "user",
parts: [
{
type: "text",
content: "What's the weather like in Portland?",
},
],
},
];
const metadata = {
attributes: {
"gen_ai.provider.name": "microsoft.agent_framework",
"gen_ai.operation.name": "invoke_agent",
"gen_ai.tool.definitions": [
{
type: "function",
function: {
name: "get_weather",
description: "Get the weather for a given location.",
parameters: {
properties: {
location: {
description: "The location to get the weather for.",
title: "Location",
type: "string",
},
},
required: ["location"],
title: "get_weather_input",
type: "object",
},
},
},
{
type: "function",
function: {
name: "get_time",
description: "Get the current time in a timezone.",
parameters: {
properties: {
timezone: {
description: "IANA timezone identifier",
type: "string",
},
},
required: ["timezone"],
type: "object",
},
},
},
],
},
scope: {
name: "agent_framework",
version: "1.0.0b251007",
},
};
// Test extractTools with metadata parameter
const tools = extractTools(input, metadata);
expect(tools.length).toBe(2);
expect(tools[0]).toEqual({
id: expect.any(String),
name: "get_weather",
description: "Get the weather for a given location.",
parameters: {
properties: {
location: {
description: "The location to get the weather for.",
title: "Location",
type: "string",
},
},
required: ["location"],
title: "get_weather_input",
type: "object",
},
});
expect(tools[1].name).toBe("get_time");
expect(tools[1].description).toBe("Get the current time in a timezone.");
});
it("should preserve rich tool results when jumping to playground", () => {
// Regression test: tool results with many keys(6+) are spread into json passthrough by adapters
// (for PrettyJsonView table rendering in trace view). playgroundConverter must fallback
// to jsonData when content is undefined to preserve tool result data for playground.
const input = {
messages: [
{
role: "assistant",
content: "",
tool_calls: [
{
id: "c1",
type: "function",
function: { name: "verify", arguments: {} },
},
],
},
{
role: "tool",
content: {
PatientNo: "123",
First: "J",
Last: "D",
Email: "[email protected]",
Mobile: "555",
Active: true,
},
tool_call_id: "c1",
},
],
};
const playgroundMessages = normalizeInput(input, { metadata: input })
.data?.map(convertChatMlToPlayground)
.filter((m) => m !== null);
const secondMsg = playgroundMessages?.[1];
expect(secondMsg).toBeDefined();
expect(secondMsg && "role" in secondMsg ? secondMsg.role : null).toBe(
"tool",
);
expect(
secondMsg && "content" in secondMsg ? secondMsg.content : "",
).toContain("PatientNo");
});
it("should handle OpenAI Agents function_call and function_call_output", () => {
// user message, function_call (tool call), function_call_output (tool result)
const input = [
{ content: "What's the weather in Tokyo?", role: "user" },
{
arguments: { city: "Tokyo" },
call_id: "call_abc123",
name: "get_weather",
type: "function_call",
id: "fc_xyz",
status: "completed",
},
{
call_id: "call_abc123",
output: "The weather in Tokyo is sunny.",
type: "function_call_output",
},
];
const inResult = normalizeInput(input, { framework: "openai" });
expect(inResult.success).toBe(true);
const playgroundMessages = inResult
.data!.map(convertChatMlToPlayground)
.filter((msg) => msg !== null);
// Should have 3 messages
expect(playgroundMessages).toHaveLength(3);
// Message 1: user
expect(playgroundMessages[0]?.type).toBe("public-api-created");
if (playgroundMessages[0]?.type === "public-api-created") {
expect(playgroundMessages[0].role).toBe("user");
expect(playgroundMessages[0].content).toBe(
"What's the weather in Tokyo?",
);
}
// Message 2: assistant with tool call
expect(playgroundMessages[1]?.type).toBe("assistant-tool-call");
if (playgroundMessages[1]?.type === "assistant-tool-call") {
expect(playgroundMessages[1].toolCalls[0].id).toBe("call_abc123");
expect(playgroundMessages[1].toolCalls[0].name).toBe("get_weather");
}
// Message 3: tool result
expect(playgroundMessages[2]?.type).toBe("tool-result");
if (playgroundMessages[2]?.type === "tool-result") {
expect(playgroundMessages[2].toolCallId).toBe("call_abc123");
expect(playgroundMessages[2].content).toBe(
"The weather in Tokyo is sunny.",
);
}
});
it("should normalize standalone OpenAI Responses function_call output", () => {
const output = {
id: "fc_order_status",
type: "function_call",
status: "completed",
arguments: '{"orderIds":null}',
call_id: "call_order_status",
name: "getOrderStatus",
};
const outputResult = normalizeOutput(output, { framework: "openai" });
expect(outputResult.success).toBe(true);
expect(outputResult.data![0]).toMatchObject({
role: "assistant",
tool_calls: [
{
id: "call_order_status",
name: "getOrderStatus",
arguments: '{"orderIds":null}',
type: "function",
},
],
});
});
it("should normalize OpenAI Responses function_call output arrays", () => {
const output = [
{
id: "fc_order_status",
type: "function_call",
status: "completed",
arguments: '{"orderIds":null}',
call_id: "call_order_status",
name: "getOrderStatus",
},
{
id: "fc_promotions",
type: "function_call",
status: "completed",
arguments: "{}",
call_id: "call_promotions",
name: "getPromotions",
},
];
const outputResult = normalizeOutput(output, {
observationName: "OpenAI.responses",
});
expect(outputResult.success).toBe(true);
expect(outputResult.data).toEqual([
expect.objectContaining({
role: "assistant",
tool_calls: [
{
id: "call_order_status",
name: "getOrderStatus",
arguments: '{"orderIds":null}',
type: "function",
},
],
}),
expect.objectContaining({
role: "assistant",
tool_calls: [
{
id: "call_promotions",
name: "getPromotions",
arguments: "{}",
type: "function",
},
],
}),
]);
});
it("should handle VAPI camelCase toolCalls and preserve IDs", () => {
// VAPI uses camelCase toolCalls instead of tool_calls
// Critical: Tool call IDs must be preserved for OpenAI API compatibility
const input = {
tools: [
{
type: "function",
function: {
name: "verify_user",
description: "Verify user",
parameters: { type: "object", properties: {}, required: [] },
},
},
],
messages: [
{
role: "assistant",
content: "Checking...",
toolCalls: [
{
id: "call_123",
type: "function",
function: { name: "verify_user", arguments: "{}" },
},
],
},
{
role: "tool",
content: '{"verified": true}',
tool_call_id: "call_123",
},
],
};
const inResult = normalizeInput(input, {
observationName: "chat-completion",
});
expect(inResult.success).toBe(true);
const playgroundMessages = inResult
.data!.map(convertChatMlToPlayground)
.filter((msg) => msg !== null);
// Critical assertions: IDs must match
const toolCallMsg = playgroundMessages[0];
expect(toolCallMsg?.type).toBe("assistant-tool-call");
if (toolCallMsg?.type === "assistant-tool-call") {
expect(toolCallMsg.toolCalls[0].id).toBe("call_123");
}
const toolResultMsg = playgroundMessages[1];
expect(toolResultMsg?.type).toBe("tool-result");
if (toolResultMsg?.type === "tool-result") {
expect(toolResultMsg.toolCallId).toBe("call_123");
}
// Verify tools extracted
expect(extractTools(input)).toHaveLength(1);
});
it("should handle stringified tools in metadata (like from vercel AI SDK v5 + bedrock)", () => {
// metadata contains stringified tools instead of actual objects, causing flattenToolDefinition to return {}
// and ChatML validation to fail with "normalizeInput success: false"
const input = [
{
role: "system",
content: "You are a helpful assistant.",
},
{
role: "user",
content: [{ type: "text", text: "Test" }],
},
];
// metadata.tools array has STRINGIFIED elements
const metadata = {
tools: [
JSON.stringify({
type: "function",
name: "get_weather",
description: "Get weather info",
inputSchema: {
type: "object",
properties: { city: { type: "string" } },
required: ["city"],
},
}),
JSON.stringify({
type: "function",
name: "search_web",
description: "Search the web",
inputSchema: {
type: "object",
properties: { query: { type: "string" } },
},
}),
],
scope: { name: "ai" },
};
const inResult = normalizeInput(input, { metadata });
// Should succeed (flattenToolDefinition parses strings)
expect(inResult.success).toBe(true);
expect(inResult.data).toBeDefined();
expect(inResult.data!.length).toBeGreaterThan(0);
// Verify tools are attached to messages
const firstMessage = inResult.data![0];
expect(firstMessage.tools).toBeDefined();
expect(firstMessage.tools!.length).toBe(2);
expect(firstMessage.tools![0].name).toBe("get_weather");
expect(firstMessage.tools![1].name).toBe("search_web");
});
it("should extract AI SDK flat tools and provider tools from normalized input", () => {
const tools = extractTools({
messages: [{ role: "user", content: "Need weather and search" }],
tools: [
{
type: "function",
name: "get_weather",
description: "Get weather info",
inputSchema: {
type: "object",
properties: { city: { type: "string" } },
},
},
{
type: "web_search_preview",
},
],
});
expect(tools).toHaveLength(2);
expect(tools[0]).toMatchObject({
name: "get_weather",
description: "Get weather info",
parameters: {
type: "object",
properties: { city: { type: "string" } },
},
});
expect(tools[1]).toMatchObject({
name: "web_search_preview",
description: "",
});
});
it("should extract tools from legacy metadata, normalized input, and output", () => {
const metadataTools = extractTools(null, {
attributes: {
"llm.tools.0.tool.json_schema": {
type: "function",
function: {
name: "metadata_search",
description: "Search from metadata",
parameters: {
type: "object",
properties: { query: { type: "string" } },
},
},
},
},
});
expect(metadataTools).toHaveLength(1);
expect(metadataTools[0]).toMatchObject({
name: "metadata_search",
description: "Search from metadata",
});
const inputTools = extractTools({
messages: [{ role: "user", content: "Need a joke" }],
tools: [
{
type: "function",
name: "input_joke",
description: "Get a joke from normalized input",
inputSchema: { type: "object" },
},
],
});
expect(inputTools).toHaveLength(1);
expect(inputTools[0]).toMatchObject({
name: "input_joke",
description: "Get a joke from normalized input",
parameters: { type: "object" },
});
const outputTools = extractTools({
output: [{ role: "assistant", content: "Done" }],
tools: [
{
type: "function",
name: "output_tool",
description: "Tool carried by output",
inputSchema: { type: "object" },
},
],
});
expect(outputTools).toHaveLength(1);
expect(outputTools[0]).toMatchObject({
name: "output_tool",
description: "Tool carried by output",
});
});
it("should preserve numeric order for indexed metadata tools", () => {
const tools = extractTools(null, {
attributes: {
"llm.tools.10.tool.json_schema": {
type: "function",
function: {
name: "tool_ten",
},
},
"llm.tools.2.tool.json_schema": {
type: "function",
function: {
name: "tool_two",
},
},
"llm.tools.1.tool.json_schema": {
type: "function",
function: {
name: "tool_one",
},
},
},
});
expect(tools.map((tool) => tool.name)).toEqual([
"tool_one",
"tool_two",
"tool_ten",
]);
});
it("should extract tools from stringified metadata attributes", () => {
const metadataTools = extractTools(null, {
attributes: JSON.stringify({
"ai.prompt.tools": JSON.stringify([
{
type: "function",
name: "metadata_joke",
description: "Get a joke from metadata",
inputSchema: {
type: "object",
properties: { topic: { type: "string" } },
},
},
]),
}),
});
expect(metadataTools).toHaveLength(1);
expect(metadataTools[0]).toMatchObject({
name: "metadata_joke",
description: "Get a joke from metadata",
parameters: {
type: "object",
properties: { topic: { type: "string" } },
},
});
});
it("should pass metadata attributes.tools into playground tools", () => {
const input = [
{
role: "assistant",
tool_calls: [
{
id: "call_demo_lookup",
function: {
name: "demo_lookup",
arguments: '{"query":"example"}',
},
type: "function",
},
],
},
{
role: "tool",
content: '{"status":"ok"}',
tool_call_id: "call_demo_lookup",
},
];
const expectedTool = {
name: "demo_lookup",
description: "Lookup a demo value.",
parameters: {
type: "object",
properties: { query: { type: "string" } },
required: ["query"],
},
};
const metadata = {
attributes: {
tools: JSON.stringify([
{
type: "function",
function: expectedTool,
},
]),
tool_count: "1",
},
};
const ctx = { metadata, observationName: "llm" };
const inResult = normalizeInput(input, ctx);
expect(inResult.success).toBe(true);
if (!inResult.data) throw new Error("Expected data to be defined");
const playgroundMessages = inResult.data
.map(convertChatMlToPlayground)
.filter((msg) => msg !== null);
expect(
playgroundMessages.some((msg) => msg.type === "assistant-tool-call"),
).toBe(true);
expect(playgroundMessages.some((msg) => msg.type === "tool-result")).toBe(
true,
);
// Same merge behavior as JumpToPlaygroundButton.parseGeneration.
const normalizedTools = extractTools(inResult.data, ctx.metadata);
const rawTools = extractTools(input, metadata);
const mergedTools = normalizedTools.length > 0 ? normalizedTools : rawTools;
expect(mergedTools).toHaveLength(1);
expect(mergedTools[0]).toMatchObject(expectedTool);
});
it("should handle double-stringified messages array", () => {
// ClickHouse can store messages as double-stringified:
// { "messages": "[{\"role\":\"user\",\"content\":\"...\"}]" }
// instead of: { "messages": [{role: "user", ...}] }
//
// This caused validation to fail: expected array, received string
const input = {
messages: JSON.stringify([
// ← DOUBLE STRINGIFIED
{ role: "system", content: "You are a helpful assistant." },
{ role: "user", content: "Hello" },
{ role: "assistant", content: "Hi there!" },
{ role: "user", content: "How are you?" },
]),
};
const ctx = {};
const inResult = normalizeInput(input, ctx);
expect(inResult.success).toBe(true);
if (!inResult.data) throw new Error("Expected data to be defined");
const playgroundMessages = inResult.data
.map(convertChatMlToPlayground)
.filter((msg) => msg !== null);
// Must have all 4 messages for playground button to be enabled
expect(playgroundMessages).toHaveLength(4);
expect(playgroundMessages[0]?.type).toBe("public-api-created");
if (playgroundMessages[0] && "role" in playgroundMessages[0]) {
expect(playgroundMessages[0].role).toBe("system");
}
});
it("should handle Semantic Kernel input and output formats", () => {
// Semantic Kernel (Microsoft.SemanticKernel.Diagnostics) uses OpenTelemetry GenAI semantic conventions
// where actual message content is stored in gen_ai.event.content as a JSON string
// This format is different from MS Agent Framework's parts[] format
const metadata = {
scope: { name: "Microsoft.SemanticKernel.Diagnostics", version: "" },
};
// Input: array format with flat message structure
const input = [
{
role: "user",
"gen_ai.event.content":
'{"role":"user","content":"What is the weather?","tool_calls":[]}',
},
];
const inputResult = normalizeInput(input, { metadata });
expect(inputResult.success).toBe(true);
const inputMsg = inputResult
.data!.map(convertChatMlToPlayground)
.filter((m) => m)[0]!;
expect(inputMsg).toBeDefined();
if ("role" in inputMsg) {
expect(inputMsg.role).toBe("user");
expect(inputMsg.content).toBe("What is the weather?");
// Ensure gen_ai.event.content was unwrapped, not passed through
expect(inputMsg).not.toHaveProperty("gen_ai.event.content");
} else {
throw new Error("Expected message with role property");
}
// Output: single object with nested message structure
const output = {
"gen_ai.event.content":
'{"message":{"role":"Assistant","content":"The answer is positive."}}',
};
const outputResult = normalizeOutput(output, { metadata });
expect(outputResult.success).toBe(true);
const outputMsg = outputResult
.data!.map(convertChatMlToPlayground)
.filter((m) => m)[0];
expect(outputMsg).toBeDefined();
if (outputMsg && "role" in outputMsg) {
expect(outputMsg.role).toBe("assistant");
expect(outputMsg.content).toBe("The answer is positive.");
// Ensure gen_ai.event.content was unwrapped, not passed through
expect(outputMsg).not.toHaveProperty("gen_ai.event.content");
} else {
throw new Error("Expected message with role property");
}
});
it("should respect includeOutput flag when jumping to playground, default to no output added", () => {
// kinda a bad test mocking UI behavior and not really testing the UI.
// but it should illustrate that the default behavior is to not include output! and document that.
const input = {
messages: [
{ role: "system", content: "You are a helpful assistant." },
{ role: "user", content: "What can you help me with?" },
],
};
const output = {
role: "assistant",
content: "I can help with many tasks!",
};
const ctx = {};
// Process input (always happens)
const inputMessages = normalizeInput(input, ctx)
.data!.map(convertChatMlToPlayground)
.filter((msg) => msg !== null);
// Test 1: Default behavior (includeOutput=false) - only input passed to the playground
expect(inputMessages.length).toBe(2);
const secondMsg = inputMessages[1];
if (secondMsg && "role" in secondMsg) {
expect(secondMsg.role).toBe("user");
}
// Test 2: With includeOutput=true - input + output
const outputMessages = normalizeOutput(output, ctx)
.data!.map(convertChatMlToPlayground)
.filter((msg) => msg !== null && msg.type !== "assistant-tool-call");
const withOutput = [...inputMessages, ...outputMessages];
expect(withOutput.length).toBe(3);
const thirdMsg = withOutput[2];
if (thirdMsg && "role" in thirdMsg) {
expect(thirdMsg.role).toBe("assistant");
expect(thirdMsg.content).toBe("I can help with many tasks!");
}
});
});