🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(chatml): treat Zod safeParse throws as non-matches (#17034)
cursor[bot]2503c205e370
diff --git a/packages/shared/src/utils/chatml/adapters/index.ts b/packages/shared/src/utils/chatml/adapters/index.tsindex 1e45f3f9cd..6a82fffd9c 100644--- a/packages/shared/src/utils/chatml/adapters/index.ts+++ b/packages/shared/src/utils/chatml/adapters/index.ts@@ -27,10 +27,15 @@ function selectAdapter(ctx: NormalizerContext): ProviderAdapter {if (adapter) return adapter;}- // First adapter that matches wins+ // First adapter that matches wins. Detection is best-effort: a throwing+ // adapter is treated as a non-match so I/O preview can fall through.for (const adapter of adapters) {- if (adapter.detect(ctx)) {- return adapter;+ try {+ if (adapter.detect(ctx)) {+ return adapter;+ }+ } catch {+ continue;}}diff --git a/packages/shared/src/utils/chatml/adapters/langgraph.ts b/packages/shared/src/utils/chatml/adapters/langgraph.tsindex f4d58c9de4..31faedf35f 100644--- a/packages/shared/src/utils/chatml/adapters/langgraph.ts+++ b/packages/shared/src/utils/chatml/adapters/langgraph.ts@@ -6,6 +6,7 @@ import {isRichToolResult,attachToolDefinitionsToMessages,normalizeToolDefinitionsForChatMl,+ schemaMatches,} from "../helpers";import { z } from "zod";@@ -372,37 +373,33 @@ export const langgraphAdapter: ProviderAdapter = {}// STRUCTURAL: Schema-based detection on metadata- if (LangChainMessageSchema.safeParse(ctx.metadata).success) return true;- if (LangGraphMessageSchema.safeParse(ctx.metadata).success) return true;+ if (schemaMatches(LangChainMessageSchema, ctx.metadata)) return true;+ if (schemaMatches(LangGraphMessageSchema, ctx.metadata)) return true;// Check wrapped messages format- if (LangGraphWrappedSchema.safeParse(ctx.metadata).success) {+ if (schemaMatches(LangGraphWrappedSchema, ctx.metadata)) {const wrapped = ctx.metadata as { messages: unknown[]; tools?: unknown };// reject OpenAI Chat Completions format {tools: [...], messages: [...]}if (Array.isArray(wrapped.tools)) {return false;}- if (LangChainMessageSchema.safeParse(wrapped.messages).success)- return true;- if (LangGraphMessageSchema.safeParse(wrapped.messages).success)- return true;+ if (schemaMatches(LangChainMessageSchema, wrapped.messages)) return true;+ if (schemaMatches(LangGraphMessageSchema, wrapped.messages)) return true;}// finally Schema-based detection on data b/c of performance- if (LangChainMessageSchema.safeParse(ctx.data).success) return true;- if (LangGraphMessageSchema.safeParse(ctx.data).success) return true;+ if (schemaMatches(LangChainMessageSchema, ctx.data)) return true;+ if (schemaMatches(LangGraphMessageSchema, ctx.data)) return true;// Check wrapped messages format on data- if (LangGraphWrappedSchema.safeParse(ctx.data).success) {+ if (schemaMatches(LangGraphWrappedSchema, ctx.data)) {const wrapped = ctx.data as { messages: unknown[]; tools?: unknown };// reject OpenAI Chat Completions format {tools: [...], messages: [...]}if (Array.isArray(wrapped.tools)) {return false;}- if (LangChainMessageSchema.safeParse(wrapped.messages).success)- return true;- if (LangGraphMessageSchema.safeParse(wrapped.messages).success)- return true;+ if (schemaMatches(LangChainMessageSchema, wrapped.messages)) return true;+ if (schemaMatches(LangGraphMessageSchema, wrapped.messages)) return true;}return false;diff --git a/packages/shared/src/utils/chatml/core.ts b/packages/shared/src/utils/chatml/core.tsindex cd955af467..39d6203a28 100644--- a/packages/shared/src/utils/chatml/core.ts+++ b/packages/shared/src/utils/chatml/core.ts@@ -11,6 +11,20 @@ import {ChatMlMessageSchema,} from "../IORepresentation/chatML/types";import { selectAdapter, type NormalizerContext } from "./adapters";+import { safeSchemaParse } from "./helpers";++type ChatMlParseResult = ReturnType<typeof ChatMlArraySchema.safeParse>;++function failedChatMlParse(): ChatMlParseResult {+ // Bare `{ success: false }` at runtime (SES-safe). Cast keeps the historical+ // Zod SafeParse shape so callers that read `.data` after a success assert+ // (Vitest does not narrow) typecheck without rewriting every test.+ return { success: false } as ChatMlParseResult;+}++function parseChatMlArray(data: unknown): ChatMlParseResult {+ return safeSchemaParse(ChatMlArraySchema, data) as ChatMlParseResult;+}type ChatMlMessage = z.infer<typeof ChatMlMessageSchema>;@@ -26,44 +40,41 @@ function isSingleChatMlMessage();}-export function mapToChatMl(- input: unknown,-): ReturnType<typeof ChatMlArraySchema.safeParse> {- let result = ChatMlArraySchema.safeParse(input);+export function mapToChatMl(input: unknown): ChatMlParseResult {+ const result = parseChatMlArray(input);if (result.success) {return result;}// Check if input is an array of length 1 including an array of ChatMlMessageSchema// e.g. [[ChatMlMessageSchema, ...]]- const inputArray = z.array(ChatMlArraySchema).safeParse(input);+ const inputArray = safeSchemaParse(z.array(ChatMlArraySchema), input);if (inputArray.success && inputArray.data.length === 1) {- return ChatMlArraySchema.safeParse(inputArray.data[0]);+ return parseChatMlArray(inputArray.data[0]);}// Check if input is an object with a messages key// e.g. { messages: [ChatMlMessageSchema, ...] }- const inputObject = z- .object({+ const inputObject = safeSchemaParse(+ z.object({messages: ChatMlArraySchema,- })- .safeParse(input);+ }),+ input,+ );if (inputObject.success) {- return ChatMlArraySchema.safeParse(inputObject.data.messages);+ return parseChatMlArray(inputObject.data.messages);}// Single message object, e.g. { role: "user", content: "..." }if (isSingleChatMlMessage(input)) {- return ChatMlArraySchema.safeParse([input]);+ return parseChatMlArray([input]);}return result;}-export function mapOutputToChatMl(- output: unknown,-): ReturnType<typeof ChatMlArraySchema.safeParse> {+export function mapOutputToChatMl(output: unknown): ChatMlParseResult {// Check if output has messages key (LangGraph/LangChain format)if (output &&@@ -72,14 +83,10 @@ export function mapOutputToChatMl("messages" in output) {const obj = output as Record<string, unknown>;- return ChatMlArraySchema.safeParse(obj.messages);+ return parseChatMlArray(obj.messages);}- const result = ChatMlArraySchema.safeParse(- Array.isArray(output) ? output : [output],- );-- return result;+ return parseChatMlArray(Array.isArray(output) ? output : [output]);}export function cleanLegacyOutput(output: unknown, fallback?: unknown) {@@ -89,8 +96,10 @@ export function cleanLegacyOutput(output: unknown, fallback?: unknown) {}).refine((value) => Object.keys(value).length === 1);- const outLegacyCompletionSchemaParsed =- outLegacyCompletionSchema.safeParse(output);+ const outLegacyCompletionSchemaParsed = safeSchemaParse(+ outLegacyCompletionSchema,+ output,+ );const outputClean = outLegacyCompletionSchemaParsed.success? outLegacyCompletionSchemaParsed.data: (fallback ?? null);@@ -155,14 +164,18 @@ export function combineInputOutputMessages(export function normalizeInput(input: unknown,ctx: NormalizerContext = {},-): ReturnType<typeof ChatMlArraySchema.safeParse> {- const adapter = selectAdapter({- ...ctx,- metadata: ctx.metadata ?? input,- data: input,- });- const preprocessed = adapter.preprocess(input, "input", ctx);- return mapToChatMl(preprocessed);+): ChatMlParseResult {+ try {+ const adapter = selectAdapter({+ ...ctx,+ metadata: ctx.metadata ?? input,+ data: input,+ });+ const preprocessed = adapter.preprocess(input, "input", ctx);+ return mapToChatMl(preprocessed);+ } catch {+ return failedChatMlParse();+ }}/**@@ -172,12 +185,16 @@ export function normalizeInput(export function normalizeOutput(output: unknown,ctx: NormalizerContext = {},-): ReturnType<typeof ChatMlArraySchema.safeParse> {- const adapter = selectAdapter({- ...ctx,- metadata: ctx.metadata ?? output,- data: output,- });- const preprocessed = adapter.preprocess(output, "output", ctx);- return mapOutputToChatMl(preprocessed);+): ChatMlParseResult {+ try {+ const adapter = selectAdapter({+ ...ctx,+ metadata: ctx.metadata ?? output,+ data: output,+ });+ const preprocessed = adapter.preprocess(output, "output", ctx);+ return mapOutputToChatMl(preprocessed);+ } catch {+ return failedChatMlParse();+ }}diff --git a/packages/shared/src/utils/chatml/helpers.ts b/packages/shared/src/utils/chatml/helpers.tsindex 94d8909ea1..b572b8bdb7 100644--- a/packages/shared/src/utils/chatml/helpers.ts+++ b/packages/shared/src/utils/chatml/helpers.ts@@ -1,5 +1,38 @@import { parseJsonIfString as parseIfString } from "../json";+type SchemaWithSafeParse<T> = {+ safeParse: (data: unknown) => T;+};++/** Failure from a `safeParse` path that threw before Zod returned a result. */+export type SafeParseCatchFailure = { success: false };++/**+ * Zod 4 `safeParse` constructs a `$ZodError` on failure and assigns+ * `inst.name`. That assignment throws when `Error.name` is non-writable+ * (SES/lockdown and some browser extensions). Treat a throw as a failed parse.+ *+ * Catch returns `{ success: false }` only — do not invent a `ZodError` here;+ * constructing one can hit the same non-writable `Error.name` path under SES.+ */+export function safeSchemaParse<T extends { success: boolean }>(+ schema: SchemaWithSafeParse<T>,+ data: unknown,+): T | SafeParseCatchFailure {+ try {+ return schema.safeParse(data);+ } catch {+ return { success: false };+ }+}++export function schemaMatches(+ schema: SchemaWithSafeParse<{ success: boolean }>,+ data: unknown,+): boolean {+ return safeSchemaParse(schema, data).success;+}+export function removeNullFields(obj: unknown): Record<string, unknown> {if (!obj || typeof obj !== "object") return {};diff --git a/web/src/features/traces/hooks/useChatMLParser.ts b/web/src/features/traces/hooks/useChatMLParser.tsindex 6858953aab..7093fc53dd 100644--- a/web/src/features/traces/hooks/useChatMLParser.ts+++ b/web/src/features/traces/hooks/useChatMLParser.ts@@ -187,32 +187,54 @@ export function computeToolCallBookkeeping(* result has any conversation representation without relying on observation* types.*/+function emptyChatMLParserResult(): ChatMLParserResult {+ return {+ canDisplayAsChat: false,+ allMessages: [],+ additionalInput: undefined,+ allTools: [],+ toolCallCounts: new Map(),+ toolCallsByName: new Map(),+ messageToToolCallNumbers: new Map(),+ toolNameToDefinitionNumber: new Map(),+ inputMessageCount: 0,+ };+}+export function parseChatML(parsedInput: unknown,parsedOutput: unknown,parsedMetadata: unknown,observationName: string | undefined,): ChatMLParserResult {- const ctx = { metadata: parsedMetadata, observationName };- const inResult = normalizeInput(parsedInput, ctx);- const outResult = normalizeOutput(parsedOutput, ctx);- const outputClean = cleanLegacyOutput(parsedOutput, parsedOutput);- const messages = combineInputOutputMessages(inResult, outResult, outputClean);+ try {+ const ctx = { metadata: parsedMetadata, observationName };+ const inResult = normalizeInput(parsedInput, ctx);+ const outResult = normalizeOutput(parsedOutput, ctx);+ const outputClean = cleanLegacyOutput(parsedOutput, parsedOutput);+ const messages = combineInputOutputMessages(+ inResult,+ outResult,+ outputClean,+ );- const inputMessageCount = inResult.success ? inResult.data.length : 0;- const toolBookkeeping = computeToolCallBookkeeping(- messages,- inputMessageCount,- );+ const inputMessageCount = inResult.success ? inResult.data.length : 0;+ const toolBookkeeping = computeToolCallBookkeeping(+ messages,+ inputMessageCount,+ );- return {- canDisplayAsChat:- (inResult.success || outResult.success) && messages.length > 0,- allMessages: messages as ChatMlMessage[],- additionalInput: extractAdditionalInput(parsedInput),- ...toolBookkeeping,- inputMessageCount,- };+ return {+ canDisplayAsChat:+ (inResult.success || outResult.success) && messages.length > 0,+ allMessages: messages as ChatMlMessage[],+ additionalInput: extractAdditionalInput(parsedInput),+ ...toolBookkeeping,+ inputMessageCount,+ };+ } catch {+ return emptyChatMLParserResult();+ }}/**diff --git a/worker/src/__tests__/chatml/integration.test.ts b/worker/src/__tests__/chatml/integration.test.tsindex 28df9cb23e..e270d9b850 100644--- a/worker/src/__tests__/chatml/integration.test.ts+++ b/worker/src/__tests__/chatml/integration.test.ts@@ -133,6 +133,55 @@ describe("ChatML Integration", () => {expect(() => normalizeOutput({ scope: null, answer: "..." })).not.toThrow();});+ it("does not throw when Error.name is non-writable", () => {+ // Zod 4 safeParse builds a $ZodError on mismatch and assigns inst.name.+ // That assignment throws when Error.name is non-writable (SES/lockdown).+ // Adapter detection runs safeParse on arbitrary observation metadata, so+ // a failed parse must stay a non-match instead of crashing I/O preview.+ const descriptor = Object.getOwnPropertyDescriptor(Error.prototype, "name");+ Object.defineProperty(Error.prototype, "name", {+ writable: false,+ configurable: true,+ enumerable: descriptor?.enumerable ?? false,+ value: descriptor?.value ?? "Error",+ });++ try {+ const input = {+ messages: [{ role: "user", content: "hello" }],+ };+ const metadata = {+ provider: "openrouter",+ model: "openrouter-completion",+ };++ expect(() =>+ normalizeInput(input, {+ metadata,+ observationName: "openrouter-completion",+ }),+ ).not.toThrow();+ expect(() =>+ normalizeOutput({ role: "assistant", content: "hi" }, { metadata }),+ ).not.toThrow();++ const result = normalizeInput(input, {+ metadata,+ observationName: "openrouter-completion",+ });+ expect(result.success).toBe(true);++ // Mismatch path: safeParse throws under locked Error.name; treat as+ // non-match without inventing a ZodError (that can re-hit the same lock).+ const failed = normalizeInput({ not: "chatml" }, { metadata });+ expect(failed.success).toBe(false);+ } finally {+ if (descriptor) {+ Object.defineProperty(Error.prototype, "name", descriptor);+ }+ }+ });+it("should handle very large inputs", () => {const largeContent = "x".repeat(1000000);const input = [{ role: "user", content: largeContent }];