Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(chatml): treat Zod safeParse throws as non-matches (#17034)

cursor[bot]2503c205e370
diff --git a/packages/shared/src/utils/chatml/adapters/index.ts b/packages/shared/src/utils/chatml/adapters/index.ts
index 1e45f3f9cd..6a82fffd9c 100644
--- a/packages/shared/src/utils/chatml/adapters/index.ts
+++ b/packages/shared/src/utils/chatml/adapters/index.ts
@@ -27,10 +27,15 @@ function selectAdapter(ctx: NormalizerContext): ProviderAdapter {
if (adapter) return adapter;
}
- // First adapter that matches wins
+ // First adapter that matches wins. Detection is best-effort: a throwing
+ // adapter is treated as a non-match so I/O preview can fall through.
for (const adapter of adapters) {
- if (adapter.detect(ctx)) {
- return adapter;
+ try {
+ if (adapter.detect(ctx)) {
+ return adapter;
+ }
+ } catch {
+ continue;
}
}
diff --git a/packages/shared/src/utils/chatml/adapters/langgraph.ts b/packages/shared/src/utils/chatml/adapters/langgraph.ts
index f4d58c9de4..31faedf35f 100644
--- a/packages/shared/src/utils/chatml/adapters/langgraph.ts
+++ b/packages/shared/src/utils/chatml/adapters/langgraph.ts
@@ -6,6 +6,7 @@ import {
isRichToolResult,
attachToolDefinitionsToMessages,
normalizeToolDefinitionsForChatMl,
+ schemaMatches,
} from "../helpers";
import { z } from "zod";
@@ -372,37 +373,33 @@ export const langgraphAdapter: ProviderAdapter = {
}
// STRUCTURAL: Schema-based detection on metadata
- if (LangChainMessageSchema.safeParse(ctx.metadata).success) return true;
- if (LangGraphMessageSchema.safeParse(ctx.metadata).success) return true;
+ if (schemaMatches(LangChainMessageSchema, ctx.metadata)) return true;
+ if (schemaMatches(LangGraphMessageSchema, ctx.metadata)) return true;
// Check wrapped messages format
- if (LangGraphWrappedSchema.safeParse(ctx.metadata).success) {
+ if (schemaMatches(LangGraphWrappedSchema, ctx.metadata)) {
const wrapped = ctx.metadata as { messages: unknown[]; tools?: unknown };
// reject OpenAI Chat Completions format {tools: [...], messages: [...]}
if (Array.isArray(wrapped.tools)) {
return false;
}
- if (LangChainMessageSchema.safeParse(wrapped.messages).success)
- return true;
- if (LangGraphMessageSchema.safeParse(wrapped.messages).success)
- return true;
+ if (schemaMatches(LangChainMessageSchema, wrapped.messages)) return true;
+ if (schemaMatches(LangGraphMessageSchema, wrapped.messages)) return true;
}
// finally Schema-based detection on data b/c of performance
- if (LangChainMessageSchema.safeParse(ctx.data).success) return true;
- if (LangGraphMessageSchema.safeParse(ctx.data).success) return true;
+ if (schemaMatches(LangChainMessageSchema, ctx.data)) return true;
+ if (schemaMatches(LangGraphMessageSchema, ctx.data)) return true;
// Check wrapped messages format on data
- if (LangGraphWrappedSchema.safeParse(ctx.data).success) {
+ if (schemaMatches(LangGraphWrappedSchema, ctx.data)) {
const wrapped = ctx.data as { messages: unknown[]; tools?: unknown };
// reject OpenAI Chat Completions format {tools: [...], messages: [...]}
if (Array.isArray(wrapped.tools)) {
return false;
}
- if (LangChainMessageSchema.safeParse(wrapped.messages).success)
- return true;
- if (LangGraphMessageSchema.safeParse(wrapped.messages).success)
- return true;
+ if (schemaMatches(LangChainMessageSchema, wrapped.messages)) return true;
+ if (schemaMatches(LangGraphMessageSchema, wrapped.messages)) return true;
}
return false;
diff --git a/packages/shared/src/utils/chatml/core.ts b/packages/shared/src/utils/chatml/core.ts
index cd955af467..39d6203a28 100644
--- a/packages/shared/src/utils/chatml/core.ts
+++ b/packages/shared/src/utils/chatml/core.ts
@@ -11,6 +11,20 @@ import {
ChatMlMessageSchema,
} from "../IORepresentation/chatML/types";
import { selectAdapter, type NormalizerContext } from "./adapters";
+import { safeSchemaParse } from "./helpers";
+
+type ChatMlParseResult = ReturnType<typeof ChatMlArraySchema.safeParse>;
+
+function failedChatMlParse(): ChatMlParseResult {
+ // Bare `{ success: false }` at runtime (SES-safe). Cast keeps the historical
+ // Zod SafeParse shape so callers that read `.data` after a success assert
+ // (Vitest does not narrow) typecheck without rewriting every test.
+ return { success: false } as ChatMlParseResult;
+}
+
+function parseChatMlArray(data: unknown): ChatMlParseResult {
+ return safeSchemaParse(ChatMlArraySchema, data) as ChatMlParseResult;
+}
type ChatMlMessage = z.infer<typeof ChatMlMessageSchema>;
@@ -26,44 +40,41 @@ function isSingleChatMlMessage(
);
}
-export function mapToChatMl(
- input: unknown,
-): ReturnType<typeof ChatMlArraySchema.safeParse> {
- let result = ChatMlArraySchema.safeParse(input);
+export function mapToChatMl(input: unknown): ChatMlParseResult {
+ const result = parseChatMlArray(input);
if (result.success) {
return result;
}
// Check if input is an array of length 1 including an array of ChatMlMessageSchema
// e.g. [[ChatMlMessageSchema, ...]]
- const inputArray = z.array(ChatMlArraySchema).safeParse(input);
+ const inputArray = safeSchemaParse(z.array(ChatMlArraySchema), input);
if (inputArray.success && inputArray.data.length === 1) {
- return ChatMlArraySchema.safeParse(inputArray.data[0]);
+ return parseChatMlArray(inputArray.data[0]);
}
// Check if input is an object with a messages key
// e.g. { messages: [ChatMlMessageSchema, ...] }
- const inputObject = z
- .object({
+ const inputObject = safeSchemaParse(
+ z.object({
messages: ChatMlArraySchema,
- })
- .safeParse(input);
+ }),
+ input,
+ );
if (inputObject.success) {
- return ChatMlArraySchema.safeParse(inputObject.data.messages);
+ return parseChatMlArray(inputObject.data.messages);
}
// Single message object, e.g. { role: "user", content: "..." }
if (isSingleChatMlMessage(input)) {
- return ChatMlArraySchema.safeParse([input]);
+ return parseChatMlArray([input]);
}
return result;
}
-export function mapOutputToChatMl(
- output: unknown,
-): ReturnType<typeof ChatMlArraySchema.safeParse> {
+export function mapOutputToChatMl(output: unknown): ChatMlParseResult {
// Check if output has messages key (LangGraph/LangChain format)
if (
output &&
@@ -72,14 +83,10 @@ export function mapOutputToChatMl(
"messages" in output
) {
const obj = output as Record<string, unknown>;
- return ChatMlArraySchema.safeParse(obj.messages);
+ return parseChatMlArray(obj.messages);
}
- const result = ChatMlArraySchema.safeParse(
- Array.isArray(output) ? output : [output],
- );
-
- return result;
+ return parseChatMlArray(Array.isArray(output) ? output : [output]);
}
export function cleanLegacyOutput(output: unknown, fallback?: unknown) {
@@ -89,8 +96,10 @@ export function cleanLegacyOutput(output: unknown, fallback?: unknown) {
})
.refine((value) => Object.keys(value).length === 1);
- const outLegacyCompletionSchemaParsed =
- outLegacyCompletionSchema.safeParse(output);
+ const outLegacyCompletionSchemaParsed = safeSchemaParse(
+ outLegacyCompletionSchema,
+ output,
+ );
const outputClean = outLegacyCompletionSchemaParsed.success
? outLegacyCompletionSchemaParsed.data
: (fallback ?? null);
@@ -155,14 +164,18 @@ export function combineInputOutputMessages(
export function normalizeInput(
input: unknown,
ctx: NormalizerContext = {},
-): ReturnType<typeof ChatMlArraySchema.safeParse> {
- const adapter = selectAdapter({
- ...ctx,
- metadata: ctx.metadata ?? input,
- data: input,
- });
- const preprocessed = adapter.preprocess(input, "input", ctx);
- return mapToChatMl(preprocessed);
+): ChatMlParseResult {
+ try {
+ const adapter = selectAdapter({
+ ...ctx,
+ metadata: ctx.metadata ?? input,
+ data: input,
+ });
+ const preprocessed = adapter.preprocess(input, "input", ctx);
+ return mapToChatMl(preprocessed);
+ } catch {
+ return failedChatMlParse();
+ }
}
/**
@@ -172,12 +185,16 @@ export function normalizeInput(
export function normalizeOutput(
output: unknown,
ctx: NormalizerContext = {},
-): ReturnType<typeof ChatMlArraySchema.safeParse> {
- const adapter = selectAdapter({
- ...ctx,
- metadata: ctx.metadata ?? output,
- data: output,
- });
- const preprocessed = adapter.preprocess(output, "output", ctx);
- return mapOutputToChatMl(preprocessed);
+): ChatMlParseResult {
+ try {
+ const adapter = selectAdapter({
+ ...ctx,
+ metadata: ctx.metadata ?? output,
+ data: output,
+ });
+ const preprocessed = adapter.preprocess(output, "output", ctx);
+ return mapOutputToChatMl(preprocessed);
+ } catch {
+ return failedChatMlParse();
+ }
}
diff --git a/packages/shared/src/utils/chatml/helpers.ts b/packages/shared/src/utils/chatml/helpers.ts
index 94d8909ea1..b572b8bdb7 100644
--- a/packages/shared/src/utils/chatml/helpers.ts
+++ b/packages/shared/src/utils/chatml/helpers.ts
@@ -1,5 +1,38 @@
import { parseJsonIfString as parseIfString } from "../json";
+type SchemaWithSafeParse<T> = {
+ safeParse: (data: unknown) => T;
+};
+
+/** Failure from a `safeParse` path that threw before Zod returned a result. */
+export type SafeParseCatchFailure = { success: false };
+
+/**
+ * Zod 4 `safeParse` constructs a `$ZodError` on failure and assigns
+ * `inst.name`. That assignment throws when `Error.name` is non-writable
+ * (SES/lockdown and some browser extensions). Treat a throw as a failed parse.
+ *
+ * Catch returns `{ success: false }` only — do not invent a `ZodError` here;
+ * constructing one can hit the same non-writable `Error.name` path under SES.
+ */
+export function safeSchemaParse<T extends { success: boolean }>(
+ schema: SchemaWithSafeParse<T>,
+ data: unknown,
+): T | SafeParseCatchFailure {
+ try {
+ return schema.safeParse(data);
+ } catch {
+ return { success: false };
+ }
+}
+
+export function schemaMatches(
+ schema: SchemaWithSafeParse<{ success: boolean }>,
+ data: unknown,
+): boolean {
+ return safeSchemaParse(schema, data).success;
+}
+
export function removeNullFields(obj: unknown): Record<string, unknown> {
if (!obj || typeof obj !== "object") return {};
diff --git a/web/src/features/traces/hooks/useChatMLParser.ts b/web/src/features/traces/hooks/useChatMLParser.ts
index 6858953aab..7093fc53dd 100644
--- a/web/src/features/traces/hooks/useChatMLParser.ts
+++ b/web/src/features/traces/hooks/useChatMLParser.ts
@@ -187,32 +187,54 @@ export function computeToolCallBookkeeping(
* result has any conversation representation without relying on observation
* types.
*/
+function emptyChatMLParserResult(): ChatMLParserResult {
+ return {
+ canDisplayAsChat: false,
+ allMessages: [],
+ additionalInput: undefined,
+ allTools: [],
+ toolCallCounts: new Map(),
+ toolCallsByName: new Map(),
+ messageToToolCallNumbers: new Map(),
+ toolNameToDefinitionNumber: new Map(),
+ inputMessageCount: 0,
+ };
+}
+
export function parseChatML(
parsedInput: unknown,
parsedOutput: unknown,
parsedMetadata: unknown,
observationName: string | undefined,
): ChatMLParserResult {
- const ctx = { metadata: parsedMetadata, observationName };
- const inResult = normalizeInput(parsedInput, ctx);
- const outResult = normalizeOutput(parsedOutput, ctx);
- const outputClean = cleanLegacyOutput(parsedOutput, parsedOutput);
- const messages = combineInputOutputMessages(inResult, outResult, outputClean);
+ try {
+ const ctx = { metadata: parsedMetadata, observationName };
+ const inResult = normalizeInput(parsedInput, ctx);
+ const outResult = normalizeOutput(parsedOutput, ctx);
+ const outputClean = cleanLegacyOutput(parsedOutput, parsedOutput);
+ const messages = combineInputOutputMessages(
+ inResult,
+ outResult,
+ outputClean,
+ );
- const inputMessageCount = inResult.success ? inResult.data.length : 0;
- const toolBookkeeping = computeToolCallBookkeeping(
- messages,
- inputMessageCount,
- );
+ const inputMessageCount = inResult.success ? inResult.data.length : 0;
+ const toolBookkeeping = computeToolCallBookkeeping(
+ messages,
+ inputMessageCount,
+ );
- return {
- canDisplayAsChat:
- (inResult.success || outResult.success) && messages.length > 0,
- allMessages: messages as ChatMlMessage[],
- additionalInput: extractAdditionalInput(parsedInput),
- ...toolBookkeeping,
- inputMessageCount,
- };
+ return {
+ canDisplayAsChat:
+ (inResult.success || outResult.success) && messages.length > 0,
+ allMessages: messages as ChatMlMessage[],
+ additionalInput: extractAdditionalInput(parsedInput),
+ ...toolBookkeeping,
+ inputMessageCount,
+ };
+ } catch {
+ return emptyChatMLParserResult();
+ }
}
/**
diff --git a/worker/src/__tests__/chatml/integration.test.ts b/worker/src/__tests__/chatml/integration.test.ts
index 28df9cb23e..e270d9b850 100644
--- a/worker/src/__tests__/chatml/integration.test.ts
+++ b/worker/src/__tests__/chatml/integration.test.ts
@@ -133,6 +133,55 @@ describe("ChatML Integration", () => {
expect(() => normalizeOutput({ scope: null, answer: "..." })).not.toThrow();
});
+ it("does not throw when Error.name is non-writable", () => {
+ // Zod 4 safeParse builds a $ZodError on mismatch and assigns inst.name.
+ // That assignment throws when Error.name is non-writable (SES/lockdown).
+ // Adapter detection runs safeParse on arbitrary observation metadata, so
+ // a failed parse must stay a non-match instead of crashing I/O preview.
+ const descriptor = Object.getOwnPropertyDescriptor(Error.prototype, "name");
+ Object.defineProperty(Error.prototype, "name", {
+ writable: false,
+ configurable: true,
+ enumerable: descriptor?.enumerable ?? false,
+ value: descriptor?.value ?? "Error",
+ });
+
+ try {
+ const input = {
+ messages: [{ role: "user", content: "hello" }],
+ };
+ const metadata = {
+ provider: "openrouter",
+ model: "openrouter-completion",
+ };
+
+ expect(() =>
+ normalizeInput(input, {
+ metadata,
+ observationName: "openrouter-completion",
+ }),
+ ).not.toThrow();
+ expect(() =>
+ normalizeOutput({ role: "assistant", content: "hi" }, { metadata }),
+ ).not.toThrow();
+
+ const result = normalizeInput(input, {
+ metadata,
+ observationName: "openrouter-completion",
+ });
+ expect(result.success).toBe(true);
+
+ // Mismatch path: safeParse throws under locked Error.name; treat as
+ // non-match without inventing a ZodError (that can re-hit the same lock).
+ const failed = normalizeInput({ not: "chatml" }, { metadata });
+ expect(failed.success).toBe(false);
+ } finally {
+ if (descriptor) {
+ Object.defineProperty(Error.prototype, "name", descriptor);
+ }
+ }
+ });
+
it("should handle very large inputs", () => {
const largeContent = "x".repeat(1000000);
const input = [{ role: "user", content: largeContent }];