Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(io-preview): add parser comparison event capture (#17203)

marliessophief733b5eb73b1
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index fda0cdb705..e299b9ad3e 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -26,6 +26,7 @@ const events = {
"observation_tree_toggle_scores",
"observation_tree_toggle_metrics",
"io_mode_switch",
+ "io_parser_comparison",
"io_pretty_format_toggle_group",
"test_in_playground_button_click",
"display_mode_switch",
diff --git a/web/src/features/traces/components/IOPreview/IOPreview.tsx b/web/src/features/traces/components/IOPreview/IOPreview.tsx
index 5218d8c367..eac079b4d4 100644
--- a/web/src/features/traces/components/IOPreview/IOPreview.tsx
+++ b/web/src/features/traces/components/IOPreview/IOPreview.tsx
@@ -12,6 +12,7 @@ import { IOPreviewJSON, type IOPreviewJSONProps } from "./IOPreviewJSON";
import { IOPreviewJSONSimple } from "./IOPreviewJSONSimple";
import { IOPreviewPretty } from "./IOPreviewPretty";
import { type ChatMLParserResult } from "../../hooks/useChatMLParser";
+import type { IOPreviewParserComparisonOutcome } from "../../hooks/useIOPreviewParser";
import { Button } from "@/src/components/ui/button";
import { ActionButton } from "@/src/components/ActionButton";
import { BookOpen, X } from "lucide-react";
@@ -317,6 +318,9 @@ export function IOPreview({
? "normalized"
: "legacy"
}
+ onParserComparison={(outcome: IOPreviewParserComparisonOutcome) =>
+ capture("trace_detail:io_parser_comparison", { outcome })
+ }
observationName={observationName}
showMetadata={showMetadata}
contentMode={contentMode}
diff --git a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx
index eaa7cac99f..94e2d6f51b 100644
--- a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx
+++ b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx
@@ -1,4 +1,4 @@
-import { useMemo } from "react";
+import { useEffect, useMemo, useRef } from "react";
import { type Prisma, type ScoreDomain, deepParseJson } from "@langfuse/shared";
import { PrettyJsonView } from "@/src/components/ui/PrettyJsonView";
import { type MetadataFilterActions } from "@/src/components/table/ValueCell";
@@ -6,7 +6,9 @@ import { useMarkdownRenderCharacterLimit } from "@/src/hooks/useMarkdownRenderCh
import { type MediaReturnType } from "@/src/features/media/validation";
import { type ChatMLParserResult } from "../../hooks/useChatMLParser";
import {
+ type IOPreviewParserComparisonOutcome,
type IOPreviewParserMode,
+ hasRenderableChatMessages,
useIOPreviewParser,
} from "../../hooks/useIOPreviewParser";
import { ChatMessageList } from "../ChatMessageList";
@@ -16,7 +18,6 @@ import {
type IOPreviewContentMode,
} from "./IOPreview";
import { CorrectedOutputField } from "./components/CorrectedOutputField";
-import { isOnlyJsonMessage } from "../../fns/chatMessageUtils";
import { StatusMessageSection } from "./components/StatusMessageSection";
import type { ObservationStatusMessage } from "./components/statusMessagePresentation";
@@ -116,6 +117,8 @@ export interface IOPreviewPrettyProps extends ExpansionStateProps {
// Which parser produces the preview; the normalized parser is admin-only
// while it is being validated. Legacy remains the safe default.
parser?: IOPreviewParserMode;
+ // Called once after a normalized parser comparison has settled.
+ onParserComparison?: (outcome: IOPreviewParserComparisonOutcome) => void;
}
/**
@@ -162,6 +165,7 @@ export function IOPreviewPretty({
contentMode = "all",
showSystemPrompt,
parser = "legacy",
+ onParserComparison,
}: IOPreviewPrettyProps) {
// Use pre-parsed data if available (from useParsedObservation hook),
// otherwise parse with size/depth limits to prevent UI freeze
@@ -191,17 +195,7 @@ export function IOPreviewPretty({
// Parse into the shared preview contract. The normalized parser is opt-in
// while it is being rolled out; legacy remains the safe default.
- const {
- canDisplayAsChat,
- allMessages,
- additionalInput,
- allTools,
- toolCallCounts,
- toolCallsByName,
- messageToToolCallNumbers,
- toolNameToDefinitionNumber,
- inputMessageCount,
- } = useIOPreviewParser(
+ const { result: parserResult, comparisonOutcome } = useIOPreviewParser(
parser,
input,
output,
@@ -213,6 +207,44 @@ export function IOPreviewPretty({
chatMLParserResult,
);
+ const {
+ allMessages,
+ additionalInput,
+ allTools,
+ toolCallCounts,
+ toolCallsByName,
+ messageToToolCallNumbers,
+ toolNameToDefinitionNumber,
+ inputMessageCount,
+ } = parserResult;
+
+ const capturedComparisonRecord = useRef<string | undefined>(undefined);
+ useEffect(() => {
+ if (
+ parser !== "normalized" ||
+ comparisonOutcome === undefined ||
+ !onParserComparison ||
+ isLoading ||
+ isParsing
+ ) {
+ return;
+ }
+
+ const recordKey = `${observationId ? "observation" : "trace"}:${observationId ?? traceId}`;
+ if (capturedComparisonRecord.current === recordKey) return;
+
+ capturedComparisonRecord.current = recordKey;
+ onParserComparison(comparisonOutcome);
+ }, [
+ comparisonOutcome,
+ isLoading,
+ isParsing,
+ observationId,
+ onParserComparison,
+ parser,
+ traceId,
+ ]);
+
const characterLimit = useMarkdownRenderCharacterLimit();
// Determine if markdown is safe to render (content size check)
@@ -278,8 +310,7 @@ export function IOPreviewPretty({
// Determine if metadata should be shown
const shouldShowMetadata = showMetadata && parsedMetadata !== undefined;
const showData = contentMode !== "conversation";
- const shouldRenderMessages =
- canDisplayAsChat && !allMessages.every(isOnlyJsonMessage);
+ const shouldRenderMessages = hasRenderableChatMessages(parserResult);
return (
<div>
diff --git a/web/src/features/traces/hooks/useIOPreviewParser.clienttest.ts b/web/src/features/traces/hooks/useIOPreviewParser.clienttest.ts
new file mode 100644
index 0000000000..8b41d32858
--- /dev/null
+++ b/web/src/features/traces/hooks/useIOPreviewParser.clienttest.ts
@@ -0,0 +1,66 @@
+import { describe, expect, it } from "vitest";
+import type { ChatMLParserResult, ChatMlMessage } from "./useChatMLParser";
+import {
+ hasRenderableChatMessages,
+ selectIOPreviewParserResult,
+} from "./useIOPreviewParser";
+
+const createMessage = (fields: Partial<ChatMlMessage>): ChatMlMessage =>
+ fields as ChatMlMessage;
+
+const result = (
+ overrides: Partial<ChatMLParserResult> = {},
+): ChatMLParserResult => ({
+ canDisplayAsChat: true,
+ allMessages: [createMessage({ role: "user", content: "hello" })],
+ additionalInput: undefined,
+ allTools: [],
+ toolCallCounts: new Map(),
+ toolCallsByName: new Map(),
+ messageToToolCallNumbers: new Map(),
+ toolNameToDefinitionNumber: new Map(),
+ inputMessageCount: 1,
+ ...overrides,
+});
+
+describe("IO preview parser comparison", () => {
+ it.each([
+ ["both", result(), result()],
+ ["normalized_only", result(), result({ canDisplayAsChat: false })],
+ ["legacy_only", result({ canDisplayAsChat: false }), result()],
+ [
+ "neither",
+ result({ canDisplayAsChat: false }),
+ result({ canDisplayAsChat: false }),
+ ],
+ ] as const)("reports %s", (outcome, normalized, legacy) => {
+ expect(
+ selectIOPreviewParserResult(normalized, legacy).comparisonOutcome,
+ ).toBe(outcome);
+ });
+
+ it("treats a JSON-only message as not renderable chat", () => {
+ expect(
+ hasRenderableChatMessages(
+ result({
+ allMessages: [
+ createMessage({
+ role: "user",
+ json: { raw: "value" },
+ }),
+ ],
+ }),
+ ),
+ ).toBe(false);
+ });
+
+ it("falls back to legacy when normalized has no renderable messages", () => {
+ const normalized = result({ canDisplayAsChat: false });
+ const legacy = result();
+
+ expect(selectIOPreviewParserResult(normalized, legacy)).toEqual({
+ result: legacy,
+ comparisonOutcome: "legacy_only",
+ });
+ });
+});
diff --git a/web/src/features/traces/hooks/useIOPreviewParser.ts b/web/src/features/traces/hooks/useIOPreviewParser.ts
index d27007078d..488cbd0acb 100644
--- a/web/src/features/traces/hooks/useIOPreviewParser.ts
+++ b/web/src/features/traces/hooks/useIOPreviewParser.ts
@@ -3,8 +3,64 @@ import { type Prisma, deepParseJson } from "@langfuse/shared";
import { normalizeSpanIO } from "@langfuse/shared/src/utils/normalized-io";
import { toIOPreview } from "../parsers/toIOPreview";
import { parseChatML, type ChatMLParserResult } from "./useChatMLParser";
+import { isOnlyJsonMessage } from "../fns/chatMessageUtils";
export type IOPreviewParserMode = "legacy" | "normalized";
+export type IOPreviewParserComparisonOutcome =
+ | "both"
+ | "normalized_only"
+ | "legacy_only"
+ | "neither";
+
+export interface IOPreviewParserResult {
+ result: ChatMLParserResult;
+ comparisonOutcome?: IOPreviewParserComparisonOutcome;
+}
+
+/** The same message-level displayability check used by the pretty preview. */
+export function hasRenderableChatMessages(result: ChatMLParserResult): boolean {
+ return (
+ result.canDisplayAsChat && !result.allMessages.every(isOnlyJsonMessage)
+ );
+}
+
+export function selectIOPreviewParserResult(
+ normalizedResult: ChatMLParserResult | undefined,
+ legacyResult: ChatMLParserResult,
+): IOPreviewParserResult {
+ const normalizedWorks =
+ normalizedResult !== undefined &&
+ hasRenderableChatMessages(normalizedResult);
+ const legacyWorks = hasRenderableChatMessages(legacyResult);
+
+ if (normalizedWorks && legacyWorks) {
+ return {
+ result: normalizedResult,
+ comparisonOutcome: "both",
+ };
+ }
+
+ if (normalizedWorks) {
+ return {
+ result: normalizedResult,
+ comparisonOutcome: "normalized_only",
+ };
+ }
+
+ if (legacyWorks) {
+ return {
+ result: legacyResult,
+ comparisonOutcome: "legacy_only",
+ };
+ }
+
+ return {
+ // Keep normalized data when available so tool definitions and other
+ // extracted details remain visible while the UI falls back to JSON.
+ result: normalizedResult ?? legacyResult,
+ comparisonOutcome: "neither",
+ };
+}
/**
* Selects the parser used by the pretty I/O preview.
@@ -24,7 +80,7 @@ export function useIOPreviewParser(
preParsedOutput?: unknown,
preParsedMetadata?: unknown,
preParsedResult?: ChatMLParserResult,
-): ChatMLParserResult {
+): IOPreviewParserResult {
const parsedInput = preParsedResult
? undefined
: preParsedInput !== undefined
@@ -41,26 +97,31 @@ export function useIOPreviewParser(
? preParsedMetadata
: deepParseJson(metadata, { maxSize: 100_000, maxDepth: 25 });
- return useMemo(() => {
+ return useMemo<IOPreviewParserResult>(() => {
// Precomputed results win regardless of parser mode; surfaces that supply
// one disable the normalized-beta tab (see IOPreview) so labels stay honest.
- if (preParsedResult) return preParsedResult;
+ if (preParsedResult) return { result: preParsedResult };
const parseLegacy = () =>
parseChatML(parsedInput, parsedOutput, parsedMetadata, observationName);
- if (parser === "legacy") return parseLegacy();
+ if (parser === "legacy") return { result: parseLegacy() };
+ let normalizedResult: ChatMLParserResult | undefined;
try {
const normalized = normalizeSpanIO({
input: parsedInput,
output: parsedOutput,
metadata: parsedMetadata,
});
- return toIOPreview(normalized, parsedInput);
+ normalizedResult = toIOPreview(normalized, parsedInput);
} catch {
- return parseLegacy();
+ // The legacy parser remains the visible fallback for malformed input.
}
+
+ const legacyResult = parseLegacy();
+
+ return selectIOPreviewParserResult(normalizedResult, legacyResult);
}, [
parser,
preParsedResult,