🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(io-preview): add parser comparison event capture (#17203)
marliessophief733b5eb73b1
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.tsindex fda0cdb705..e299b9ad3e 100644--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts@@ -26,6 +26,7 @@ const events = {"observation_tree_toggle_scores","observation_tree_toggle_metrics","io_mode_switch",+ "io_parser_comparison","io_pretty_format_toggle_group","test_in_playground_button_click","display_mode_switch",diff --git a/web/src/features/traces/components/IOPreview/IOPreview.tsx b/web/src/features/traces/components/IOPreview/IOPreview.tsxindex 5218d8c367..eac079b4d4 100644--- a/web/src/features/traces/components/IOPreview/IOPreview.tsx+++ b/web/src/features/traces/components/IOPreview/IOPreview.tsx@@ -12,6 +12,7 @@ import { IOPreviewJSON, type IOPreviewJSONProps } from "./IOPreviewJSON";import { IOPreviewJSONSimple } from "./IOPreviewJSONSimple";import { IOPreviewPretty } from "./IOPreviewPretty";import { type ChatMLParserResult } from "../../hooks/useChatMLParser";+import type { IOPreviewParserComparisonOutcome } from "../../hooks/useIOPreviewParser";import { Button } from "@/src/components/ui/button";import { ActionButton } from "@/src/components/ActionButton";import { BookOpen, X } from "lucide-react";@@ -317,6 +318,9 @@ export function IOPreview({? "normalized": "legacy"}+ onParserComparison={(outcome: IOPreviewParserComparisonOutcome) =>+ capture("trace_detail:io_parser_comparison", { outcome })+ }observationName={observationName}showMetadata={showMetadata}contentMode={contentMode}diff --git a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsxindex eaa7cac99f..94e2d6f51b 100644--- a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx+++ b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx@@ -1,4 +1,4 @@-import { useMemo } from "react";+import { useEffect, useMemo, useRef } from "react";import { type Prisma, type ScoreDomain, deepParseJson } from "@langfuse/shared";import { PrettyJsonView } from "@/src/components/ui/PrettyJsonView";import { type MetadataFilterActions } from "@/src/components/table/ValueCell";@@ -6,7 +6,9 @@ import { useMarkdownRenderCharacterLimit } from "@/src/hooks/useMarkdownRenderChimport { type MediaReturnType } from "@/src/features/media/validation";import { type ChatMLParserResult } from "../../hooks/useChatMLParser";import {+ type IOPreviewParserComparisonOutcome,type IOPreviewParserMode,+ hasRenderableChatMessages,useIOPreviewParser,} from "../../hooks/useIOPreviewParser";import { ChatMessageList } from "../ChatMessageList";@@ -16,7 +18,6 @@ import {type IOPreviewContentMode,} from "./IOPreview";import { CorrectedOutputField } from "./components/CorrectedOutputField";-import { isOnlyJsonMessage } from "../../fns/chatMessageUtils";import { StatusMessageSection } from "./components/StatusMessageSection";import type { ObservationStatusMessage } from "./components/statusMessagePresentation";@@ -116,6 +117,8 @@ export interface IOPreviewPrettyProps extends ExpansionStateProps {// Which parser produces the preview; the normalized parser is admin-only// while it is being validated. Legacy remains the safe default.parser?: IOPreviewParserMode;+ // Called once after a normalized parser comparison has settled.+ onParserComparison?: (outcome: IOPreviewParserComparisonOutcome) => void;}/**@@ -162,6 +165,7 @@ export function IOPreviewPretty({contentMode = "all",showSystemPrompt,parser = "legacy",+ onParserComparison,}: IOPreviewPrettyProps) {// Use pre-parsed data if available (from useParsedObservation hook),// otherwise parse with size/depth limits to prevent UI freeze@@ -191,17 +195,7 @@ export function IOPreviewPretty({// Parse into the shared preview contract. The normalized parser is opt-in// while it is being rolled out; legacy remains the safe default.- const {- canDisplayAsChat,- allMessages,- additionalInput,- allTools,- toolCallCounts,- toolCallsByName,- messageToToolCallNumbers,- toolNameToDefinitionNumber,- inputMessageCount,- } = useIOPreviewParser(+ const { result: parserResult, comparisonOutcome } = useIOPreviewParser(parser,input,output,@@ -213,6 +207,44 @@ export function IOPreviewPretty({chatMLParserResult,);+ const {+ allMessages,+ additionalInput,+ allTools,+ toolCallCounts,+ toolCallsByName,+ messageToToolCallNumbers,+ toolNameToDefinitionNumber,+ inputMessageCount,+ } = parserResult;++ const capturedComparisonRecord = useRef<string | undefined>(undefined);+ useEffect(() => {+ if (+ parser !== "normalized" ||+ comparisonOutcome === undefined ||+ !onParserComparison ||+ isLoading ||+ isParsing+ ) {+ return;+ }++ const recordKey = `${observationId ? "observation" : "trace"}:${observationId ?? traceId}`;+ if (capturedComparisonRecord.current === recordKey) return;++ capturedComparisonRecord.current = recordKey;+ onParserComparison(comparisonOutcome);+ }, [+ comparisonOutcome,+ isLoading,+ isParsing,+ observationId,+ onParserComparison,+ parser,+ traceId,+ ]);+const characterLimit = useMarkdownRenderCharacterLimit();// Determine if markdown is safe to render (content size check)@@ -278,8 +310,7 @@ export function IOPreviewPretty({// Determine if metadata should be shownconst shouldShowMetadata = showMetadata && parsedMetadata !== undefined;const showData = contentMode !== "conversation";- const shouldRenderMessages =- canDisplayAsChat && !allMessages.every(isOnlyJsonMessage);+ const shouldRenderMessages = hasRenderableChatMessages(parserResult);return (<div>diff --git a/web/src/features/traces/hooks/useIOPreviewParser.clienttest.ts b/web/src/features/traces/hooks/useIOPreviewParser.clienttest.tsnew file mode 100644index 0000000000..8b41d32858--- /dev/null+++ b/web/src/features/traces/hooks/useIOPreviewParser.clienttest.ts@@ -0,0 +1,66 @@+import { describe, expect, it } from "vitest";+import type { ChatMLParserResult, ChatMlMessage } from "./useChatMLParser";+import {+ hasRenderableChatMessages,+ selectIOPreviewParserResult,+} from "./useIOPreviewParser";++const createMessage = (fields: Partial<ChatMlMessage>): ChatMlMessage =>+ fields as ChatMlMessage;++const result = (+ overrides: Partial<ChatMLParserResult> = {},+): ChatMLParserResult => ({+ canDisplayAsChat: true,+ allMessages: [createMessage({ role: "user", content: "hello" })],+ additionalInput: undefined,+ allTools: [],+ toolCallCounts: new Map(),+ toolCallsByName: new Map(),+ messageToToolCallNumbers: new Map(),+ toolNameToDefinitionNumber: new Map(),+ inputMessageCount: 1,+ ...overrides,+});++describe("IO preview parser comparison", () => {+ it.each([+ ["both", result(), result()],+ ["normalized_only", result(), result({ canDisplayAsChat: false })],+ ["legacy_only", result({ canDisplayAsChat: false }), result()],+ [+ "neither",+ result({ canDisplayAsChat: false }),+ result({ canDisplayAsChat: false }),+ ],+ ] as const)("reports %s", (outcome, normalized, legacy) => {+ expect(+ selectIOPreviewParserResult(normalized, legacy).comparisonOutcome,+ ).toBe(outcome);+ });++ it("treats a JSON-only message as not renderable chat", () => {+ expect(+ hasRenderableChatMessages(+ result({+ allMessages: [+ createMessage({+ role: "user",+ json: { raw: "value" },+ }),+ ],+ }),+ ),+ ).toBe(false);+ });++ it("falls back to legacy when normalized has no renderable messages", () => {+ const normalized = result({ canDisplayAsChat: false });+ const legacy = result();++ expect(selectIOPreviewParserResult(normalized, legacy)).toEqual({+ result: legacy,+ comparisonOutcome: "legacy_only",+ });+ });+});diff --git a/web/src/features/traces/hooks/useIOPreviewParser.ts b/web/src/features/traces/hooks/useIOPreviewParser.tsindex d27007078d..488cbd0acb 100644--- a/web/src/features/traces/hooks/useIOPreviewParser.ts+++ b/web/src/features/traces/hooks/useIOPreviewParser.ts@@ -3,8 +3,64 @@ import { type Prisma, deepParseJson } from "@langfuse/shared";import { normalizeSpanIO } from "@langfuse/shared/src/utils/normalized-io";import { toIOPreview } from "../parsers/toIOPreview";import { parseChatML, type ChatMLParserResult } from "./useChatMLParser";+import { isOnlyJsonMessage } from "../fns/chatMessageUtils";export type IOPreviewParserMode = "legacy" | "normalized";+export type IOPreviewParserComparisonOutcome =+ | "both"+ | "normalized_only"+ | "legacy_only"+ | "neither";++export interface IOPreviewParserResult {+ result: ChatMLParserResult;+ comparisonOutcome?: IOPreviewParserComparisonOutcome;+}++/** The same message-level displayability check used by the pretty preview. */+export function hasRenderableChatMessages(result: ChatMLParserResult): boolean {+ return (+ result.canDisplayAsChat && !result.allMessages.every(isOnlyJsonMessage)+ );+}++export function selectIOPreviewParserResult(+ normalizedResult: ChatMLParserResult | undefined,+ legacyResult: ChatMLParserResult,+): IOPreviewParserResult {+ const normalizedWorks =+ normalizedResult !== undefined &&+ hasRenderableChatMessages(normalizedResult);+ const legacyWorks = hasRenderableChatMessages(legacyResult);++ if (normalizedWorks && legacyWorks) {+ return {+ result: normalizedResult,+ comparisonOutcome: "both",+ };+ }++ if (normalizedWorks) {+ return {+ result: normalizedResult,+ comparisonOutcome: "normalized_only",+ };+ }++ if (legacyWorks) {+ return {+ result: legacyResult,+ comparisonOutcome: "legacy_only",+ };+ }++ return {+ // Keep normalized data when available so tool definitions and other+ // extracted details remain visible while the UI falls back to JSON.+ result: normalizedResult ?? legacyResult,+ comparisonOutcome: "neither",+ };+}/*** Selects the parser used by the pretty I/O preview.@@ -24,7 +80,7 @@ export function useIOPreviewParser(preParsedOutput?: unknown,preParsedMetadata?: unknown,preParsedResult?: ChatMLParserResult,-): ChatMLParserResult {+): IOPreviewParserResult {const parsedInput = preParsedResult? undefined: preParsedInput !== undefined@@ -41,26 +97,31 @@ export function useIOPreviewParser(? preParsedMetadata: deepParseJson(metadata, { maxSize: 100_000, maxDepth: 25 });- return useMemo(() => {+ return useMemo<IOPreviewParserResult>(() => {// Precomputed results win regardless of parser mode; surfaces that supply// one disable the normalized-beta tab (see IOPreview) so labels stay honest.- if (preParsedResult) return preParsedResult;+ if (preParsedResult) return { result: preParsedResult };const parseLegacy = () =>parseChatML(parsedInput, parsedOutput, parsedMetadata, observationName);- if (parser === "legacy") return parseLegacy();+ if (parser === "legacy") return { result: parseLegacy() };+ let normalizedResult: ChatMLParserResult | undefined;try {const normalized = normalizeSpanIO({input: parsedInput,output: parsedOutput,metadata: parsedMetadata,});- return toIOPreview(normalized, parsedInput);+ normalizedResult = toIOPreview(normalized, parsedInput);} catch {- return parseLegacy();+ // The legacy parser remains the visible fallback for malformed input.}++ const legacyResult = parseLegacy();++ return selectIOPreviewParserResult(normalizedResult, legacyResult);}, [parser,preParsedResult,