🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(traces): integrate PostHog analytics for IO parser comparison tracking (#18896)
marliessophie4063c022086e
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.tsindex 0decc233ba..f9aa25cfea 100644--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts@@ -38,6 +38,7 @@ const events = {"observation_tree_toggle_scores","observation_tree_toggle_metrics","io_mode_switch",+ "io_parser_comparison","test_in_playground_button_click","display_mode_switch","download_button_click",diff --git a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsxindex 1c9353c400..4c988b55ad 100644--- a/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx+++ b/web/src/features/traces/components/IOPreview/IOPreviewPretty.tsx@@ -1,5 +1,5 @@/* eslint-disable no-nested-ternary */-import { useMemo } from "react";+import { useEffect, useMemo, useRef } from "react";import { cn } from "@/src/utils/tailwind";import {type ObservationType,@@ -15,6 +15,7 @@ import {hasRenderableChatMessages,useIOPreviewParser,} from "../../hooks/useIOPreviewParser";+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import { ChatMessageList } from "../ChatMessageList";import { SectionToolDefinitions } from "./components/SectionToolDefinitions";import {@@ -193,6 +194,19 @@ export function IOPreviewPretty({chatMLParserResult,);+ const shouldRenderMessages = hasRenderableChatMessages(parserResult);++ useIOParserComparisonCapture({+ input,+ output,+ chatMLParserResult,+ isLoading,+ isParsing,+ observationId,+ traceId,+ shouldRenderMessages,+ });+const {allMessages,additionalInput,@@ -267,7 +281,6 @@ export function IOPreviewPretty({};const showData = contentMode !== "conversation";- const shouldRenderMessages = hasRenderableChatMessages(parserResult);// Only generations hold a conversation; tools, spans, agents and the rest// are titled Input / Output rather than by role. (`isGenerationLike` is// wider: it also covers TOOL and AGENT, which carry model usage, not chat.)@@ -336,3 +349,60 @@ export function IOPreviewPretty({</div>);}++/** Captures the normalization result for each displayed record. */+function useIOParserComparisonCapture({+ input,+ output,+ chatMLParserResult,+ isLoading,+ isParsing,+ observationId,+ traceId,+ shouldRenderMessages,+}: Pick<+ IOPreviewPrettyProps,+ | "input"+ | "output"+ | "chatMLParserResult"+ | "isLoading"+ | "isParsing"+ | "observationId"+ | "traceId"+> & { shouldRenderMessages: boolean }) {+ const capture = usePostHogClientCapture();+ const capturedNormalizationRecord = useRef<string | undefined>(undefined);++ /** Empty payloads are excluded from the normalization success metric. */+ const hasPayloadToNormalize =+ (input !== null && input !== undefined) ||+ (output !== null && output !== undefined);++ useEffect(() => {+ if (+ chatMLParserResult !== undefined ||+ !hasPayloadToNormalize ||+ isLoading ||+ isParsing+ ) {+ return;+ }++ const recordKey = `${observationId ? "observation" : "trace"}:${observationId ?? traceId}`;+ if (capturedNormalizationRecord.current === recordKey) return;++ capturedNormalizationRecord.current = recordKey;+ capture("trace_detail:io_parser_comparison", {+ isNormalizedSuccess: shouldRenderMessages,+ });+ }, [+ capture,+ chatMLParserResult,+ hasPayloadToNormalize,+ isLoading,+ isParsing,+ observationId,+ shouldRenderMessages,+ traceId,+ ]);+}