🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(analytics): track user-visible error toasts (#18324)
Nikita Kabardin1a21a4229c98
diff --git a/web/src/features/notifications/ErrorNotification.clienttest.tsx b/web/src/features/notifications/ErrorNotification.clienttest.tsxnew file mode 100644index 0000000000..f693a1e1a9--- /dev/null+++ b/web/src/features/notifications/ErrorNotification.clienttest.tsx@@ -0,0 +1,114 @@+import { render, waitFor } from "@testing-library/react";+import { vi } from "vitest";+import { ErrorNotification } from "@/src/features/notifications/ErrorNotification";++const mocks = vi.hoisted(() => ({+ capture: vi.fn(),+ copy: vi.fn().mockResolvedValue(undefined),+ setMigrationPanelOpen: vi.fn(),+ setSupportDrawerOpen: vi.fn(),+}));++vi.mock("@/src/features/posthog-analytics", () => ({+ usePostHogClientCapture: () => mocks.capture,+}));++vi.mock("@/src/features/support-chat", () => ({+ useSupportDrawer: () => ({ setOpen: mocks.setSupportDrawerOpen }),+}));++vi.mock("@/src/features/v4-migration/V4MigrationPanelProvider", () => ({+ useV4MigrationPanel: () => ({ setOpen: mocks.setMigrationPanelOpen }),+}));++vi.mock("@/src/hooks/useCopyToClipboard", () => ({+ useCopyToClipboard: () => ({ copy: mocks.copy, isCopied: false }),+}));++describe("ErrorNotification", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("captures one metadata-only shown event for a rendered error toast", async () => {+ const props = {+ error: "Sensitive error title",+ description: "Sensitive customer-controlled details",+ type: "ERROR" as const,+ dismissToast: vi.fn(),+ toast: "toast-1",+ source: "trpc" as const,+ path: "traces.byId",+ traceId: "0123456789abcdef",+ };++ const view = render(<ErrorNotification {...props} />);++ await waitFor(() => {+ expect(mocks.capture).toHaveBeenCalledWith("toast:shown", {+ toastType: "ERROR",+ source: "trpc",+ path: "traces.byId",+ hasErrorId: true,+ });+ });++ view.rerender(+ <ErrorNotification+ {...props}+ type="WARNING"+ path="scores.byId"+ traceId={undefined}+ />,+ );++ expect(mocks.capture).toHaveBeenCalledTimes(1);+ expect(JSON.stringify(mocks.capture.mock.calls[0]?.[1])).not.toContain(+ "Sensitive",+ );+ expect(JSON.stringify(mocks.capture.mock.calls[0]?.[1])).not.toContain(+ props.traceId,+ );+ });++ it("classifies a local warning without inventing tRPC metadata", async () => {+ render(+ <ErrorNotification+ error="Warning"+ description="Try again"+ type="WARNING"+ dismissToast={vi.fn()}+ toast="toast-2"+ />,+ );++ await waitFor(() => {+ expect(mocks.capture).toHaveBeenCalledWith("toast:shown", {+ toastType: "WARNING",+ source: "application",+ hasErrorId: false,+ });+ });+ });++ it("classifies a pathless tRPC warning by its explicit source", async () => {+ render(+ <ErrorNotification+ error="Unexpected Response"+ description="Try again"+ type="WARNING"+ source="trpc"+ dismissToast={vi.fn()}+ toast="toast-3"+ />,+ );++ await waitFor(() => {+ expect(mocks.capture).toHaveBeenCalledWith("toast:shown", {+ toastType: "WARNING",+ source: "trpc",+ hasErrorId: false,+ });+ });+ });+});diff --git a/web/src/features/notifications/ErrorNotification.tsx b/web/src/features/notifications/ErrorNotification.tsxindex 44fd2b212c..0a9b662771 100644--- a/web/src/features/notifications/ErrorNotification.tsx+++ b/web/src/features/notifications/ErrorNotification.tsx@@ -4,6 +4,7 @@ import { useSupportDrawer } from "@/src/features/support-chat";import { useV4MigrationPanel } from "@/src/features/v4-migration/V4MigrationPanelProvider";import { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";import { AlertTriangle, Check, Copy, X } from "lucide-react";+import { useRef } from "react";interface ErrorNotificationProps {error: string;@@ -11,6 +12,7 @@ interface ErrorNotificationProps {type: "WARNING" | "ERROR";dismissToast: (t?: string | number | undefined) => void;toast: string | number;+ source?: "application" | "trpc";path?: string;traceId?: string;}@@ -21,6 +23,7 @@ export const ErrorNotification: React.FC<ErrorNotificationProps> = ({type,dismissToast,toast,+ source = "application",path,traceId,}) => {@@ -28,13 +31,26 @@ export const ErrorNotification: React.FC<ErrorNotificationProps> = ({const { setOpen: setMigrationPanelOpen } = useV4MigrationPanel();const capture = usePostHogClientCapture();const { copy, isCopied } = useCopyToClipboard();+ const didCaptureShown = useRef(false);const isError = type === "ERROR";const textColor = isError? "text-destructive-foreground": "text-dark-yellow";+ const captureShown = (element: HTMLDivElement | null) => {+ if (!element || didCaptureShown.current) return;+ didCaptureShown.current = true;++ capture("toast:shown", {+ toastType: type,+ source,+ ...(path ? { path } : {}),+ hasErrorId: Boolean(traceId),+ });+ };+return (- <div className="flex justify-between">+ <div ref={captureShown} className="flex justify-between"><div className="flex min-w-[300px] flex-1 flex-col gap-2"><div className="flex items-center gap-2"><AlertTriangle className={`icon-lg ${textColor}`} />diff --git a/web/src/features/notifications/showErrorToast.tsx b/web/src/features/notifications/showErrorToast.tsxindex 68404bec69..81ed91476e 100644--- a/web/src/features/notifications/showErrorToast.tsx+++ b/web/src/features/notifications/showErrorToast.tsx@@ -22,6 +22,7 @@ export const showErrorToast = (type: "WARNING" | "ERROR" = "ERROR",path?: string,traceId?: string,+ source: "application" | "trpc" = "application",) => {toast.custom((t) => (@@ -29,6 +30,7 @@ export const showErrorToast = (error={error}description={description}type={type}+ source={source}path={path}traceId={traceId}dismissToast={toast.dismiss}diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.tsindex 12d6a0026d..7c94bdd0fd 100644--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts@@ -6,6 +6,15 @@ import type { EvalOnboardingEventMap } from "@/src/features/evals/v2/types/evalOexport const V4_BETA_ENABLED_POSTHOG_PROPERTY = "v4BetaEnabled";+type ToastEventMap = {+ "toast:shown": {+ toastType: "WARNING" | "ERROR";+ source: "trpc" | "application";+ path?: string;+ hasErrorId: boolean;+ };+};+// resource:action, only use snake_case// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via typeofconst events = {@@ -361,7 +370,9 @@ const events = {// props carry user content.version_update: ["banner_shown", "reload_clicked", "dismissed"],notification: ["click_link", "dismiss_notification"],- toast: ["report_issue", "dismiss"],+ // User-visible error-frustration denominator. Metadata only: `path` is the+ // static tRPC procedure name; never send toast text or the raw Error ID.+ toast: ["shown", "report_issue", "dismiss"],tag: ["add_existing_tag","remove_tag",@@ -487,7 +498,9 @@ type EventName = {[Resource in keyof typeof events]: `${Resource}:${(typeof events)[Resource][number]}`;}[keyof typeof events];-type TypedEventMap = AnnotationEventMap & EvalOnboardingEventMap;+type TypedEventMap = AnnotationEventMap &+ EvalOnboardingEventMap &+ ToastEventMap;type EventProperties = TypedEventMap & {[E in Exclude<EventName, keyof TypedEventMap>]: Record<string, any> | null;diff --git a/web/src/utils/trpcErrorToast.clienttest.ts b/web/src/utils/trpcErrorToast.clienttest.tsindex 0912ff344d..5608aed4f1 100644--- a/web/src/utils/trpcErrorToast.clienttest.ts+++ b/web/src/utils/trpcErrorToast.clienttest.ts@@ -110,6 +110,23 @@ describe("trpcErrorToast", () => {showErrorToastMock.mockClear();});+ it("classifies a pathless response parse failure as tRPC", () => {+ trpcErrorToast(+ new TRPCClientError("Unexpected token", {+ cause: new SyntaxError("Unexpected token"),+ }),+ );++ expect(showErrorToastMock).toHaveBeenCalledWith(+ "Unexpected Response",+ "The request could not be completed. Please try again or contact support if this persists.",+ "WARNING",+ undefined,+ undefined,+ "trpc",+ );+ });+it("shows a readable Invalid input toast instead of the Zod JSON dump", () => {trpcErrorToast(trpcError({@@ -126,6 +143,7 @@ describe("trpcErrorToast", () => {"WARNING","prompts.create",undefined,+ "trpc",);});@@ -145,6 +163,7 @@ describe("trpcErrorToast", () => {"WARNING","prompts.create",undefined,+ "trpc",);});@@ -165,6 +184,7 @@ describe("trpcErrorToast", () => {"ERROR","prompts.create","abc123def456",+ "trpc",);});});diff --git a/web/src/utils/trpcErrorToast.tsx b/web/src/utils/trpcErrorToast.tsxindex 2d3b1d6eb0..f2844b4041 100644--- a/web/src/utils/trpcErrorToast.tsx+++ b/web/src/utils/trpcErrorToast.tsx@@ -89,6 +89,9 @@ export const trpcErrorToast = (error: unknown) => {"Unexpected Response","The request could not be completed. Please try again or contact support if this persists.","WARNING",+ undefined,+ undefined,+ "trpc",);return;}@@ -111,12 +114,16 @@ export const trpcErrorToast = (error: unknown) => {httpStatus >= 500 && httpStatus < 600 ? "ERROR" : "WARNING",path,traceId,+ "trpc",);} else {showErrorToast("Unexpected Error","An unexpected error occurred.","ERROR",+ undefined,+ undefined,+ "trpc",);}};