Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(analytics): track user-visible error toasts (#18324)

Nikita Kabardin1a21a4229c98
diff --git a/web/src/features/notifications/ErrorNotification.clienttest.tsx b/web/src/features/notifications/ErrorNotification.clienttest.tsx
new file mode 100644
index 0000000000..f693a1e1a9
--- /dev/null
+++ b/web/src/features/notifications/ErrorNotification.clienttest.tsx
@@ -0,0 +1,114 @@
+import { render, waitFor } from "@testing-library/react";
+import { vi } from "vitest";
+import { ErrorNotification } from "@/src/features/notifications/ErrorNotification";
+
+const mocks = vi.hoisted(() => ({
+ capture: vi.fn(),
+ copy: vi.fn().mockResolvedValue(undefined),
+ setMigrationPanelOpen: vi.fn(),
+ setSupportDrawerOpen: vi.fn(),
+}));
+
+vi.mock("@/src/features/posthog-analytics", () => ({
+ usePostHogClientCapture: () => mocks.capture,
+}));
+
+vi.mock("@/src/features/support-chat", () => ({
+ useSupportDrawer: () => ({ setOpen: mocks.setSupportDrawerOpen }),
+}));
+
+vi.mock("@/src/features/v4-migration/V4MigrationPanelProvider", () => ({
+ useV4MigrationPanel: () => ({ setOpen: mocks.setMigrationPanelOpen }),
+}));
+
+vi.mock("@/src/hooks/useCopyToClipboard", () => ({
+ useCopyToClipboard: () => ({ copy: mocks.copy, isCopied: false }),
+}));
+
+describe("ErrorNotification", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("captures one metadata-only shown event for a rendered error toast", async () => {
+ const props = {
+ error: "Sensitive error title",
+ description: "Sensitive customer-controlled details",
+ type: "ERROR" as const,
+ dismissToast: vi.fn(),
+ toast: "toast-1",
+ source: "trpc" as const,
+ path: "traces.byId",
+ traceId: "0123456789abcdef",
+ };
+
+ const view = render(<ErrorNotification {...props} />);
+
+ await waitFor(() => {
+ expect(mocks.capture).toHaveBeenCalledWith("toast:shown", {
+ toastType: "ERROR",
+ source: "trpc",
+ path: "traces.byId",
+ hasErrorId: true,
+ });
+ });
+
+ view.rerender(
+ <ErrorNotification
+ {...props}
+ type="WARNING"
+ path="scores.byId"
+ traceId={undefined}
+ />,
+ );
+
+ expect(mocks.capture).toHaveBeenCalledTimes(1);
+ expect(JSON.stringify(mocks.capture.mock.calls[0]?.[1])).not.toContain(
+ "Sensitive",
+ );
+ expect(JSON.stringify(mocks.capture.mock.calls[0]?.[1])).not.toContain(
+ props.traceId,
+ );
+ });
+
+ it("classifies a local warning without inventing tRPC metadata", async () => {
+ render(
+ <ErrorNotification
+ error="Warning"
+ description="Try again"
+ type="WARNING"
+ dismissToast={vi.fn()}
+ toast="toast-2"
+ />,
+ );
+
+ await waitFor(() => {
+ expect(mocks.capture).toHaveBeenCalledWith("toast:shown", {
+ toastType: "WARNING",
+ source: "application",
+ hasErrorId: false,
+ });
+ });
+ });
+
+ it("classifies a pathless tRPC warning by its explicit source", async () => {
+ render(
+ <ErrorNotification
+ error="Unexpected Response"
+ description="Try again"
+ type="WARNING"
+ source="trpc"
+ dismissToast={vi.fn()}
+ toast="toast-3"
+ />,
+ );
+
+ await waitFor(() => {
+ expect(mocks.capture).toHaveBeenCalledWith("toast:shown", {
+ toastType: "WARNING",
+ source: "trpc",
+ hasErrorId: false,
+ });
+ });
+ });
+});
diff --git a/web/src/features/notifications/ErrorNotification.tsx b/web/src/features/notifications/ErrorNotification.tsx
index 44fd2b212c..0a9b662771 100644
--- a/web/src/features/notifications/ErrorNotification.tsx
+++ b/web/src/features/notifications/ErrorNotification.tsx
@@ -4,6 +4,7 @@ import { useSupportDrawer } from "@/src/features/support-chat";
import { useV4MigrationPanel } from "@/src/features/v4-migration/V4MigrationPanelProvider";
import { useCopyToClipboard } from "@/src/hooks/useCopyToClipboard";
import { AlertTriangle, Check, Copy, X } from "lucide-react";
+import { useRef } from "react";
interface ErrorNotificationProps {
error: string;
@@ -11,6 +12,7 @@ interface ErrorNotificationProps {
type: "WARNING" | "ERROR";
dismissToast: (t?: string | number | undefined) => void;
toast: string | number;
+ source?: "application" | "trpc";
path?: string;
traceId?: string;
}
@@ -21,6 +23,7 @@ export const ErrorNotification: React.FC<ErrorNotificationProps> = ({
type,
dismissToast,
toast,
+ source = "application",
path,
traceId,
}) => {
@@ -28,13 +31,26 @@ export const ErrorNotification: React.FC<ErrorNotificationProps> = ({
const { setOpen: setMigrationPanelOpen } = useV4MigrationPanel();
const capture = usePostHogClientCapture();
const { copy, isCopied } = useCopyToClipboard();
+ const didCaptureShown = useRef(false);
const isError = type === "ERROR";
const textColor = isError
? "text-destructive-foreground"
: "text-dark-yellow";
+ const captureShown = (element: HTMLDivElement | null) => {
+ if (!element || didCaptureShown.current) return;
+ didCaptureShown.current = true;
+
+ capture("toast:shown", {
+ toastType: type,
+ source,
+ ...(path ? { path } : {}),
+ hasErrorId: Boolean(traceId),
+ });
+ };
+
return (
- <div className="flex justify-between">
+ <div ref={captureShown} className="flex justify-between">
<div className="flex min-w-[300px] flex-1 flex-col gap-2">
<div className="flex items-center gap-2">
<AlertTriangle className={`icon-lg ${textColor}`} />
diff --git a/web/src/features/notifications/showErrorToast.tsx b/web/src/features/notifications/showErrorToast.tsx
index 68404bec69..81ed91476e 100644
--- a/web/src/features/notifications/showErrorToast.tsx
+++ b/web/src/features/notifications/showErrorToast.tsx
@@ -22,6 +22,7 @@ export const showErrorToast = (
type: "WARNING" | "ERROR" = "ERROR",
path?: string,
traceId?: string,
+ source: "application" | "trpc" = "application",
) => {
toast.custom(
(t) => (
@@ -29,6 +30,7 @@ export const showErrorToast = (
error={error}
description={description}
type={type}
+ source={source}
path={path}
traceId={traceId}
dismissToast={toast.dismiss}
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index 12d6a0026d..7c94bdd0fd 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -6,6 +6,15 @@ import type { EvalOnboardingEventMap } from "@/src/features/evals/v2/types/evalO
export const V4_BETA_ENABLED_POSTHOG_PROPERTY = "v4BetaEnabled";
+type ToastEventMap = {
+ "toast:shown": {
+ toastType: "WARNING" | "ERROR";
+ source: "trpc" | "application";
+ path?: string;
+ hasErrorId: boolean;
+ };
+};
+
// resource:action, only use snake_case
// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Used via typeof
const events = {
@@ -361,7 +370,9 @@ const events = {
// props carry user content.
version_update: ["banner_shown", "reload_clicked", "dismissed"],
notification: ["click_link", "dismiss_notification"],
- toast: ["report_issue", "dismiss"],
+ // User-visible error-frustration denominator. Metadata only: `path` is the
+ // static tRPC procedure name; never send toast text or the raw Error ID.
+ toast: ["shown", "report_issue", "dismiss"],
tag: [
"add_existing_tag",
"remove_tag",
@@ -487,7 +498,9 @@ type EventName = {
[Resource in keyof typeof events]: `${Resource}:${(typeof events)[Resource][number]}`;
}[keyof typeof events];
-type TypedEventMap = AnnotationEventMap & EvalOnboardingEventMap;
+type TypedEventMap = AnnotationEventMap &
+ EvalOnboardingEventMap &
+ ToastEventMap;
type EventProperties = TypedEventMap & {
[E in Exclude<EventName, keyof TypedEventMap>]: Record<string, any> | null;
diff --git a/web/src/utils/trpcErrorToast.clienttest.ts b/web/src/utils/trpcErrorToast.clienttest.ts
index 0912ff344d..5608aed4f1 100644
--- a/web/src/utils/trpcErrorToast.clienttest.ts
+++ b/web/src/utils/trpcErrorToast.clienttest.ts
@@ -110,6 +110,23 @@ describe("trpcErrorToast", () => {
showErrorToastMock.mockClear();
});
+ it("classifies a pathless response parse failure as tRPC", () => {
+ trpcErrorToast(
+ new TRPCClientError("Unexpected token", {
+ cause: new SyntaxError("Unexpected token"),
+ }),
+ );
+
+ expect(showErrorToastMock).toHaveBeenCalledWith(
+ "Unexpected Response",
+ "The request could not be completed. Please try again or contact support if this persists.",
+ "WARNING",
+ undefined,
+ undefined,
+ "trpc",
+ );
+ });
+
it("shows a readable Invalid input toast instead of the Zod JSON dump", () => {
trpcErrorToast(
trpcError({
@@ -126,6 +143,7 @@ describe("trpcErrorToast", () => {
"WARNING",
"prompts.create",
undefined,
+ "trpc",
);
});
@@ -145,6 +163,7 @@ describe("trpcErrorToast", () => {
"WARNING",
"prompts.create",
undefined,
+ "trpc",
);
});
@@ -165,6 +184,7 @@ describe("trpcErrorToast", () => {
"ERROR",
"prompts.create",
"abc123def456",
+ "trpc",
);
});
});
diff --git a/web/src/utils/trpcErrorToast.tsx b/web/src/utils/trpcErrorToast.tsx
index 2d3b1d6eb0..f2844b4041 100644
--- a/web/src/utils/trpcErrorToast.tsx
+++ b/web/src/utils/trpcErrorToast.tsx
@@ -89,6 +89,9 @@ export const trpcErrorToast = (error: unknown) => {
"Unexpected Response",
"The request could not be completed. Please try again or contact support if this persists.",
"WARNING",
+ undefined,
+ undefined,
+ "trpc",
);
return;
}
@@ -111,12 +114,16 @@ export const trpcErrorToast = (error: unknown) => {
httpStatus >= 500 && httpStatus < 600 ? "ERROR" : "WARNING",
path,
traceId,
+ "trpc",
);
} else {
showErrorToast(
"Unexpected Error",
"An unexpected error occurred.",
"ERROR",
+ undefined,
+ undefined,
+ "trpc",
);
}
};