Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(v4-migration): track delay badge impressions and hovers (#16260)

Trang Nguyen120a93f1039a
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index b10bf30029..7d65d42578 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -325,6 +325,11 @@ export const events = {
v4_migration: [
"coding_agent_prompt_copied",
"delay_badge_clicked",
+ // Discoverability pair for the table delay badge: `shown` is the
+ // exposure denominator (badge actually rendered), `hovered` counts
+ // noticed-but-not-clicked (pill expanded long enough to read).
+ "delay_badge_shown",
+ "delay_badge_hovered",
"project_chip_clicked",
"contact_book_call_clicked",
"contact_support_clicked",
diff --git a/web/src/features/v4-migration/V4MigrationBadgeContent.tsx b/web/src/features/v4-migration/V4MigrationBadgeContent.tsx
index 11f44a1731..e7aa94c41c 100644
--- a/web/src/features/v4-migration/V4MigrationBadgeContent.tsx
+++ b/web/src/features/v4-migration/V4MigrationBadgeContent.tsx
@@ -27,6 +27,11 @@ export function V4MigrationStatusDot({
type V4MigrationBadgeContentProps = {
onClick: () => void;
+ // Discoverability callbacks: the pill expands its description on hover AND
+ // on keyboard focus (group-focus-visible), so either counts as "noticed
+ // the badge". Wired to mouseenter/mouseleave and focus/blur alike.
+ onHoverStart?: () => void;
+ onHoverEnd?: () => void;
title: string;
description?: string;
showChevron?: boolean;
@@ -35,6 +40,8 @@ type V4MigrationBadgeContentProps = {
export function V4MigrationBadgeContent({
onClick,
+ onHoverStart,
+ onHoverEnd,
title,
description,
showChevron = true,
@@ -45,6 +52,10 @@ export function V4MigrationBadgeContent({
<button
type="button"
onClick={onClick}
+ onMouseEnter={onHoverStart}
+ onMouseLeave={onHoverEnd}
+ onFocus={onHoverStart}
+ onBlur={onHoverEnd}
className="hover:bg-muted/50 hover:text-foreground inline-flex w-fit shrink-0 items-center gap-1.5 rounded-full border px-2 py-0.5 text-xs font-bold whitespace-nowrap"
>
<span
@@ -75,6 +86,10 @@ export function V4MigrationBadgeContent({
<button
type="button"
onClick={onClick}
+ onMouseEnter={onHoverStart}
+ onMouseLeave={onHoverEnd}
+ onFocus={onHoverStart}
+ onBlur={onHoverEnd}
className="group ring-input hover:bg-muted/50 hover:text-foreground col-start-1 row-start-1 inline-flex w-fit flex-none shrink-0 items-center gap-1.5 justify-self-start rounded-full bg-transparent px-2 py-0.5 text-xs font-bold whitespace-nowrap ring"
>
<V4MigrationStatusDot variant="action" />
@@ -83,6 +98,9 @@ export function V4MigrationBadgeContent({
{description ? (
// 0fr -> 1fr animates to the intrinsic text width; a max-width cap
// would clip any description longer than the magic number (LF-90).
+ // The 300ms duration is load-bearing for analytics: HOVER_DWELL_MS
+ // in V4MigrationDelayBadge assumes the description finishes
+ // expanding before the dwell elapses. Keep dwell > duration.
<span className="grid grid-cols-[0fr] transition-[grid-template-columns] duration-300 ease-out group-hover:grid-cols-[1fr] group-focus-visible:grid-cols-[1fr]">
<span className="min-w-0 overflow-hidden">
<span className="whitespace-nowrap">.&nbsp;{description}.</span>
diff --git a/web/src/features/v4-migration/V4MigrationDelayBadge.clienttest.tsx b/web/src/features/v4-migration/V4MigrationDelayBadge.clienttest.tsx
index 45ed5ce7fb..a1b42a26fc 100644
--- a/web/src/features/v4-migration/V4MigrationDelayBadge.clienttest.tsx
+++ b/web/src/features/v4-migration/V4MigrationDelayBadge.clienttest.tsx
@@ -1,5 +1,5 @@
import { fireEvent, render, screen } from "@testing-library/react";
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { V4MigrationDelayBadge } from "./V4MigrationDelayBadge";
import {
@@ -122,7 +122,9 @@ const mocks = vi.hoisted(() => ({
v4UpgradeUiEnabled: true,
forceV3: false,
isBetaEnabled: false,
+ projectId: "project-1",
openMigrationPanel: vi.fn(),
+ capture: vi.fn(),
}));
vi.mock("@/src/features/v4-migration/useV4UpgradeUiEnabled", () => ({
@@ -139,12 +141,12 @@ vi.mock("@/src/features/events/hooks/useV4Beta", () => ({
}));
vi.mock("@/src/features/posthog-analytics/usePostHogClientCapture", () => ({
- usePostHogClientCapture: () => vi.fn(),
+ usePostHogClientCapture: () => mocks.capture,
}));
vi.mock("@/src/features/projects/hooks", () => ({
useQueryProject: () => ({
- project: { id: "project-1", name: "Project One" },
+ project: { id: mocks.projectId, name: "Project One" },
organization: { id: "org-1" },
}),
}));
@@ -198,17 +200,18 @@ describe("V4MigrationDelayBadge", () => {
mocks.v4UpgradeUiEnabled = true;
mocks.forceV3 = false;
mocks.isBetaEnabled = false;
+ mocks.projectId = "project-1";
setSdk("latest", [makeSdkUsageSeries({})]);
});
it("stays hidden when every ingestion path is clean", () => {
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();
});
it("shows SDK copy for outdated SDK traffic", () => {
setSdk("legacy", [outdatedSdkSeries()]);
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(screen.getAllByText("New data in ~15 min").length).toBeGreaterThan(
0,
);
@@ -219,7 +222,7 @@ describe("V4MigrationDelayBadge", () => {
it("shows OTel copy for delayed OTel exporters", () => {
setSdk("otel_header_required", [delayedOtelSeries()]);
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(
screen.getAllByText(/Update your OTel instrumentation for real-time data/)
.length,
@@ -228,7 +231,7 @@ describe("V4MigrationDelayBadge", () => {
it("shows instrumentation copy for headerless ingestion traffic", () => {
setSdk("unknown", [customIngestionSeries()]);
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(
screen.getAllByText(/Upgrade your instrumentation for real-time data/)
.length,
@@ -237,7 +240,7 @@ describe("V4MigrationDelayBadge", () => {
it("shows the generic copy when several delayed paths fire", () => {
setSdk("legacy", [outdatedSdkSeries(), delayedOtelSeries()]);
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(
screen.getAllByText(/Upgrade to v4 for real-time data/).length,
).toBeGreaterThan(0);
@@ -263,20 +266,20 @@ describe("V4MigrationDelayBadge", () => {
upgradeRequiredCount: 0,
delayedOtelIngestionCount: 0,
};
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();
});
it("stays hidden while the check is still running", () => {
setSdk("checking", []);
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();
});
it("stays hidden without the v4 upgrade UI flag", () => {
mocks.v4UpgradeUiEnabled = false;
setSdk("legacy", [outdatedSdkSeries()]);
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();
});
@@ -285,7 +288,7 @@ describe("V4MigrationDelayBadge", () => {
mocks.forceV3 = true;
mocks.isBetaEnabled = false;
setSdk("legacy", [outdatedSdkSeries()]);
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();
});
@@ -295,7 +298,7 @@ describe("V4MigrationDelayBadge", () => {
setSdk("legacy", [outdatedSdkSeries()]);
const windowOpen = vi.spyOn(window, "open").mockImplementation(() => null);
- render(<V4MigrationDelayBadge />);
+ render(<V4MigrationDelayBadge page="traces" />);
expect(
screen.getAllByText(/Learn more in the docs/).length,
).toBeGreaterThan(0);
@@ -307,6 +310,181 @@ describe("V4MigrationDelayBadge", () => {
"noopener,noreferrer",
);
expect(mocks.openMigrationPanel).not.toHaveBeenCalled();
+ expect(mocks.capture).toHaveBeenCalledWith(
+ "v4_migration:delay_badge_clicked",
+ { action: "docs", page: "traces" },
+ );
windowOpen.mockRestore();
});
+
+ describe("discoverability events", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ const hoveredCalls = () =>
+ mocks.capture.mock.calls.filter(
+ ([name]) => name === "v4_migration:delay_badge_hovered",
+ );
+
+ it("fires delay_badge_shown once per mount with the page dimension", () => {
+ setSdk("legacy", [outdatedSdkSeries()]);
+ const { rerender } = render(<V4MigrationDelayBadge page="traces" />);
+ rerender(<V4MigrationDelayBadge page="traces" />);
+ expect(
+ mocks.capture.mock.calls.filter(
+ ([name]) => name === "v4_migration:delay_badge_shown",
+ ),
+ ).toEqual([["v4_migration:delay_badge_shown", { page: "traces" }]]);
+ });
+
+ it("does not fire delay_badge_shown while the badge is hidden", () => {
+ setSdk("checking", []);
+ render(<V4MigrationDelayBadge page="traces" />);
+ expect(mocks.capture).not.toHaveBeenCalled();
+ });
+
+ it("fires delay_badge_hovered once after the dwell threshold", () => {
+ vi.useFakeTimers();
+ setSdk("legacy", [outdatedSdkSeries()]);
+ render(<V4MigrationDelayBadge page="observations" />);
+ const badge = screen.getByRole("button");
+
+ fireEvent.mouseEnter(badge);
+ vi.advanceTimersByTime(500);
+ fireEvent.mouseLeave(badge);
+ // A later hover on the same mount must not double-count.
+ fireEvent.mouseEnter(badge);
+ vi.advanceTimersByTime(500);
+
+ expect(hoveredCalls()).toEqual([
+ ["v4_migration:delay_badge_hovered", { page: "observations" }],
+ ]);
+ });
+
+ it("does not count a drive-by hover shorter than the dwell", () => {
+ vi.useFakeTimers();
+ setSdk("legacy", [outdatedSdkSeries()]);
+ render(<V4MigrationDelayBadge page="traces" />);
+ const badge = screen.getByRole("button");
+
+ fireEvent.mouseEnter(badge);
+ vi.advanceTimersByTime(300);
+ fireEvent.mouseLeave(badge);
+ vi.advanceTimersByTime(1000);
+
+ expect(hoveredCalls()).toHaveLength(0);
+ });
+
+ it("fires delay_badge_shown again when the project changes in place", () => {
+ setSdk("legacy", [outdatedSdkSeries()]);
+ const { rerender } = render(<V4MigrationDelayBadge page="traces" />);
+ // The project switcher navigates within the same route, so the badge is
+ // reconciled (not remounted) while the new project's SDK check runs.
+ setSdk("checking", []);
+ mocks.projectId = "project-2";
+ rerender(<V4MigrationDelayBadge page="traces" />);
+ setSdk("legacy", [outdatedSdkSeries()]);
+ rerender(<V4MigrationDelayBadge page="traces" />);
+ expect(
+ mocks.capture.mock.calls.filter(
+ ([name]) => name === "v4_migration:delay_badge_shown",
+ ),
+ ).toHaveLength(2);
+ });
+
+ it("does not count a hover that ends in a click", () => {
+ vi.useFakeTimers();
+ setSdk("legacy", [outdatedSdkSeries()]);
+ render(<V4MigrationDelayBadge page="traces" />);
+ const badge = screen.getByRole("button");
+
+ fireEvent.mouseEnter(badge);
+ vi.advanceTimersByTime(300);
+ // Also the touch path: a tap synthesizes mouseenter + click and never
+ // delivers a mouseleave to cancel the dwell timer.
+ fireEvent.click(badge);
+ vi.advanceTimersByTime(1000);
+
+ expect(hoveredCalls()).toHaveLength(0);
+ });
+
+ it("cancels a pending dwell when the badge hides without unmounting", () => {
+ vi.useFakeTimers();
+ setSdk("legacy", [outdatedSdkSeries()]);
+ const { rerender } = render(<V4MigrationDelayBadge page="traces" />);
+ fireEvent.mouseEnter(screen.getByRole("button"));
+ vi.advanceTimersByTime(300);
+ // React removes the button without a mouseleave when the SDK check
+ // turns transient mid-hover.
+ setSdk("checking", []);
+ rerender(<V4MigrationDelayBadge page="traces" />);
+ vi.advanceTimersByTime(1000);
+
+ expect(hoveredCalls()).toHaveLength(0);
+ });
+
+ it("does not count a re-hover after the badge was clicked", () => {
+ vi.useFakeTimers();
+ setSdk("legacy", [outdatedSdkSeries()]);
+ render(<V4MigrationDelayBadge page="traces" />);
+ const badge = screen.getByRole("button");
+
+ // `hovered` counts noticed-but-NOT-clicked: once the user clicked,
+ // later dwells on the same exposure must not file them under it.
+ fireEvent.click(badge);
+ fireEvent.mouseEnter(badge);
+ vi.advanceTimersByTime(1000);
+
+ expect(hoveredCalls()).toHaveLength(0);
+ });
+
+ it("re-arms hovered when the user returns to a project in place", () => {
+ vi.useFakeTimers();
+ setSdk("legacy", [outdatedSdkSeries()]);
+ const { rerender } = render(<V4MigrationDelayBadge page="traces" />);
+ const dwell = () => {
+ fireEvent.mouseEnter(screen.getByRole("button"));
+ vi.advanceTimersByTime(500);
+ fireEvent.mouseLeave(screen.getByRole("button"));
+ };
+
+ dwell();
+ // Switch away and back without unmounting (the project switcher stays
+ // on the route). The return re-fires `shown` as a new exposure, so a
+ // fresh dwell must count again — otherwise the pair drifts apart.
+ mocks.projectId = "project-2";
+ rerender(<V4MigrationDelayBadge page="traces" />);
+ mocks.projectId = "project-1";
+ rerender(<V4MigrationDelayBadge page="traces" />);
+ dwell();
+
+ expect(hoveredCalls()).toHaveLength(2);
+ });
+
+ it("counts a keyboard focus dwell as hovered", () => {
+ vi.useFakeTimers();
+ setSdk("legacy", [outdatedSdkSeries()]);
+ render(<V4MigrationDelayBadge page="traces" />);
+
+ // The description also expands via group-focus-visible, so a keyboard
+ // reveal counts as noticing the badge.
+ fireEvent.focus(screen.getByRole("button"));
+ vi.advanceTimersByTime(500);
+
+ expect(hoveredCalls()).toEqual([
+ ["v4_migration:delay_badge_hovered", { page: "traces" }],
+ ]);
+ });
+
+ it("carries the page dimension on delay_badge_clicked", () => {
+ setSdk("legacy", [outdatedSdkSeries()]);
+ render(<V4MigrationDelayBadge page="observations" />);
+ fireEvent.click(screen.getAllByText("New data in ~15 min")[0]);
+ expect(mocks.capture).toHaveBeenCalledWith(
+ "v4_migration:delay_badge_clicked",
+ { page: "observations" },
+ );
+ });
+ });
});
diff --git a/web/src/features/v4-migration/V4MigrationDelayBadge.tsx b/web/src/features/v4-migration/V4MigrationDelayBadge.tsx
index 0925741c93..1bfc4e2990 100644
--- a/web/src/features/v4-migration/V4MigrationDelayBadge.tsx
+++ b/web/src/features/v4-migration/V4MigrationDelayBadge.tsx
@@ -1,4 +1,4 @@
-import { useState } from "react";
+import { useEffect, useRef, useState } from "react";
import { useRouter } from "next/router";
import {
useV4UpgradeUiEnabled,
@@ -23,7 +23,17 @@ import { EvaluatorMigrationDialog } from "@/src/features/v4-migration/EvaluatorM
import { buildDeprecatedEvaluatorsUrl } from "@/src/features/v4-migration/evaluatorMigrationUrls";
import { useV4Beta } from "@/src/features/events/hooks/useV4Beta";
-export function V4MigrationDelayBadge() {
+// The pill's description finishes expanding after 300ms (V4MigrationBadgeContent),
+// so a 500ms dwell means the full text was on screen — a drive-by mouse pass
+// does not count as "noticed".
+const HOVER_DWELL_MS = 500;
+
+export function V4MigrationDelayBadge({
+ page,
+}: {
+ // Which table hosts the badge — clicks/hovers/impressions segment by it.
+ page: "traces" | "observations" | "experiments";
+}) {
const v4UpgradeUiFlagEnabled = useV4UpgradeUiFlag();
const openMigrationPanel = useOpenV4MigrationPanel();
const { project } = useQueryProject();
@@ -56,17 +66,86 @@ export function V4MigrationDelayBadge() {
customActionable,
].filter(Boolean).length;
- if (!enabled || !project || actionablePaths === 0) {
+ const visible = enabled && Boolean(project) && actionablePaths > 0;
+ const projectId = project?.id;
+
+ // Impression + hover exist to answer one question: do users notice the
+ // badge? `shown` is the exposure denominator, `hovered` the
+ // noticed-but-not-clicked middle. Both fire at most once per exposure,
+ // where an exposure ends when the hosting project changes: the project
+ // switcher navigates within the same route, so the component survives the
+ // switch, and a plain once-per-mount boolean would swallow the next
+ // project's exposure (clicks with no matching shown). The effect
+ // synchronizes visibility with PostHog (it can flip to true only after the
+ // SDK check resolves).
+ const shownCapturedForRef = useRef<string | null>(null);
+ const hoverCapturedForRef = useRef<string | null>(null);
+ const hoverTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+ useEffect(() => {
+ if (visible && projectId && shownCapturedForRef.current !== projectId) {
+ shownCapturedForRef.current = projectId;
+ // A new exposure re-arms the pair together: drop any dwell still
+ // pending from the previous project (the switch can happen without a
+ // mouseleave) and clear the hover marker, so a project the user
+ // returns to in place gets a `hovered` chance for each `shown` it
+ // fires — otherwise numerator and denominator drift apart.
+ hoverCapturedForRef.current = null;
+ if (hoverTimerRef.current) {
+ clearTimeout(hoverTimerRef.current);
+ hoverTimerRef.current = null;
+ }
+ capture("v4_migration:delay_badge_shown", { page });
+ }
+ }, [visible, projectId, page, capture]);
+ // A pending dwell timer must not outlive the badge: React removes the
+ // button without a mouseleave when `visible` flips false (SDK check turns
+ // transient, project switch), so clear on visibility changes, not only on
+ // unmount.
+ useEffect(() => {
+ return () => {
+ if (hoverTimerRef.current) {
+ clearTimeout(hoverTimerRef.current);
+ hoverTimerRef.current = null;
+ }
+ };
+ }, [visible]);
+
+ if (!visible || !project) {
return null;
}
+ const handleHoverStart = () => {
+ if (hoverCapturedForRef.current === project.id || hoverTimerRef.current) {
+ return;
+ }
+ hoverTimerRef.current = setTimeout(() => {
+ hoverTimerRef.current = null;
+ hoverCapturedForRef.current = project.id;
+ capture("v4_migration:delay_badge_hovered", { page });
+ }, HOVER_DWELL_MS);
+ };
+
+ const handleHoverEnd = () => {
+ if (hoverTimerRef.current) {
+ clearTimeout(hoverTimerRef.current);
+ hoverTimerRef.current = null;
+ }
+ };
+
const handleClick = () => {
+ // A click consumes this exposure's `hovered` budget: the event counts
+ // noticed but NOT clicked, so cancel the pending dwell (touch taps
+ // synthesize a mouseenter with no mouseleave, so an uncancelled timer
+ // would count every tap as a hover) and mark the project as spent — a
+ // re-hover after the click must not file the user under "never clicked".
+ handleHoverEnd();
+ hoverCapturedForRef.current = project.id;
if (forceV3) {
- capture("v4_migration:delay_badge_clicked", { action: "docs" });
+ capture("v4_migration:delay_badge_clicked", { action: "docs", page });
window.open(PARTNER_INTEGRATION_FAQ_URL, "_blank", "noopener,noreferrer");
return;
}
- capture("v4_migration:delay_badge_clicked");
+ capture("v4_migration:delay_badge_clicked", { page });
openMigrationPanel({ id: project.id, name: project.name }, "delay_badge");
};
@@ -84,6 +163,8 @@ export function V4MigrationDelayBadge() {
return (
<V4MigrationBadgeContent
onClick={handleClick}
+ onHoverStart={handleHoverStart}
+ onHoverEnd={handleHoverEnd}
title="New data in ~15 min"
description={forceV3 ? "Learn more in the docs" : description}
/>
diff --git a/web/src/pages/project/[projectId]/experiments/index.tsx b/web/src/pages/project/[projectId]/experiments/index.tsx
index 42fff771ba..43532f2a52 100644
--- a/web/src/pages/project/[projectId]/experiments/index.tsx
+++ b/web/src/pages/project/[projectId]/experiments/index.tsx
@@ -60,7 +60,7 @@ export default function Experiments() {
<Page
headerProps={{
title: "Experiments",
- titleBadges: <V4MigrationDelayBadge />,
+ titleBadges: <V4MigrationDelayBadge page="experiments" />,
actionButtonsRight: (
<div className="flex items-center gap-2">
<Dialog
diff --git a/web/src/pages/project/[projectId]/observations/index.tsx b/web/src/pages/project/[projectId]/observations/index.tsx
index 672548f4a6..c9ac0ecff1 100644
--- a/web/src/pages/project/[projectId]/observations/index.tsx
+++ b/web/src/pages/project/[projectId]/observations/index.tsx
@@ -44,7 +44,9 @@ export default function Generations() {
title: "Tracing",
// Match traces/index.tsx: no delay badge while onboarding tells the
// user to set up tracing for the first time.
- titleBadges: showOnboarding ? undefined : <V4MigrationDelayBadge />,
+ titleBadges: showOnboarding ? undefined : (
+ <V4MigrationDelayBadge page="observations" />
+ ),
help: {
description:
"An observation captures a single function call in an application. See docs to learn more.",
diff --git a/web/src/pages/project/[projectId]/traces/index.tsx b/web/src/pages/project/[projectId]/traces/index.tsx
index deddec8f00..be7320b7fc 100644
--- a/web/src/pages/project/[projectId]/traces/index.tsx
+++ b/web/src/pages/project/[projectId]/traces/index.tsx
@@ -60,7 +60,7 @@ export default function Traces() {
<Page
headerProps={{
title: "Tracing",
- titleBadges: <V4MigrationDelayBadge />,
+ titleBadges: <V4MigrationDelayBadge page="traces" />,
help: {
description: (
<>