🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(v4-migration): track delay badge impressions and hovers (#16260)
Trang Nguyen120a93f1039a
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.tsindex b10bf30029..7d65d42578 100644--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts@@ -325,6 +325,11 @@ export const events = {v4_migration: ["coding_agent_prompt_copied","delay_badge_clicked",+ // Discoverability pair for the table delay badge: `shown` is the+ // exposure denominator (badge actually rendered), `hovered` counts+ // noticed-but-not-clicked (pill expanded long enough to read).+ "delay_badge_shown",+ "delay_badge_hovered","project_chip_clicked","contact_book_call_clicked","contact_support_clicked",diff --git a/web/src/features/v4-migration/V4MigrationBadgeContent.tsx b/web/src/features/v4-migration/V4MigrationBadgeContent.tsxindex 11f44a1731..e7aa94c41c 100644--- a/web/src/features/v4-migration/V4MigrationBadgeContent.tsx+++ b/web/src/features/v4-migration/V4MigrationBadgeContent.tsx@@ -27,6 +27,11 @@ export function V4MigrationStatusDot({type V4MigrationBadgeContentProps = {onClick: () => void;+ // Discoverability callbacks: the pill expands its description on hover AND+ // on keyboard focus (group-focus-visible), so either counts as "noticed+ // the badge". Wired to mouseenter/mouseleave and focus/blur alike.+ onHoverStart?: () => void;+ onHoverEnd?: () => void;title: string;description?: string;showChevron?: boolean;@@ -35,6 +40,8 @@ type V4MigrationBadgeContentProps = {export function V4MigrationBadgeContent({onClick,+ onHoverStart,+ onHoverEnd,title,description,showChevron = true,@@ -45,6 +52,10 @@ export function V4MigrationBadgeContent({<buttontype="button"onClick={onClick}+ onMouseEnter={onHoverStart}+ onMouseLeave={onHoverEnd}+ onFocus={onHoverStart}+ onBlur={onHoverEnd}className="hover:bg-muted/50 hover:text-foreground inline-flex w-fit shrink-0 items-center gap-1.5 rounded-full border px-2 py-0.5 text-xs font-bold whitespace-nowrap"><span@@ -75,6 +86,10 @@ export function V4MigrationBadgeContent({<buttontype="button"onClick={onClick}+ onMouseEnter={onHoverStart}+ onMouseLeave={onHoverEnd}+ onFocus={onHoverStart}+ onBlur={onHoverEnd}className="group ring-input hover:bg-muted/50 hover:text-foreground col-start-1 row-start-1 inline-flex w-fit flex-none shrink-0 items-center gap-1.5 justify-self-start rounded-full bg-transparent px-2 py-0.5 text-xs font-bold whitespace-nowrap ring"><V4MigrationStatusDot variant="action" />@@ -83,6 +98,9 @@ export function V4MigrationBadgeContent({{description ? (// 0fr -> 1fr animates to the intrinsic text width; a max-width cap// would clip any description longer than the magic number (LF-90).+ // The 300ms duration is load-bearing for analytics: HOVER_DWELL_MS+ // in V4MigrationDelayBadge assumes the description finishes+ // expanding before the dwell elapses. Keep dwell > duration.<span className="grid grid-cols-[0fr] transition-[grid-template-columns] duration-300 ease-out group-hover:grid-cols-[1fr] group-focus-visible:grid-cols-[1fr]"><span className="min-w-0 overflow-hidden"><span className="whitespace-nowrap">. {description}.</span>diff --git a/web/src/features/v4-migration/V4MigrationDelayBadge.clienttest.tsx b/web/src/features/v4-migration/V4MigrationDelayBadge.clienttest.tsxindex 45ed5ce7fb..a1b42a26fc 100644--- a/web/src/features/v4-migration/V4MigrationDelayBadge.clienttest.tsx+++ b/web/src/features/v4-migration/V4MigrationDelayBadge.clienttest.tsx@@ -1,5 +1,5 @@import { fireEvent, render, screen } from "@testing-library/react";-import { beforeEach, describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { V4MigrationDelayBadge } from "./V4MigrationDelayBadge";import {@@ -122,7 +122,9 @@ const mocks = vi.hoisted(() => ({v4UpgradeUiEnabled: true,forceV3: false,isBetaEnabled: false,+ projectId: "project-1",openMigrationPanel: vi.fn(),+ capture: vi.fn(),}));vi.mock("@/src/features/v4-migration/useV4UpgradeUiEnabled", () => ({@@ -139,12 +141,12 @@ vi.mock("@/src/features/events/hooks/useV4Beta", () => ({}));vi.mock("@/src/features/posthog-analytics/usePostHogClientCapture", () => ({- usePostHogClientCapture: () => vi.fn(),+ usePostHogClientCapture: () => mocks.capture,}));vi.mock("@/src/features/projects/hooks", () => ({useQueryProject: () => ({- project: { id: "project-1", name: "Project One" },+ project: { id: mocks.projectId, name: "Project One" },organization: { id: "org-1" },}),}));@@ -198,17 +200,18 @@ describe("V4MigrationDelayBadge", () => {mocks.v4UpgradeUiEnabled = true;mocks.forceV3 = false;mocks.isBetaEnabled = false;+ mocks.projectId = "project-1";setSdk("latest", [makeSdkUsageSeries({})]);});it("stays hidden when every ingestion path is clean", () => {- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();});it("shows SDK copy for outdated SDK traffic", () => {setSdk("legacy", [outdatedSdkSeries()]);- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.getAllByText("New data in ~15 min").length).toBeGreaterThan(0,);@@ -219,7 +222,7 @@ describe("V4MigrationDelayBadge", () => {it("shows OTel copy for delayed OTel exporters", () => {setSdk("otel_header_required", [delayedOtelSeries()]);- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.getAllByText(/Update your OTel instrumentation for real-time data/).length,@@ -228,7 +231,7 @@ describe("V4MigrationDelayBadge", () => {it("shows instrumentation copy for headerless ingestion traffic", () => {setSdk("unknown", [customIngestionSeries()]);- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.getAllByText(/Upgrade your instrumentation for real-time data/).length,@@ -237,7 +240,7 @@ describe("V4MigrationDelayBadge", () => {it("shows the generic copy when several delayed paths fire", () => {setSdk("legacy", [outdatedSdkSeries(), delayedOtelSeries()]);- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.getAllByText(/Upgrade to v4 for real-time data/).length,).toBeGreaterThan(0);@@ -263,20 +266,20 @@ describe("V4MigrationDelayBadge", () => {upgradeRequiredCount: 0,delayedOtelIngestionCount: 0,};- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();});it("stays hidden while the check is still running", () => {setSdk("checking", []);- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();});it("stays hidden without the v4 upgrade UI flag", () => {mocks.v4UpgradeUiEnabled = false;setSdk("legacy", [outdatedSdkSeries()]);- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();});@@ -285,7 +288,7 @@ describe("V4MigrationDelayBadge", () => {mocks.forceV3 = true;mocks.isBetaEnabled = false;setSdk("legacy", [outdatedSdkSeries()]);- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.queryByText("New data in ~15 min")).not.toBeInTheDocument();});@@ -295,7 +298,7 @@ describe("V4MigrationDelayBadge", () => {setSdk("legacy", [outdatedSdkSeries()]);const windowOpen = vi.spyOn(window, "open").mockImplementation(() => null);- render(<V4MigrationDelayBadge />);+ render(<V4MigrationDelayBadge page="traces" />);expect(screen.getAllByText(/Learn more in the docs/).length,).toBeGreaterThan(0);@@ -307,6 +310,181 @@ describe("V4MigrationDelayBadge", () => {"noopener,noreferrer",);expect(mocks.openMigrationPanel).not.toHaveBeenCalled();+ expect(mocks.capture).toHaveBeenCalledWith(+ "v4_migration:delay_badge_clicked",+ { action: "docs", page: "traces" },+ );windowOpen.mockRestore();});++ describe("discoverability events", () => {+ afterEach(() => {+ vi.useRealTimers();+ });++ const hoveredCalls = () =>+ mocks.capture.mock.calls.filter(+ ([name]) => name === "v4_migration:delay_badge_hovered",+ );++ it("fires delay_badge_shown once per mount with the page dimension", () => {+ setSdk("legacy", [outdatedSdkSeries()]);+ const { rerender } = render(<V4MigrationDelayBadge page="traces" />);+ rerender(<V4MigrationDelayBadge page="traces" />);+ expect(+ mocks.capture.mock.calls.filter(+ ([name]) => name === "v4_migration:delay_badge_shown",+ ),+ ).toEqual([["v4_migration:delay_badge_shown", { page: "traces" }]]);+ });++ it("does not fire delay_badge_shown while the badge is hidden", () => {+ setSdk("checking", []);+ render(<V4MigrationDelayBadge page="traces" />);+ expect(mocks.capture).not.toHaveBeenCalled();+ });++ it("fires delay_badge_hovered once after the dwell threshold", () => {+ vi.useFakeTimers();+ setSdk("legacy", [outdatedSdkSeries()]);+ render(<V4MigrationDelayBadge page="observations" />);+ const badge = screen.getByRole("button");++ fireEvent.mouseEnter(badge);+ vi.advanceTimersByTime(500);+ fireEvent.mouseLeave(badge);+ // A later hover on the same mount must not double-count.+ fireEvent.mouseEnter(badge);+ vi.advanceTimersByTime(500);++ expect(hoveredCalls()).toEqual([+ ["v4_migration:delay_badge_hovered", { page: "observations" }],+ ]);+ });++ it("does not count a drive-by hover shorter than the dwell", () => {+ vi.useFakeTimers();+ setSdk("legacy", [outdatedSdkSeries()]);+ render(<V4MigrationDelayBadge page="traces" />);+ const badge = screen.getByRole("button");++ fireEvent.mouseEnter(badge);+ vi.advanceTimersByTime(300);+ fireEvent.mouseLeave(badge);+ vi.advanceTimersByTime(1000);++ expect(hoveredCalls()).toHaveLength(0);+ });++ it("fires delay_badge_shown again when the project changes in place", () => {+ setSdk("legacy", [outdatedSdkSeries()]);+ const { rerender } = render(<V4MigrationDelayBadge page="traces" />);+ // The project switcher navigates within the same route, so the badge is+ // reconciled (not remounted) while the new project's SDK check runs.+ setSdk("checking", []);+ mocks.projectId = "project-2";+ rerender(<V4MigrationDelayBadge page="traces" />);+ setSdk("legacy", [outdatedSdkSeries()]);+ rerender(<V4MigrationDelayBadge page="traces" />);+ expect(+ mocks.capture.mock.calls.filter(+ ([name]) => name === "v4_migration:delay_badge_shown",+ ),+ ).toHaveLength(2);+ });++ it("does not count a hover that ends in a click", () => {+ vi.useFakeTimers();+ setSdk("legacy", [outdatedSdkSeries()]);+ render(<V4MigrationDelayBadge page="traces" />);+ const badge = screen.getByRole("button");++ fireEvent.mouseEnter(badge);+ vi.advanceTimersByTime(300);+ // Also the touch path: a tap synthesizes mouseenter + click and never+ // delivers a mouseleave to cancel the dwell timer.+ fireEvent.click(badge);+ vi.advanceTimersByTime(1000);++ expect(hoveredCalls()).toHaveLength(0);+ });++ it("cancels a pending dwell when the badge hides without unmounting", () => {+ vi.useFakeTimers();+ setSdk("legacy", [outdatedSdkSeries()]);+ const { rerender } = render(<V4MigrationDelayBadge page="traces" />);+ fireEvent.mouseEnter(screen.getByRole("button"));+ vi.advanceTimersByTime(300);+ // React removes the button without a mouseleave when the SDK check+ // turns transient mid-hover.+ setSdk("checking", []);+ rerender(<V4MigrationDelayBadge page="traces" />);+ vi.advanceTimersByTime(1000);++ expect(hoveredCalls()).toHaveLength(0);+ });++ it("does not count a re-hover after the badge was clicked", () => {+ vi.useFakeTimers();+ setSdk("legacy", [outdatedSdkSeries()]);+ render(<V4MigrationDelayBadge page="traces" />);+ const badge = screen.getByRole("button");++ // `hovered` counts noticed-but-NOT-clicked: once the user clicked,+ // later dwells on the same exposure must not file them under it.+ fireEvent.click(badge);+ fireEvent.mouseEnter(badge);+ vi.advanceTimersByTime(1000);++ expect(hoveredCalls()).toHaveLength(0);+ });++ it("re-arms hovered when the user returns to a project in place", () => {+ vi.useFakeTimers();+ setSdk("legacy", [outdatedSdkSeries()]);+ const { rerender } = render(<V4MigrationDelayBadge page="traces" />);+ const dwell = () => {+ fireEvent.mouseEnter(screen.getByRole("button"));+ vi.advanceTimersByTime(500);+ fireEvent.mouseLeave(screen.getByRole("button"));+ };++ dwell();+ // Switch away and back without unmounting (the project switcher stays+ // on the route). The return re-fires `shown` as a new exposure, so a+ // fresh dwell must count again — otherwise the pair drifts apart.+ mocks.projectId = "project-2";+ rerender(<V4MigrationDelayBadge page="traces" />);+ mocks.projectId = "project-1";+ rerender(<V4MigrationDelayBadge page="traces" />);+ dwell();++ expect(hoveredCalls()).toHaveLength(2);+ });++ it("counts a keyboard focus dwell as hovered", () => {+ vi.useFakeTimers();+ setSdk("legacy", [outdatedSdkSeries()]);+ render(<V4MigrationDelayBadge page="traces" />);++ // The description also expands via group-focus-visible, so a keyboard+ // reveal counts as noticing the badge.+ fireEvent.focus(screen.getByRole("button"));+ vi.advanceTimersByTime(500);++ expect(hoveredCalls()).toEqual([+ ["v4_migration:delay_badge_hovered", { page: "traces" }],+ ]);+ });++ it("carries the page dimension on delay_badge_clicked", () => {+ setSdk("legacy", [outdatedSdkSeries()]);+ render(<V4MigrationDelayBadge page="observations" />);+ fireEvent.click(screen.getAllByText("New data in ~15 min")[0]);+ expect(mocks.capture).toHaveBeenCalledWith(+ "v4_migration:delay_badge_clicked",+ { page: "observations" },+ );+ });+ });});diff --git a/web/src/features/v4-migration/V4MigrationDelayBadge.tsx b/web/src/features/v4-migration/V4MigrationDelayBadge.tsxindex 0925741c93..1bfc4e2990 100644--- a/web/src/features/v4-migration/V4MigrationDelayBadge.tsx+++ b/web/src/features/v4-migration/V4MigrationDelayBadge.tsx@@ -1,4 +1,4 @@-import { useState } from "react";+import { useEffect, useRef, useState } from "react";import { useRouter } from "next/router";import {useV4UpgradeUiEnabled,@@ -23,7 +23,17 @@ import { EvaluatorMigrationDialog } from "@/src/features/v4-migration/EvaluatorMimport { buildDeprecatedEvaluatorsUrl } from "@/src/features/v4-migration/evaluatorMigrationUrls";import { useV4Beta } from "@/src/features/events/hooks/useV4Beta";-export function V4MigrationDelayBadge() {+// The pill's description finishes expanding after 300ms (V4MigrationBadgeContent),+// so a 500ms dwell means the full text was on screen — a drive-by mouse pass+// does not count as "noticed".+const HOVER_DWELL_MS = 500;++export function V4MigrationDelayBadge({+ page,+}: {+ // Which table hosts the badge — clicks/hovers/impressions segment by it.+ page: "traces" | "observations" | "experiments";+}) {const v4UpgradeUiFlagEnabled = useV4UpgradeUiFlag();const openMigrationPanel = useOpenV4MigrationPanel();const { project } = useQueryProject();@@ -56,17 +66,86 @@ export function V4MigrationDelayBadge() {customActionable,].filter(Boolean).length;- if (!enabled || !project || actionablePaths === 0) {+ const visible = enabled && Boolean(project) && actionablePaths > 0;+ const projectId = project?.id;++ // Impression + hover exist to answer one question: do users notice the+ // badge? `shown` is the exposure denominator, `hovered` the+ // noticed-but-not-clicked middle. Both fire at most once per exposure,+ // where an exposure ends when the hosting project changes: the project+ // switcher navigates within the same route, so the component survives the+ // switch, and a plain once-per-mount boolean would swallow the next+ // project's exposure (clicks with no matching shown). The effect+ // synchronizes visibility with PostHog (it can flip to true only after the+ // SDK check resolves).+ const shownCapturedForRef = useRef<string | null>(null);+ const hoverCapturedForRef = useRef<string | null>(null);+ const hoverTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);+ useEffect(() => {+ if (visible && projectId && shownCapturedForRef.current !== projectId) {+ shownCapturedForRef.current = projectId;+ // A new exposure re-arms the pair together: drop any dwell still+ // pending from the previous project (the switch can happen without a+ // mouseleave) and clear the hover marker, so a project the user+ // returns to in place gets a `hovered` chance for each `shown` it+ // fires — otherwise numerator and denominator drift apart.+ hoverCapturedForRef.current = null;+ if (hoverTimerRef.current) {+ clearTimeout(hoverTimerRef.current);+ hoverTimerRef.current = null;+ }+ capture("v4_migration:delay_badge_shown", { page });+ }+ }, [visible, projectId, page, capture]);+ // A pending dwell timer must not outlive the badge: React removes the+ // button without a mouseleave when `visible` flips false (SDK check turns+ // transient, project switch), so clear on visibility changes, not only on+ // unmount.+ useEffect(() => {+ return () => {+ if (hoverTimerRef.current) {+ clearTimeout(hoverTimerRef.current);+ hoverTimerRef.current = null;+ }+ };+ }, [visible]);++ if (!visible || !project) {return null;}+ const handleHoverStart = () => {+ if (hoverCapturedForRef.current === project.id || hoverTimerRef.current) {+ return;+ }+ hoverTimerRef.current = setTimeout(() => {+ hoverTimerRef.current = null;+ hoverCapturedForRef.current = project.id;+ capture("v4_migration:delay_badge_hovered", { page });+ }, HOVER_DWELL_MS);+ };++ const handleHoverEnd = () => {+ if (hoverTimerRef.current) {+ clearTimeout(hoverTimerRef.current);+ hoverTimerRef.current = null;+ }+ };+const handleClick = () => {+ // A click consumes this exposure's `hovered` budget: the event counts+ // noticed but NOT clicked, so cancel the pending dwell (touch taps+ // synthesize a mouseenter with no mouseleave, so an uncancelled timer+ // would count every tap as a hover) and mark the project as spent — a+ // re-hover after the click must not file the user under "never clicked".+ handleHoverEnd();+ hoverCapturedForRef.current = project.id;if (forceV3) {- capture("v4_migration:delay_badge_clicked", { action: "docs" });+ capture("v4_migration:delay_badge_clicked", { action: "docs", page });window.open(PARTNER_INTEGRATION_FAQ_URL, "_blank", "noopener,noreferrer");return;}- capture("v4_migration:delay_badge_clicked");+ capture("v4_migration:delay_badge_clicked", { page });openMigrationPanel({ id: project.id, name: project.name }, "delay_badge");};@@ -84,6 +163,8 @@ export function V4MigrationDelayBadge() {return (<V4MigrationBadgeContentonClick={handleClick}+ onHoverStart={handleHoverStart}+ onHoverEnd={handleHoverEnd}title="New data in ~15 min"description={forceV3 ? "Learn more in the docs" : description}/>diff --git a/web/src/pages/project/[projectId]/experiments/index.tsx b/web/src/pages/project/[projectId]/experiments/index.tsxindex 42fff771ba..43532f2a52 100644--- a/web/src/pages/project/[projectId]/experiments/index.tsx+++ b/web/src/pages/project/[projectId]/experiments/index.tsx@@ -60,7 +60,7 @@ export default function Experiments() {<PageheaderProps={{title: "Experiments",- titleBadges: <V4MigrationDelayBadge />,+ titleBadges: <V4MigrationDelayBadge page="experiments" />,actionButtonsRight: (<div className="flex items-center gap-2"><Dialogdiff --git a/web/src/pages/project/[projectId]/observations/index.tsx b/web/src/pages/project/[projectId]/observations/index.tsxindex 672548f4a6..c9ac0ecff1 100644--- a/web/src/pages/project/[projectId]/observations/index.tsx+++ b/web/src/pages/project/[projectId]/observations/index.tsx@@ -44,7 +44,9 @@ export default function Generations() {title: "Tracing",// Match traces/index.tsx: no delay badge while onboarding tells the// user to set up tracing for the first time.- titleBadges: showOnboarding ? undefined : <V4MigrationDelayBadge />,+ titleBadges: showOnboarding ? undefined : (+ <V4MigrationDelayBadge page="observations" />+ ),help: {description:"An observation captures a single function call in an application. See docs to learn more.",diff --git a/web/src/pages/project/[projectId]/traces/index.tsx b/web/src/pages/project/[projectId]/traces/index.tsxindex deddec8f00..be7320b7fc 100644--- a/web/src/pages/project/[projectId]/traces/index.tsx+++ b/web/src/pages/project/[projectId]/traces/index.tsx@@ -60,7 +60,7 @@ export default function Traces() {<PageheaderProps={{title: "Tracing",- titleBadges: <V4MigrationDelayBadge />,+ titleBadges: <V4MigrationDelayBadge page="traces" />,help: {description: (<>