π OpenHands: AI-Driven Development
feat: instrument PostHog analytics for the onboarding funnel (#1547)
Hiep Leb6727865b711
diff --git a/__tests__/components/onboarding/onboarding-modal.test.tsx b/__tests__/components/onboarding/onboarding-modal.test.tsxindex 8550abf507..ed79313bdb 100644--- a/__tests__/components/onboarding/onboarding-modal.test.tsx+++ b/__tests__/components/onboarding/onboarding-modal.test.tsx@@ -20,6 +20,13 @@ import { DEFAULT_SETTINGS } from "#/services/settings";const llmSettingsScreenMock = vi.hoisted(() => vi.fn());const getServerInfoMock = vi.hoisted(() => vi.fn());+const captureMock = vi.hoisted(() => vi.fn());++// useTracking depends on PostHog's `usePostHog`. Mock that underlying service+// (not useTracking itself) so the onboarding analytics events can be asserted.+vi.mock("posthog-js/react", () => ({+ usePostHog: () => ({ capture: captureMock }),+}));// Both the backend status badge in the embedded edit form and the// step-1 health probe ride on `useBackendsHealth`, which resolves@@ -194,6 +201,7 @@ function renderModal(onClose = vi.fn()) {beforeEach(() => {window.localStorage.clear();+ window.sessionStorage.clear();vi.stubEnv("VITE_BACKEND_BASE_URL", "http://localhost:9000");vi.stubEnv("VITE_SESSION_API_KEY", "session-key");__resetActiveStoreForTests();@@ -228,6 +236,7 @@ beforeEach(() => {});afterEach(() => {window.localStorage.clear();+ window.sessionStorage.clear();vi.unstubAllEnvs();__resetActiveStoreForTests();});@@ -974,4 +983,151 @@ describe("OnboardingModal", () => {);expect(onClose).toHaveBeenCalledTimes(1);});++ describe("analytics events", () => {+ // Surface only the captures for a given event; a session emits several.+ const eventCalls = (event: string) =>+ captureMock.mock.calls.filter(([name]) => name === event);++ beforeEach(() => {+ // Grant analytics consent so events pass useTracking's consent gate;+ // the outer beforeEach seeds settings with consent withheld.+ vi.spyOn(SettingsService, "getSettings").mockResolvedValue({+ ...DEFAULT_SETTINGS,+ agent_settings: { ...DEFAULT_SETTINGS.agent_settings, llm: {} },+ user_consents_to_analytics: true,+ });+ });++ it("captures onboarding_started once per session even across a remount", async () => {+ // Arrange + act: open onboarding and let the first mount settle (a Step+ // Viewed capture confirms consent resolved and backend health settled).+ const firstMount = renderModal();+ await waitFor(() =>+ expect(+ eventCalls("onboarding_step_viewed").length,+ ).toBeGreaterThanOrEqual(1),+ );++ // Act: tear the modal down and mount it again in the same session. The+ // remounted modal re-emits Step Viewed from a fresh ref, so waiting for+ // the second one proves the Started effect had its chance to re-fire.+ firstMount.unmount();+ renderModal();+ await waitFor(() =>+ expect(+ eventCalls("onboarding_step_viewed").length,+ ).toBeGreaterThanOrEqual(2),+ );++ // Assert: the sessionStorage guard kept Started at a single capture.+ expect(eventCalls("onboarding_started")).toHaveLength(1);+ });++ it("captures no onboarding events while analytics consent is not granted", async () => {+ // Arrange: withhold consent.+ vi.spyOn(SettingsService, "getSettings").mockResolvedValue({+ ...DEFAULT_SETTINGS,+ agent_settings: { ...DEFAULT_SETTINGS.agent_settings, llm: {} },+ user_consents_to_analytics: false,+ });++ // Act: open onboarding and let it settle on the first step.+ renderModal();+ await waitForConfiguredBackendToBeSkipped();++ // Assert: nothing reached PostHog.+ expect(captureMock).not.toHaveBeenCalled();+ });++ it("captures onboarding_step_viewed for each step as the user advances", async () => {+ // Arrange: the healthy backend is skipped, so the user lands on the+ // agent step (index 0 of the 3-step flow).+ renderModal();+ const user = userEvent.setup();+ await waitForConfiguredBackendToBeSkipped();++ // Assert: the entry step carries the full funnel identity.+ await waitFor(() =>+ expect(captureMock).toHaveBeenCalledWith(+ "onboarding_step_viewed",+ expect.objectContaining({+ step: "agent",+ step_index: 0,+ total_steps: 3,+ agent: "openhands",+ }),+ ),+ );++ // Act: advance to the next step.+ await completeAgentStep(user);++ // Assert: the new step is captured too β tracking follows transitions,+ // not just the initial mount.+ await waitFor(() =>+ expect(captureMock).toHaveBeenCalledWith(+ "onboarding_step_viewed",+ expect.objectContaining({+ step: "setup",+ step_index: 1,+ total_steps: 3,+ }),+ ),+ );+ });++ it("captures onboarding_completed when a conversation is launched from the final step", async () => {+ // Arrange: walk through to the final Say Hello step.+ renderModal();+ const user = userEvent.setup();+ await waitForConfiguredBackendToBeSkipped();+ await completeAgentStep(user);+ await user.click(screen.getByTestId("onboarding-llm-next"));+ await waitFor(() =>+ expect(screen.getByTestId("onboarding-slide-2")).toHaveAttribute(+ "data-active",+ "true",+ ),+ );++ // Act: launch from the final step. The stubbed recommended-automation+ // launcher invokes the same onLaunched completion callback as Say Hello.+ const recommendations = screen.getByTestId(+ "onboarding-recommended-automations",+ );+ await user.click(+ within(recommendations).getByRole("button", {+ name: "launch recommended automation",+ }),+ );++ // Assert.+ expect(captureMock).toHaveBeenCalledWith(+ "onboarding_completed",+ expect.objectContaining({ agent: "openhands" }),+ );+ });++ it("captures onboarding_skipped with the current step when the user skips", async () => {+ // Arrange: healthy backend skipped β user is on the agent step (0 of 3).+ renderModal();+ const user = userEvent.setup();+ await waitForConfiguredBackendToBeSkipped();++ // Act: skip out of onboarding.+ await user.click(screen.getByTestId("onboarding-skip"));++ // Assert.+ expect(captureMock).toHaveBeenCalledWith(+ "onboarding_skipped",+ expect.objectContaining({+ step: "agent",+ step_index: 0,+ total_steps: 3,+ agent: "openhands",+ }),+ );+ });+ });});diff --git a/src/components/features/onboarding/onboarding-modal.tsx b/src/components/features/onboarding/onboarding-modal.tsxindex 92db30982f..76e303b525 100644--- a/src/components/features/onboarding/onboarding-modal.tsx+++ b/src/components/features/onboarding/onboarding-modal.tsx@@ -11,6 +11,8 @@ import { I18nKey } from "#/i18n/declaration";import { cn } from "#/utils/utils";import { useActiveBackendContext } from "#/contexts/active-backend-context";import { useBackendsHealth } from "#/hooks/query/use-backends-health";+import { useSettings } from "#/hooks/query/use-settings";+import { useTracking } from "#/hooks/use-tracking";import { OnboardingProgressBar } from "./onboarding-progress-bar";import {ChooseAgentStep,@@ -45,6 +47,14 @@ const PHASE_ORDER_WITHOUT_BACKEND: readonly OnboardingPhase[] = ["hello",];+/**+ * sessionStorage flag marking that the one-time "onboarding started" analytics+ * event has already been captured for this browser session. It survives+ * rerenders, remounts and the lazy/Suspense flip so the event fires at most+ * once per onboarding session.+ */+const ONBOARDING_STARTED_TRACKED_KEY = "openhands-onboarding-started";+interface SlideProps {/** Index of this slide in the step sequence. */index: number;@@ -124,6 +134,14 @@ export function OnboardingModal({isPreview = false,}: OnboardingModalProps) {const { t } = useTranslation("openhands");+ const { data: settings } = useSettings();+ const analyticsEnabled = settings?.user_consents_to_analytics === true;+ const {+ trackOnboardingStarted,+ trackOnboardingStepViewed,+ trackOnboardingCompleted,+ trackOnboardingSkipped,+ } = useTracking();const { active } = useActiveBackendContext();const { backend } = active;const noBackendSelected = isNoBackend(backend);@@ -173,6 +191,15 @@ export function OnboardingModal({const currentPhase = slideOrder.includes(phase) ? phase : slideOrder[0];const currentStep = slideOrder.indexOf(currentPhase);+ // Backend connectivity is "settled" once we know whether the active backend+ // is reachable (or no backend is selected). Until then `skipBackendStep` may+ // still flip true and renumber the slides, briefly showing the backend slide+ // for an already-healthy backend. Gate Step Viewed on this so we never emit a+ // phantom "backend" view that immediately auto-skips to "agent".+ const backendHealthSettled =+ noBackendSelected ||+ healthByBackendId[backend.id]?.isConnected !== undefined;+const isOpenHands = selectedAgentId === "openhands";const hideSkip = currentStep === 0 && getLockedCloudHost() !== null;const goNext = React.useCallback(() => {@@ -194,6 +221,72 @@ export function OnboardingModal({});}, [slideOrder]);+ // --- Onboarding analytics -------------------------------------------------+ // Every event is routed through `useTracking`, which drops captures unless+ // the user has consented; the design-preview harness (`isPreview`) emits+ // nothing.++ // Onboarding Started β fire at most once per browser session. The+ // sessionStorage flag survives rerenders, remounts and the lazy/Suspense+ // flip; the consent gate means we only arm the guard once a capture can+ // actually happen, so a user who consents partway through onboarding still+ // gets the event instead of having it latched away during the no-consent+ // window.+ const startedTrackedRef = React.useRef(false);+ React.useEffect(() => {+ if (isPreview || !analyticsEnabled || startedTrackedRef.current) return;+ if (window.sessionStorage.getItem(ONBOARDING_STARTED_TRACKED_KEY)) return;+ startedTrackedRef.current = true;+ window.sessionStorage.setItem(ONBOARDING_STARTED_TRACKED_KEY, "1");+ trackOnboardingStarted();+ }, [isPreview, analyticsEnabled]);++ // Onboarding Step Viewed β fire once per phase entry. The phase string is the+ // stable identity (slide indices renumber), so dedupe on it: this absorbs+ // StrictMode's double-invoke and the index renumber, while re-entering a step+ // via Back counts as a genuine new view.+ const lastViewedPhaseRef = React.useRef<OnboardingPhase | null>(null);+ React.useEffect(() => {+ if (isPreview || !analyticsEnabled || !backendHealthSettled) return;+ if (lastViewedPhaseRef.current === currentPhase) return;+ lastViewedPhaseRef.current = currentPhase;+ trackOnboardingStepViewed({+ step: currentPhase,+ stepIndex: currentStep,+ totalSteps,+ agent: selectedAgentId,+ });+ }, [+ isPreview,+ analyticsEnabled,+ backendHealthSettled,+ currentPhase,+ currentStep,+ totalSteps,+ selectedAgentId,+ ]);++ // Onboarding Completed β `onLaunched` runs only after a conversation is+ // successfully created (the hello message or a recommended automation), so+ // wrapping it captures completion exactly once before the modal closes.+ const handleCompleted = () => {+ trackOnboardingCompleted({ agent: selectedAgentId });+ onClose();+ };++ // Onboarding Skipped/Dismissed β the user exited before completing, via the+ // skip button (non-final steps) or the final-step Close button. `currentPhase`+ // records where they left, which also distinguishes the two affordances.+ const handleSkipOrDismiss = () => {+ trackOnboardingSkipped({+ step: currentPhase,+ stepIndex: currentStep,+ totalSteps,+ agent: selectedAgentId,+ });+ onClose();+ };+return (// No `onClose`: the flow must only be dismissed via explicit actions// (the skip button or launching), never by an errant backdrop click or@@ -267,8 +360,8 @@ export function OnboardingModal({><SayHelloSteponBack={goBack}- onClose={onClose}- onLaunched={onClose}+ onClose={handleSkipOrDismiss}+ onLaunched={handleCompleted}/></Slide></div>@@ -279,7 +372,7 @@ export function OnboardingModal({<buttontype="button"data-testid="onboarding-skip"- onClick={onClose}+ onClick={handleSkipOrDismiss}className="rounded-md px-3 py-2 text-sm text-[var(--oh-muted)] transition-colors hover:bg-white/5 hover:text-white cursor-pointer">{t(I18nKey.ONBOARDING$SKIP)}diff --git a/src/hooks/use-tracking.ts b/src/hooks/use-tracking.tsindex 6e3f1de57e..d3699c4e74 100644--- a/src/hooks/use-tracking.ts+++ b/src/hooks/use-tracking.ts@@ -146,6 +146,52 @@ export const useTracking = () => {track("download_trajectory_button_clicked");};+ const trackOnboardingStarted = () => {+ track("onboarding_started");+ };++ const trackOnboardingStepViewed = ({+ step,+ stepIndex,+ totalSteps,+ agent,+ }: {+ step: string;+ stepIndex: number;+ totalSteps: number;+ agent: string;+ }) => {+ track("onboarding_step_viewed", {+ step,+ step_index: stepIndex,+ total_steps: totalSteps,+ agent,+ });+ };++ const trackOnboardingCompleted = ({ agent }: { agent: string }) => {+ track("onboarding_completed", { agent });+ };++ const trackOnboardingSkipped = ({+ step,+ stepIndex,+ totalSteps,+ agent,+ }: {+ step: string;+ stepIndex: number;+ totalSteps: number;+ agent: string;+ }) => {+ track("onboarding_skipped", {+ step,+ step_index: stepIndex,+ total_steps: totalSteps,+ agent,+ });+ };+return {trackLoginButtonClick,trackConversationCreated,@@ -160,5 +206,9 @@ export const useTracking = () => {trackSettingsSaved,trackMcpConfigUpdated,trackDownloadTrajectoryButtonClicked,+ trackOnboardingStarted,+ trackOnboardingStepViewed,+ trackOnboardingCompleted,+ trackOnboardingSkipped,};};diff --git a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts b/tests/e2e/mock-llm/utils/mock-llm-helpers.tsindex d9183f2e14..9b6b27f8be 100644--- a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts+++ b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts@@ -560,8 +560,12 @@ export async function activateProfileViaUI(page: Page, profileName: string) {return (await target.getByTestId("profile-active-badge").count()) > 0;},{+ // Each poll iteration reloads /settings/llm and re-fetches settings,+ // so a slow backend (activation PATCH + settings propagation) needs+ // headroom beyond a couple of reloads. 30s keeps this robust under+ // added proxy latency without masking a genuine activation failure.message: `Profile "${profileName}" should have an "Active" badge`,- timeout: 15_000,+ timeout: 30_000,intervals: [1_000, 2_000, 3_000],},)diff --git a/tests/e2e/support/onboarding-helpers.ts b/tests/e2e/support/onboarding-helpers.tsindex d35949c343..7a6be8604b 100644--- a/tests/e2e/support/onboarding-helpers.ts+++ b/tests/e2e/support/onboarding-helpers.ts@@ -134,6 +134,22 @@ export async function getOnboardingStepLayout(page: Page,): Promise<OnboardingStepLayout> {await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);++ // Wait for the backend health probe to settle before sampling the layout.+ // While the probe is in flight the backend slide shows a transient+ // "checking" banner at step 0. A healthy configured backend then auto-skips+ // the backend slide (starting the flow on agent selection), while an+ // unhealthy/unconfigured one keeps it. Sampling during the "checking" window+ // races that skip: we would wrongly report a backend step that immediately+ // disappears and then wait forever for a "connected" banner that never+ // renders. Waiting for "checking" to clear makes the verdict deterministic+ // regardless of how long the probe takes (e.g. when a spec slows+ // `/api/settings` via a `route.fetch()` intercept).+ await expect(+ page.getByTestId("onboarding-backend-checking"),+ "onboarding backend health probe should settle before sampling the layout",+ ).toHaveCount(0, { timeout: 15_000 });+const hasBackendStep = await page.getByTestId("onboarding-step-check-backend").isVisible()