Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat: instrument PostHog analytics for the onboarding funnel (#1547)

Hiep Leb6727865b711
diff --git a/__tests__/components/onboarding/onboarding-modal.test.tsx b/__tests__/components/onboarding/onboarding-modal.test.tsx
index 8550abf507..ed79313bdb 100644
--- a/__tests__/components/onboarding/onboarding-modal.test.tsx
+++ b/__tests__/components/onboarding/onboarding-modal.test.tsx
@@ -20,6 +20,13 @@ import { DEFAULT_SETTINGS } from "#/services/settings";
const llmSettingsScreenMock = vi.hoisted(() => vi.fn());
const getServerInfoMock = vi.hoisted(() => vi.fn());
+const captureMock = vi.hoisted(() => vi.fn());
+
+// useTracking depends on PostHog's `usePostHog`. Mock that underlying service
+// (not useTracking itself) so the onboarding analytics events can be asserted.
+vi.mock("posthog-js/react", () => ({
+ usePostHog: () => ({ capture: captureMock }),
+}));
// Both the backend status badge in the embedded edit form and the
// step-1 health probe ride on `useBackendsHealth`, which resolves
@@ -194,6 +201,7 @@ function renderModal(onClose = vi.fn()) {
beforeEach(() => {
window.localStorage.clear();
+ window.sessionStorage.clear();
vi.stubEnv("VITE_BACKEND_BASE_URL", "http://localhost:9000");
vi.stubEnv("VITE_SESSION_API_KEY", "session-key");
__resetActiveStoreForTests();
@@ -228,6 +236,7 @@ beforeEach(() => {
});
afterEach(() => {
window.localStorage.clear();
+ window.sessionStorage.clear();
vi.unstubAllEnvs();
__resetActiveStoreForTests();
});
@@ -974,4 +983,151 @@ describe("OnboardingModal", () => {
);
expect(onClose).toHaveBeenCalledTimes(1);
});
+
+ describe("analytics events", () => {
+ // Surface only the captures for a given event; a session emits several.
+ const eventCalls = (event: string) =>
+ captureMock.mock.calls.filter(([name]) => name === event);
+
+ beforeEach(() => {
+ // Grant analytics consent so events pass useTracking's consent gate;
+ // the outer beforeEach seeds settings with consent withheld.
+ vi.spyOn(SettingsService, "getSettings").mockResolvedValue({
+ ...DEFAULT_SETTINGS,
+ agent_settings: { ...DEFAULT_SETTINGS.agent_settings, llm: {} },
+ user_consents_to_analytics: true,
+ });
+ });
+
+ it("captures onboarding_started once per session even across a remount", async () => {
+ // Arrange + act: open onboarding and let the first mount settle (a Step
+ // Viewed capture confirms consent resolved and backend health settled).
+ const firstMount = renderModal();
+ await waitFor(() =>
+ expect(
+ eventCalls("onboarding_step_viewed").length,
+ ).toBeGreaterThanOrEqual(1),
+ );
+
+ // Act: tear the modal down and mount it again in the same session. The
+ // remounted modal re-emits Step Viewed from a fresh ref, so waiting for
+ // the second one proves the Started effect had its chance to re-fire.
+ firstMount.unmount();
+ renderModal();
+ await waitFor(() =>
+ expect(
+ eventCalls("onboarding_step_viewed").length,
+ ).toBeGreaterThanOrEqual(2),
+ );
+
+ // Assert: the sessionStorage guard kept Started at a single capture.
+ expect(eventCalls("onboarding_started")).toHaveLength(1);
+ });
+
+ it("captures no onboarding events while analytics consent is not granted", async () => {
+ // Arrange: withhold consent.
+ vi.spyOn(SettingsService, "getSettings").mockResolvedValue({
+ ...DEFAULT_SETTINGS,
+ agent_settings: { ...DEFAULT_SETTINGS.agent_settings, llm: {} },
+ user_consents_to_analytics: false,
+ });
+
+ // Act: open onboarding and let it settle on the first step.
+ renderModal();
+ await waitForConfiguredBackendToBeSkipped();
+
+ // Assert: nothing reached PostHog.
+ expect(captureMock).not.toHaveBeenCalled();
+ });
+
+ it("captures onboarding_step_viewed for each step as the user advances", async () => {
+ // Arrange: the healthy backend is skipped, so the user lands on the
+ // agent step (index 0 of the 3-step flow).
+ renderModal();
+ const user = userEvent.setup();
+ await waitForConfiguredBackendToBeSkipped();
+
+ // Assert: the entry step carries the full funnel identity.
+ await waitFor(() =>
+ expect(captureMock).toHaveBeenCalledWith(
+ "onboarding_step_viewed",
+ expect.objectContaining({
+ step: "agent",
+ step_index: 0,
+ total_steps: 3,
+ agent: "openhands",
+ }),
+ ),
+ );
+
+ // Act: advance to the next step.
+ await completeAgentStep(user);
+
+ // Assert: the new step is captured too β€” tracking follows transitions,
+ // not just the initial mount.
+ await waitFor(() =>
+ expect(captureMock).toHaveBeenCalledWith(
+ "onboarding_step_viewed",
+ expect.objectContaining({
+ step: "setup",
+ step_index: 1,
+ total_steps: 3,
+ }),
+ ),
+ );
+ });
+
+ it("captures onboarding_completed when a conversation is launched from the final step", async () => {
+ // Arrange: walk through to the final Say Hello step.
+ renderModal();
+ const user = userEvent.setup();
+ await waitForConfiguredBackendToBeSkipped();
+ await completeAgentStep(user);
+ await user.click(screen.getByTestId("onboarding-llm-next"));
+ await waitFor(() =>
+ expect(screen.getByTestId("onboarding-slide-2")).toHaveAttribute(
+ "data-active",
+ "true",
+ ),
+ );
+
+ // Act: launch from the final step. The stubbed recommended-automation
+ // launcher invokes the same onLaunched completion callback as Say Hello.
+ const recommendations = screen.getByTestId(
+ "onboarding-recommended-automations",
+ );
+ await user.click(
+ within(recommendations).getByRole("button", {
+ name: "launch recommended automation",
+ }),
+ );
+
+ // Assert.
+ expect(captureMock).toHaveBeenCalledWith(
+ "onboarding_completed",
+ expect.objectContaining({ agent: "openhands" }),
+ );
+ });
+
+ it("captures onboarding_skipped with the current step when the user skips", async () => {
+ // Arrange: healthy backend skipped β†’ user is on the agent step (0 of 3).
+ renderModal();
+ const user = userEvent.setup();
+ await waitForConfiguredBackendToBeSkipped();
+
+ // Act: skip out of onboarding.
+ await user.click(screen.getByTestId("onboarding-skip"));
+
+ // Assert.
+ expect(captureMock).toHaveBeenCalledWith(
+ "onboarding_skipped",
+ expect.objectContaining({
+ step: "agent",
+ step_index: 0,
+ total_steps: 3,
+ agent: "openhands",
+ }),
+ );
+ });
+ });
});
diff --git a/src/components/features/onboarding/onboarding-modal.tsx b/src/components/features/onboarding/onboarding-modal.tsx
index 92db30982f..76e303b525 100644
--- a/src/components/features/onboarding/onboarding-modal.tsx
+++ b/src/components/features/onboarding/onboarding-modal.tsx
@@ -11,6 +11,8 @@ import { I18nKey } from "#/i18n/declaration";
import { cn } from "#/utils/utils";
import { useActiveBackendContext } from "#/contexts/active-backend-context";
import { useBackendsHealth } from "#/hooks/query/use-backends-health";
+import { useSettings } from "#/hooks/query/use-settings";
+import { useTracking } from "#/hooks/use-tracking";
import { OnboardingProgressBar } from "./onboarding-progress-bar";
import {
ChooseAgentStep,
@@ -45,6 +47,14 @@ const PHASE_ORDER_WITHOUT_BACKEND: readonly OnboardingPhase[] = [
"hello",
];
+/**
+ * sessionStorage flag marking that the one-time "onboarding started" analytics
+ * event has already been captured for this browser session. It survives
+ * rerenders, remounts and the lazy/Suspense flip so the event fires at most
+ * once per onboarding session.
+ */
+const ONBOARDING_STARTED_TRACKED_KEY = "openhands-onboarding-started";
+
interface SlideProps {
/** Index of this slide in the step sequence. */
index: number;
@@ -124,6 +134,14 @@ export function OnboardingModal({
isPreview = false,
}: OnboardingModalProps) {
const { t } = useTranslation("openhands");
+ const { data: settings } = useSettings();
+ const analyticsEnabled = settings?.user_consents_to_analytics === true;
+ const {
+ trackOnboardingStarted,
+ trackOnboardingStepViewed,
+ trackOnboardingCompleted,
+ trackOnboardingSkipped,
+ } = useTracking();
const { active } = useActiveBackendContext();
const { backend } = active;
const noBackendSelected = isNoBackend(backend);
@@ -173,6 +191,15 @@ export function OnboardingModal({
const currentPhase = slideOrder.includes(phase) ? phase : slideOrder[0];
const currentStep = slideOrder.indexOf(currentPhase);
+ // Backend connectivity is "settled" once we know whether the active backend
+ // is reachable (or no backend is selected). Until then `skipBackendStep` may
+ // still flip true and renumber the slides, briefly showing the backend slide
+ // for an already-healthy backend. Gate Step Viewed on this so we never emit a
+ // phantom "backend" view that immediately auto-skips to "agent".
+ const backendHealthSettled =
+ noBackendSelected ||
+ healthByBackendId[backend.id]?.isConnected !== undefined;
+
const isOpenHands = selectedAgentId === "openhands";
const hideSkip = currentStep === 0 && getLockedCloudHost() !== null;
const goNext = React.useCallback(() => {
@@ -194,6 +221,72 @@ export function OnboardingModal({
});
}, [slideOrder]);
+ // --- Onboarding analytics -------------------------------------------------
+ // Every event is routed through `useTracking`, which drops captures unless
+ // the user has consented; the design-preview harness (`isPreview`) emits
+ // nothing.
+
+ // Onboarding Started β€” fire at most once per browser session. The
+ // sessionStorage flag survives rerenders, remounts and the lazy/Suspense
+ // flip; the consent gate means we only arm the guard once a capture can
+ // actually happen, so a user who consents partway through onboarding still
+ // gets the event instead of having it latched away during the no-consent
+ // window.
+ const startedTrackedRef = React.useRef(false);
+ React.useEffect(() => {
+ if (isPreview || !analyticsEnabled || startedTrackedRef.current) return;
+ if (window.sessionStorage.getItem(ONBOARDING_STARTED_TRACKED_KEY)) return;
+ startedTrackedRef.current = true;
+ window.sessionStorage.setItem(ONBOARDING_STARTED_TRACKED_KEY, "1");
+ trackOnboardingStarted();
+ }, [isPreview, analyticsEnabled]);
+
+ // Onboarding Step Viewed β€” fire once per phase entry. The phase string is the
+ // stable identity (slide indices renumber), so dedupe on it: this absorbs
+ // StrictMode's double-invoke and the index renumber, while re-entering a step
+ // via Back counts as a genuine new view.
+ const lastViewedPhaseRef = React.useRef<OnboardingPhase | null>(null);
+ React.useEffect(() => {
+ if (isPreview || !analyticsEnabled || !backendHealthSettled) return;
+ if (lastViewedPhaseRef.current === currentPhase) return;
+ lastViewedPhaseRef.current = currentPhase;
+ trackOnboardingStepViewed({
+ step: currentPhase,
+ stepIndex: currentStep,
+ totalSteps,
+ agent: selectedAgentId,
+ });
+ }, [
+ isPreview,
+ analyticsEnabled,
+ backendHealthSettled,
+ currentPhase,
+ currentStep,
+ totalSteps,
+ selectedAgentId,
+ ]);
+
+ // Onboarding Completed β€” `onLaunched` runs only after a conversation is
+ // successfully created (the hello message or a recommended automation), so
+ // wrapping it captures completion exactly once before the modal closes.
+ const handleCompleted = () => {
+ trackOnboardingCompleted({ agent: selectedAgentId });
+ onClose();
+ };
+
+ // Onboarding Skipped/Dismissed β€” the user exited before completing, via the
+ // skip button (non-final steps) or the final-step Close button. `currentPhase`
+ // records where they left, which also distinguishes the two affordances.
+ const handleSkipOrDismiss = () => {
+ trackOnboardingSkipped({
+ step: currentPhase,
+ stepIndex: currentStep,
+ totalSteps,
+ agent: selectedAgentId,
+ });
+ onClose();
+ };
+
return (
// No `onClose`: the flow must only be dismissed via explicit actions
// (the skip button or launching), never by an errant backdrop click or
@@ -267,8 +360,8 @@ export function OnboardingModal({
>
<SayHelloStep
onBack={goBack}
- onClose={onClose}
- onLaunched={onClose}
+ onClose={handleSkipOrDismiss}
+ onLaunched={handleCompleted}
/>
</Slide>
</div>
@@ -279,7 +372,7 @@ export function OnboardingModal({
<button
type="button"
data-testid="onboarding-skip"
- onClick={onClose}
+ onClick={handleSkipOrDismiss}
className="rounded-md px-3 py-2 text-sm text-[var(--oh-muted)] transition-colors hover:bg-white/5 hover:text-white cursor-pointer"
>
{t(I18nKey.ONBOARDING$SKIP)}
diff --git a/src/hooks/use-tracking.ts b/src/hooks/use-tracking.ts
index 6e3f1de57e..d3699c4e74 100644
--- a/src/hooks/use-tracking.ts
+++ b/src/hooks/use-tracking.ts
@@ -146,6 +146,52 @@ export const useTracking = () => {
track("download_trajectory_button_clicked");
};
+ const trackOnboardingStarted = () => {
+ track("onboarding_started");
+ };
+
+ const trackOnboardingStepViewed = ({
+ step,
+ stepIndex,
+ totalSteps,
+ agent,
+ }: {
+ step: string;
+ stepIndex: number;
+ totalSteps: number;
+ agent: string;
+ }) => {
+ track("onboarding_step_viewed", {
+ step,
+ step_index: stepIndex,
+ total_steps: totalSteps,
+ agent,
+ });
+ };
+
+ const trackOnboardingCompleted = ({ agent }: { agent: string }) => {
+ track("onboarding_completed", { agent });
+ };
+
+ const trackOnboardingSkipped = ({
+ step,
+ stepIndex,
+ totalSteps,
+ agent,
+ }: {
+ step: string;
+ stepIndex: number;
+ totalSteps: number;
+ agent: string;
+ }) => {
+ track("onboarding_skipped", {
+ step,
+ step_index: stepIndex,
+ total_steps: totalSteps,
+ agent,
+ });
+ };
+
return {
trackLoginButtonClick,
trackConversationCreated,
@@ -160,5 +206,9 @@ export const useTracking = () => {
trackSettingsSaved,
trackMcpConfigUpdated,
trackDownloadTrajectoryButtonClicked,
+ trackOnboardingStarted,
+ trackOnboardingStepViewed,
+ trackOnboardingCompleted,
+ trackOnboardingSkipped,
};
};
diff --git a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts
index d9183f2e14..9b6b27f8be 100644
--- a/tests/e2e/mock-llm/utils/mock-llm-helpers.ts
+++ b/tests/e2e/mock-llm/utils/mock-llm-helpers.ts
@@ -560,8 +560,12 @@ export async function activateProfileViaUI(page: Page, profileName: string) {
return (await target.getByTestId("profile-active-badge").count()) > 0;
},
{
+ // Each poll iteration reloads /settings/llm and re-fetches settings,
+ // so a slow backend (activation PATCH + settings propagation) needs
+ // headroom beyond a couple of reloads. 30s keeps this robust under
+ // added proxy latency without masking a genuine activation failure.
message: `Profile "${profileName}" should have an "Active" badge`,
- timeout: 15_000,
+ timeout: 30_000,
intervals: [1_000, 2_000, 3_000],
},
)
diff --git a/tests/e2e/support/onboarding-helpers.ts b/tests/e2e/support/onboarding-helpers.ts
index d35949c343..7a6be8604b 100644
--- a/tests/e2e/support/onboarding-helpers.ts
+++ b/tests/e2e/support/onboarding-helpers.ts
@@ -134,6 +134,22 @@ export async function getOnboardingStepLayout(
page: Page,
): Promise<OnboardingStepLayout> {
await waitForOnboardingStep(page, ONBOARDING_BACKEND_STEP);
+
+ // Wait for the backend health probe to settle before sampling the layout.
+ // While the probe is in flight the backend slide shows a transient
+ // "checking" banner at step 0. A healthy configured backend then auto-skips
+ // the backend slide (starting the flow on agent selection), while an
+ // unhealthy/unconfigured one keeps it. Sampling during the "checking" window
+ // races that skip: we would wrongly report a backend step that immediately
+ // disappears and then wait forever for a "connected" banner that never
+ // renders. Waiting for "checking" to clear makes the verdict deterministic
+ // regardless of how long the probe takes (e.g. when a spec slows
+ // `/api/settings` via a `route.fetch()` intercept).
+ await expect(
+ page.getByTestId("onboarding-backend-checking"),
+ "onboarding backend health probe should settle before sampling the layout",
+ ).toHaveCount(0, { timeout: 15_000 });
+
const hasBackendStep = await page
.getByTestId("onboarding-step-check-backend")
.isVisible()