Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(platform): record A/B experiment arms per user (#14287)

Nicholas Tindle2ea51e34971c
diff --git a/autogpt_platform/backend/backend/api/features/experiments.py b/autogpt_platform/backend/backend/api/features/experiments.py
new file mode 100644
index 0000000000..2d1145b82f
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/experiments.py
@@ -0,0 +1,54 @@
+"""Experiment assignment API.
+
+The frontend evaluates experiment flags through PostHog and reports the arm
+it showed the user here, once per experiment, so the assignment is available
+to server-side analytics and the SQL views.
+"""
+
+import logging
+from typing import Annotated
+
+import fastapi
+import pydantic
+from autogpt_libs.auth import get_user_id
+from autogpt_libs.auth.dependencies import requires_user
+
+from backend.data import experiments as experiments_db
+from backend.data.experiments import AssignmentSource, ExperimentAssignment
+
+router = fastapi.APIRouter(dependencies=[fastapi.Security(requires_user)])
+logger = logging.getLogger(__name__)
+
+
+class RecordAssignmentRequest(pydantic.BaseModel):
+ experiment_key: str = pydantic.Field(..., min_length=1, max_length=128)
+ variant: str = pydantic.Field(..., min_length=1, max_length=128)
+ source: AssignmentSource = "posthog"
+
+
+ "/assignments",
+ summary="List experiment assignments",
+ response_model=list[ExperimentAssignment],
+)
+async def list_experiment_assignments(
+ user_id: Annotated[str, fastapi.Security(get_user_id)],
+) -> list[ExperimentAssignment]:
+ return await experiments_db.list_assignments(user_id)
+
+
+ "/assignments",
+ summary="Record experiment assignment",
+ response_model=ExperimentAssignment,
+)
+async def record_experiment_assignment(
+ user_id: Annotated[str, fastapi.Security(get_user_id)],
+ request: RecordAssignmentRequest,
+) -> ExperimentAssignment:
+ return await experiments_db.record_assignment(
+ user_id=user_id,
+ experiment_key=request.experiment_key,
+ variant=request.variant,
+ source=request.source,
+ )
diff --git a/autogpt_platform/backend/backend/api/features/experiments_test.py b/autogpt_platform/backend/backend/api/features/experiments_test.py
new file mode 100644
index 0000000000..a32e10c044
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/experiments_test.py
@@ -0,0 +1,90 @@
+"""Tests for the experiment assignment API."""
+
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock
+
+import fastapi
+import fastapi.testclient
+import pytest
+import pytest_mock
+
+from backend.data.experiments import ExperimentAssignment
+
+from .experiments import router as experiments_router
+
+app = fastapi.FastAPI()
+app.include_router(experiments_router)
+
+client = fastapi.testclient.TestClient(app)
+
+
[email protected](autouse=True)
+def setup_app_auth(mock_jwt_user):
+ from autogpt_libs.auth.jwt_utils import get_jwt_payload
+
+ app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]
+ yield
+ app.dependency_overrides.clear()
+
+
+def _assignment(variant: str = "yearly-pro") -> ExperimentAssignment:
+ return ExperimentAssignment(
+ experiment_key="subscription-pricing-page-initial-state",
+ variant=variant,
+ source="posthog",
+ assigned_at=datetime(2026, 9, 2, tzinfo=UTC),
+ )
+
+
+def test_record_assignment(mocker: pytest_mock.MockFixture, test_user_id: str) -> None:
+ record = mocker.patch(
+ "backend.api.features.experiments.experiments_db.record_assignment",
+ new_callable=AsyncMock,
+ return_value=_assignment(),
+ )
+
+ response = client.post(
+ "/assignments",
+ json={
+ "experiment_key": "subscription-pricing-page-initial-state",
+ "variant": "yearly-pro",
+ },
+ )
+
+ assert response.status_code == 200, response.text
+ body = response.json()
+ assert body["variant"] == "yearly-pro"
+ assert body["experiment_key"] == "subscription-pricing-page-initial-state"
+ record.assert_awaited_once_with(
+ user_id=test_user_id,
+ experiment_key="subscription-pricing-page-initial-state",
+ variant="yearly-pro",
+ source="posthog",
+ )
+
+
+def test_record_assignment_rejects_unknown_source() -> None:
+ response = client.post(
+ "/assignments",
+ json={"experiment_key": "k", "variant": "v", "source": "guess"},
+ )
+ assert response.status_code == 422
+
+
+def test_record_assignment_rejects_empty_variant() -> None:
+ response = client.post("/assignments", json={"experiment_key": "k", "variant": ""})
+ assert response.status_code == 422
+
+
+def test_list_assignments(mocker: pytest_mock.MockFixture, test_user_id: str) -> None:
+ listing = mocker.patch(
+ "backend.api.features.experiments.experiments_db.list_assignments",
+ new_callable=AsyncMock,
+ return_value=[_assignment("control")],
+ )
+
+ response = client.get("/assignments")
+
+ assert response.status_code == 200, response.text
+ assert [row["variant"] for row in response.json()] == ["control"]
+ listing.assert_awaited_once_with(test_user_id)
diff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.py
index 43e2319ea0..aef1d57cd2 100644
--- a/autogpt_platform/backend/backend/api/rest_api.py
+++ b/autogpt_platform/backend/backend/api/rest_api.py
@@ -37,6 +37,7 @@ import backend.api.features.chat.routes as chat_routes
import backend.api.features.chat.share as chat_share
import backend.api.features.chat.speech as chat_speech
import backend.api.features.executions.review.routes
+import backend.api.features.experiments
import backend.api.features.experts.routes as experts_routes
import backend.api.features.home.routes as home_routes
import backend.api.features.library.db
@@ -405,6 +406,11 @@ app.include_router(
prefix="/api/analytics",
tags=["analytics"],
)
+app.include_router(
+ backend.api.features.experiments.router,
+ prefix="/api/experiments",
+ tags=["experiments"],
+)
app.include_router(
backend.api.features.store.routes.router, tags=["v2"], prefix="/api/store"
)
diff --git a/autogpt_platform/backend/backend/data/experiments.py b/autogpt_platform/backend/backend/data/experiments.py
new file mode 100644
index 0000000000..1f37e46346
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/experiments.py
@@ -0,0 +1,96 @@
+"""Experiment arm assignments: which variant a user saw, recorded once.
+
+PostHog owns bucketing and significance testing. This table is the durable,
+joinable copy of the arm so the ``analytics.*`` views can split activation,
+retention and cost by variant without depending on the flag provider's
+history. First write wins: a user's arm is fixed the moment it is observed.
+"""
+
+import logging
+from datetime import datetime
+from typing import Literal
+
+import prisma.models
+import prisma.types
+from prisma.errors import UniqueViolationError
+from pydantic import BaseModel
+
+logger = logging.getLogger(__name__)
+
+AssignmentSource = Literal["posthog", "launchdarkly", "backend"]
+
+
+class ExperimentAssignment(BaseModel):
+ experiment_key: str
+ variant: str
+ source: str
+ assigned_at: datetime
+
+ @classmethod
+ def from_db(cls, row: prisma.models.ExperimentAssignment) -> "ExperimentAssignment":
+ return cls(
+ experiment_key=row.experimentKey,
+ variant=row.variant,
+ source=row.source,
+ assigned_at=row.createdAt,
+ )
+
+
+async def record_assignment(
+ user_id: str,
+ experiment_key: str,
+ variant: str,
+ source: AssignmentSource = "posthog",
+) -> ExperimentAssignment:
+ where: prisma.types.ExperimentAssignmentWhereUniqueInput = {
+ "userId_experimentKey": {"userId": user_id, "experimentKey": experiment_key}
+ }
+ existing = await prisma.models.ExperimentAssignment.prisma().find_unique(
+ where=where
+ )
+ if existing is None:
+ existing = await _create_first_assignment(
+ where, user_id, experiment_key, variant, source
+ )
+ if existing.variant != variant:
+ # The flag provider re-bucketed the user (rollout changed, flag
+ # edited). Keep the arm they actually experienced first so the
+ # analysis stays intention-to-treat.
+ logger.info(
+ f"Experiment {experiment_key}: keeping first-seen variant "
+ f"{existing.variant} for user {user_id} (now {variant})"
+ )
+ return ExperimentAssignment.from_db(existing)
+
+
+async def _create_first_assignment(
+ where: prisma.types.ExperimentAssignmentWhereUniqueInput,
+ user_id: str,
+ experiment_key: str,
+ variant: str,
+ source: AssignmentSource,
+) -> prisma.models.ExperimentAssignment:
+ try:
+ return await prisma.models.ExperimentAssignment.prisma().create(
+ data=prisma.types.ExperimentAssignmentCreateInput(
+ userId=user_id,
+ experimentKey=experiment_key,
+ variant=variant,
+ source=source,
+ )
+ )
+ except UniqueViolationError:
+ # Two tabs reported the same experiment at once; the other write won.
+ existing = await prisma.models.ExperimentAssignment.prisma().find_unique(
+ where=where
+ )
+ if existing is None:
+ raise
+ return existing
+
+
+async def list_assignments(user_id: str) -> list[ExperimentAssignment]:
+ rows = await prisma.models.ExperimentAssignment.prisma().find_many(
+ where={"userId": user_id}, order={"createdAt": "asc"}
+ )
+ return [ExperimentAssignment.from_db(row) for row in rows]
diff --git a/autogpt_platform/backend/migrations/20260902120100_add_experiment_assignment/migration.sql b/autogpt_platform/backend/migrations/20260902120100_add_experiment_assignment/migration.sql
new file mode 100644
index 0000000000..09c858b61a
--- /dev/null
+++ b/autogpt_platform/backend/migrations/20260902120100_add_experiment_assignment/migration.sql
@@ -0,0 +1,20 @@
+-- CreateTable
+CREATE TABLE "ExperimentAssignment" (
+ "id" TEXT NOT NULL,
+ "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "userId" TEXT NOT NULL,
+ "experimentKey" TEXT NOT NULL,
+ "variant" TEXT NOT NULL,
+ "source" TEXT NOT NULL,
+
+ CONSTRAINT "ExperimentAssignment_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ExperimentAssignment_userId_experimentKey_key" ON "ExperimentAssignment"("userId", "experimentKey");
+
+-- CreateIndex
+CREATE INDEX "ExperimentAssignment_experimentKey_variant_idx" ON "ExperimentAssignment"("experimentKey", "variant");
+
+-- AddForeignKey
+ALTER TABLE "ExperimentAssignment" ADD CONSTRAINT "ExperimentAssignment_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prisma
index dcb9bdac6f..dcd77a9242 100644
--- a/autogpt_platform/backend/schema.prisma
+++ b/autogpt_platform/backend/schema.prisma
@@ -189,8 +189,32 @@ model User {
// Platform bot linking
PlatformLinks PlatformLink[]
PlatformUserLinks PlatformUserLink[]
+
+ ExperimentAssignments ExperimentAssignment[]
}
+// One row per (user, experiment): the arm a user was bucketed into the first
+// time the experiment was evaluated for them. Written once and never
+// overwritten, so analytics can join a stable arm to every later event
+// (runs, retention, spend) without depending on the flag provider's history.
+model ExperimentAssignment {
+ id String @id @default(uuid())
+ createdAt DateTime @default(now())
+
+ userId String
+ User User @relation(fields: [userId], references: [id], onDelete: Cascade)
+
+ experimentKey String
+ variant String
+ // "posthog" when the frontend reports a PostHog-evaluated flag,
+ // "backend" when the server bucketed the user itself.
+ source String
+
+ @@unique([userId, experimentKey])
+ @@index([experimentKey, variant])
+}
+
+
enum ChatSessionStatus {
idle
queued
diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/SubscriptionStep/useSubscriptionPricingExperiment.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/SubscriptionStep/useSubscriptionPricingExperiment.ts
index 64a79a093d..287d45168a 100644
--- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/SubscriptionStep/useSubscriptionPricingExperiment.ts
+++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/SubscriptionStep/useSubscriptionPricingExperiment.ts
@@ -1,6 +1,6 @@
"use client";
-import { useFeatureFlagVariantKey } from "@posthog/react";
+import { useExperiment } from "@/services/experiments/useExperiment";
import { useEffect } from "react";
import { useOnboardingWizardStore } from "../../store";
import {
@@ -10,7 +10,7 @@ import {
} from "./helpers";
export function useSubscriptionPricingExperiment() {
- const variant = useFeatureFlagVariantKey(
+ const { variant, isResolved } = useExperiment(
SUBSCRIPTION_PRICING_EXPERIMENT_FLAG,
);
const applyPricingExperimentBilling = useOnboardingWizardStore(
@@ -20,15 +20,20 @@ export function useSubscriptionPricingExperiment() {
const hasUserSelectedBilling = useOnboardingWizardStore(
(s) => s.hasUserSelectedBilling,
);
- const config = getSubscriptionPricingExperimentConfig(variant);
+ const config = getSubscriptionPricingExperimentConfig(variant ?? undefined);
+ // Wait for the arm before touching the store: applying "monthly" for an
+ // unresolved flag and then flipping to the variant's cycle a moment later
+ // both flashes the UI and records the wrong default.
useEffect(() => {
+ if (!isResolved) return;
applyPricingExperimentBilling(config.billing);
- }, [applyPricingExperimentBilling, config.billing]);
+ }, [isResolved, applyPricingExperimentBilling, config.billing]);
return {
billing: hasUserSelectedBilling ? selectedBilling : config.billing,
plans: getSubscriptionPricingExperimentPlans(config.highlightedPlan),
variant: config.variant,
+ isResolved,
};
}
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index d8bd176704..9a3903eefe 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -6313,6 +6313,72 @@
}
}
},
+ "/api/experiments/assignments": {
+ "get": {
+ "tags": ["experiments"],
+ "summary": "List experiment assignments",
+ "operationId": "getExperimentsList experiment assignments",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "items": {
+ "$ref": "#/components/schemas/ExperimentAssignment"
+ },
+ "type": "array",
+ "title": "Response Getexperimentslist Experiment Assignments"
+ }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ }
+ },
+ "security": [{ "HTTPBearerJWT": [] }]
+ },
+ "post": {
+ "tags": ["experiments"],
+ "summary": "Record experiment assignment",
+ "operationId": "postExperimentsRecord experiment assignment",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/RecordAssignmentRequest"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ExperimentAssignment"
+ }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ }
+ },
+ "security": [{ "HTTPBearerJWT": [] }]
+ }
+ },
"/api/experts": {
"get": {
"tags": ["v2", "experts", "experts", "private"],
@@ -20655,6 +20721,21 @@
"title": "ExecutionStartedResponse",
"description": "Response for run/schedule actions."
},
+ "ExperimentAssignment": {
+ "properties": {
+ "experiment_key": { "type": "string", "title": "Experiment Key" },
+ "variant": { "type": "string", "title": "Variant" },
+ "source": { "type": "string", "title": "Source" },
+ "assigned_at": {
+ "type": "string",
+ "format": "date-time",
+ "title": "Assigned At"
+ }
+ },
+ "type": "object",
+ "required": ["experiment_key", "variant", "source", "assigned_at"],
+ "title": "ExperimentAssignment"
+ },
"Expert": {
"properties": {
"id": { "type": "string", "title": "Id" },
@@ -26945,6 +27026,31 @@
"title": "RecommendedProvidersResponse",
"description": "Recommendations for the \"Connect your tools\" panel.\n\n``ready`` is false while the background job is still running — the\nclient keeps polling. Once true, ``providers`` is the final answer,\nempty included (\"nothing worth recommending\" is a real result)."
},
+ "RecordAssignmentRequest": {
+ "properties": {
+ "experiment_key": {
+ "type": "string",
+ "maxLength": 128,
+ "minLength": 1,
+ "title": "Experiment Key"
+ },
+ "variant": {
+ "type": "string",
+ "maxLength": 128,
+ "minLength": 1,
+ "title": "Variant"
+ },
+ "source": {
+ "type": "string",
+ "enum": ["posthog", "launchdarkly", "backend"],
+ "title": "Source",
+ "default": "posthog"
+ }
+ },
+ "type": "object",
+ "required": ["experiment_key", "variant"],
+ "title": "RecordAssignmentRequest"
+ },
"RefundRequest": {
"properties": {
"id": { "type": "string", "title": "Id" },
diff --git a/autogpt_platform/frontend/src/services/experiments/__tests__/useExperiment.test.tsx b/autogpt_platform/frontend/src/services/experiments/__tests__/useExperiment.test.tsx
new file mode 100644
index 0000000000..129509102c
--- /dev/null
+++ b/autogpt_platform/frontend/src/services/experiments/__tests__/useExperiment.test.tsx
@@ -0,0 +1,122 @@
+import { server } from "@/mocks/mock-server";
+import { environment } from "@/services/environment";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { renderHook, waitFor } from "@testing-library/react";
+import { http, HttpResponse } from "msw";
+import { ReactNode } from "react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ resetReportedAssignmentsForTests,
+ useExperiment,
+} from "../useExperiment";
+
+const postHog = vi.hoisted(() => ({
+ variant: undefined as string | boolean | undefined,
+}));
+
+const auth = vi.hoisted(() => ({
+ user: null as { id: string } | null,
+}));
+
+vi.mock("@posthog/react", () => ({
+ useFeatureFlagVariantKey: () => postHog.variant,
+}));
+
+vi.mock("@/lib/auth/hooks/useAuth", () => ({
+ useAuth: () => ({ user: auth.user, isUserLoading: false }),
+}));
+
+interface Props {
+ children: ReactNode;
+}
+
+function wrapper({ children }: Props) {
+ const client = new QueryClient({
+ defaultOptions: { queries: { retry: false }, mutations: { retry: false } },
+ });
+ return <QueryClientProvider client={client}>{children}</QueryClientProvider>;
+}
+
+function captureAssignments() {
+ const bodies: unknown[] = [];
+ server.use(
+ http.post("*/api/experiments/assignments", async ({ request }) => {
+ bodies.push(await request.json());
+ return HttpResponse.json({
+ experiment_key: "pricing-test",
+ variant: "yearly-pro",
+ source: "posthog",
+ assigned_at: "2026-09-02T00:00:00Z",
+ });
+ }),
+ );
+ return bodies;
+}
+
+beforeEach(() => {
+ postHog.variant = undefined;
+ auth.user = { id: "user-1" };
+ resetReportedAssignmentsForTests();
+ vi.spyOn(environment, "isPostHogEnabled").mockReturnValue("phc_test");
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+describe("useExperiment", () => {
+ it("is unresolved while PostHog has not loaded the flag", () => {
+ const { result } = renderHook(() => useExperiment("pricing-test"), {
+ wrapper,
+ });
+
+ expect(result.current.isResolved).toBe(false);
+ expect(result.current.variant).toBeNull();
+ });
+
+ it("resolves immediately when PostHog is disabled", () => {
+ vi.spyOn(environment, "isPostHogEnabled").mockReturnValue(false);
+
+ const { result } = renderHook(() => useExperiment("pricing-test"), {
+ wrapper,
+ });
+
+ expect(result.current.isResolved).toBe(true);
+ expect(result.current.variant).toBeNull();
+ });
+
+ it("reports a string variant to the backend once per user and experiment", async () => {
+ const bodies = captureAssignments();
+ postHog.variant = "yearly-pro";
+
+ const first = renderHook(() => useExperiment("pricing-test"), { wrapper });
+ expect(first.result.current).toEqual({
+ variant: "yearly-pro",
+ isResolved: true,
+ });
+ await waitFor(() => expect(bodies).toHaveLength(1));
+ expect(bodies[0]).toEqual({
+ experiment_key: "pricing-test",
+ variant: "yearly-pro",
+ source: "posthog",
+ });
+
+ renderHook(() => useExperiment("pricing-test"), { wrapper });
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect(bodies).toHaveLength(1);
+ });
+
+ it("does not report users who are not enrolled or not signed in", async () => {
+ const bodies = captureAssignments();
+
+ postHog.variant = false;
+ renderHook(() => useExperiment("pricing-test"), { wrapper });
+
+ postHog.variant = "control";
+ auth.user = null;
+ renderHook(() => useExperiment("pricing-test"), { wrapper });
+
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect(bodies).toHaveLength(0);
+ });
+});
diff --git a/autogpt_platform/frontend/src/services/experiments/__tests__/useLaunchDarklyExperiment.test.tsx b/autogpt_platform/frontend/src/services/experiments/__tests__/useLaunchDarklyExperiment.test.tsx
new file mode 100644
index 0000000000..3cc3ec1b1b
--- /dev/null
+++ b/autogpt_platform/frontend/src/services/experiments/__tests__/useLaunchDarklyExperiment.test.tsx
@@ -0,0 +1,206 @@
+import { server } from "@/mocks/mock-server";
+import { environment } from "@/services/environment";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { renderHook, waitFor } from "@testing-library/react";
+import { http, HttpResponse } from "msw";
+import { ReactNode } from "react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ resetExposuresForTests,
+ useLaunchDarklyExperiment,
+} from "../useLaunchDarklyExperiment";
+import { resetReportedAssignmentsForTests } from "../useReportAssignment";
+
+const ld = vi.hoisted(() => ({
+ flags: {} as Record<string, unknown>,
+ client: null as null | { waitForInitialization: () => Promise<void> },
+}));
+
+const auth = vi.hoisted(() => ({
+ user: null as { id: string } | null,
+}));
+
+const capture = vi.hoisted(() => vi.fn());
+
+vi.mock("launchdarkly-react-client-sdk", () => ({
+ useFlags: () => ld.flags,
+ useLDClient: () => ld.client ?? undefined,
+}));
+
+vi.mock("posthog-js", () => ({ default: { capture } }));
+
+vi.mock("@/lib/auth/hooks/useAuth", () => ({
+ useAuth: () => ({ user: auth.user, isUserLoading: false }),
+}));
+
+interface Props {
+ children: ReactNode;
+}
+
+function wrapper({ children }: Props) {
+ const client = new QueryClient({
+ defaultOptions: { queries: { retry: false }, mutations: { retry: false } },
+ });
+ return <QueryClientProvider client={client}>{children}</QueryClientProvider>;
+}
+
+function captureAssignments() {
+ const bodies: unknown[] = [];
+ server.use(
+ http.post("*/api/experiments/assignments", async ({ request }) => {
+ bodies.push(await request.json());
+ return HttpResponse.json({
+ experiment_key: "onboarding-copy",
+ variant: "b",
+ source: "launchdarkly",
+ assigned_at: "2026-09-02T00:00:00Z",
+ });
+ }),
+ );
+ return bodies;
+}
+
+beforeEach(() => {
+ ld.flags = {};
+ ld.client = { waitForInitialization: () => Promise.resolve() };
+ auth.user = { id: "user-1" };
+ capture.mockReset();
+ resetExposuresForTests();
+ resetReportedAssignmentsForTests();
+ vi.spyOn(environment, "areFeatureFlagsEnabled").mockReturnValue(true);
+ vi.spyOn(environment, "isPostHogEnabled").mockReturnValue("phc_test");
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+describe("useLaunchDarklyExperiment", () => {
+ it("resolves after the LaunchDarkly client initialises", async () => {
+ ld.flags = { "onboarding-copy": "b" };
+
+ const { result } = renderHook(
+ () => useLaunchDarklyExperiment("onboarding-copy"),
+ { wrapper },
+ );
+
+ expect(result.current.isResolved).toBe(false);
+ await waitFor(() => expect(result.current.isResolved).toBe(true));
+ expect(result.current.variant).toBe("b");
+ });
+
+ it("reports the arm to the backend and to PostHog once", async () => {
+ const bodies = captureAssignments();
+ ld.flags = { "onboarding-copy": "b" };
+
+ renderHook(() => useLaunchDarklyExperiment("onboarding-copy"), {
+ wrapper,
+ });
+
+ await waitFor(() => expect(bodies).toHaveLength(1));
+ expect(bodies[0]).toEqual({
+ experiment_key: "onboarding-copy",
+ variant: "b",
+ source: "launchdarkly",
+ });
+ expect(capture).toHaveBeenCalledTimes(1);
+ expect(capture).toHaveBeenCalledWith("experiment_exposed", {
+ experiment_key: "onboarding-copy",
+ variant: "b",
+ provider: "launchdarkly",
+ "$feature/onboarding-copy": "b",
+ });
+
+ renderHook(() => useLaunchDarklyExperiment("onboarding-copy"), {
+ wrapper,
+ });
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect(bodies).toHaveLength(1);
+ expect(capture).toHaveBeenCalledTimes(1);
+ });
+
+ it("reports separate exposures when the signed-in user changes", async () => {
+ const bodies = captureAssignments();
+ ld.flags = { "onboarding-copy": "b" };
+ const { rerender } = renderHook(
+ () => useLaunchDarklyExperiment("onboarding-copy"),
+ { wrapper },
+ );
+ await waitFor(() => expect(capture).toHaveBeenCalledTimes(1));
+
+ auth.user = { id: "user-2" };
+ rerender();
+ await waitFor(() => expect(bodies).toHaveLength(2));
+ expect(capture).toHaveBeenCalledTimes(2);
+
+ auth.user = { id: "user-1" };
+ rerender();
+ expect(capture).toHaveBeenCalledTimes(2);
+ });
+
+ it("waits for a signed-in user before reporting an exposure", async () => {
+ captureAssignments();
+ auth.user = null;
+ ld.flags = { "onboarding-copy": "b" };
+ const { result, rerender } = renderHook(
+ () => useLaunchDarklyExperiment("onboarding-copy"),
+ { wrapper },
+ );
+ await waitFor(() => expect(result.current.isResolved).toBe(true));
+ expect(capture).not.toHaveBeenCalled();
+
+ auth.user = { id: "user-1" };
+ rerender();
+ expect(capture).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not consume an exposure while PostHog is disabled", async () => {
+ captureAssignments();
+ const enabled = vi.spyOn(environment, "isPostHogEnabled");
+ enabled.mockReturnValue(false);
+ ld.flags = { "onboarding-copy": "b" };
+ const { result, rerender } = renderHook(
+ () => useLaunchDarklyExperiment("onboarding-copy"),
+ { wrapper },
+ );
+ await waitFor(() => expect(result.current.isResolved).toBe(true));
+ expect(capture).not.toHaveBeenCalled();
+
+ enabled.mockReturnValue("phc_test");
+ rerender();
+ expect(capture).toHaveBeenCalledTimes(1);
+ });
+
+ it("treats boolean or missing flags as not enrolled", async () => {
+ const bodies = captureAssignments();
+ ld.flags = { "onboarding-copy": true };
+
+ const { result } = renderHook(
+ () => useLaunchDarklyExperiment("onboarding-copy"),
+ { wrapper },
+ );
+
+ await waitFor(() => expect(result.current.isResolved).toBe(true));
+ expect(result.current.variant).toBeNull();
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect(bodies).toHaveLength(0);
+ expect(capture).not.toHaveBeenCalled();
+ });
+
+ it("ignores retained string variants when feature flags are disabled", async () => {
+ const bodies = captureAssignments();
+ vi.spyOn(environment, "areFeatureFlagsEnabled").mockReturnValue(false);
+ ld.flags = { "onboarding-copy": "b" };
+ ld.client = null;
+
+ const { result } = renderHook(
+ () => useLaunchDarklyExperiment("onboarding-copy"),
+ { wrapper },
+ );
+
+ expect.soft(result.current).toEqual({ variant: null, isResolved: true });
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect.soft(bodies).toHaveLength(0);
+ expect(capture).not.toHaveBeenCalled();
+ });
+});
diff --git a/autogpt_platform/frontend/src/services/experiments/__tests__/useReportAssignment.test.tsx b/autogpt_platform/frontend/src/services/experiments/__tests__/useReportAssignment.test.tsx
new file mode 100644
index 0000000000..2ffc6afcd9
--- /dev/null
+++ b/autogpt_platform/frontend/src/services/experiments/__tests__/useReportAssignment.test.tsx
@@ -0,0 +1,189 @@
+import { server } from "@/mocks/mock-server";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { renderHook, waitFor } from "@testing-library/react";
+import { http, HttpResponse } from "msw";
+import { ReactNode, StrictMode, useState } from "react";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ resetReportedAssignmentsForTests,
+ useReportAssignment,
+} from "../useReportAssignment";
+
+const auth = vi.hoisted(() => ({
+ user: { id: "user-1" } as { id: string } | null,
+}));
+
+vi.mock("@/lib/auth/hooks/useAuth", () => ({
+ useAuth: () => ({ user: auth.user, isUserLoading: false }),
+}));
+
+interface Props {
+ children: ReactNode;
+}
+
+function Wrapper({ children }: Props) {
+ const [client] = useState(
+ () => new QueryClient({ defaultOptions: { mutations: { retry: false } } }),
+ );
+ return (
+ <StrictMode>
+ <QueryClientProvider client={client}>{children}</QueryClientProvider>
+ </StrictMode>
+ );
+}
+
+function useAssignment(variant = "control") {
+ useReportAssignment({
+ experimentKey: "pricing-test",
+ variant,
+ isResolved: true,
+ source: "posthog",
+ });
+}
+
+beforeEach(() => {
+ auth.user = { id: "user-1" };
+ resetReportedAssignmentsForTests();
+});
+
+describe("useReportAssignment retry", () => {
+ it("retries a transient failure without remounting or changing inputs", async () => {
+ const bodies: unknown[] = [];
+ server.use(
+ http.post("*/api/experiments/assignments", async ({ request }) => {
+ bodies.push(await request.json());
+ if (bodies.length === 1) return new HttpResponse(null, { status: 503 });
+ return HttpResponse.json({ variant: "control" });
+ }),
+ );
+
+ renderHook(() => useAssignment(), { wrapper: Wrapper });
+ await waitFor(() => expect(bodies).toHaveLength(2), { timeout: 2500 });
+ expect(bodies[0]).toEqual(bodies[1]);
+
+ renderHook(() => useAssignment(), { wrapper: Wrapper });
+ await new Promise((resolve) => setTimeout(resolve, 30));
+ expect(bodies).toHaveLength(2);
+ });
+
+ it("limits retries and allows a later mount to try again", async () => {
+ const attempts = vi.fn();
+ server.use(
+ http.post("*/api/experiments/assignments", () => {
+ attempts();
+ return new HttpResponse(null, { status: 503 });
+ }),
+ );
+ const first = renderHook(() => useAssignment(), { wrapper: Wrapper });
+ await waitFor(() => expect(attempts).toHaveBeenCalledTimes(3), {
+ timeout: 4500,
+ });
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ first.unmount();
+
+ server.use(
+ http.post("*/api/experiments/assignments", () => {
+ attempts();
+ return HttpResponse.json({ variant: "control" });
+ }),
+ );
+ renderHook(() => useAssignment(), { wrapper: Wrapper });
+ await waitFor(() => expect(attempts).toHaveBeenCalledTimes(4));
+ });
+
+ it("replaces an in-flight report without letting its late failure erase the new claim", async () => {
+ const bodies: unknown[] = [];
+ let releaseFirst: () => void = () => {};
+ const firstResponse = new Promise<void>((resolve) => {
+ releaseFirst = resolve;
+ });
+ const firstFinished = vi.fn();
+ server.use(
+ http.post("*/api/experiments/assignments", async ({ request }) => {
+ bodies.push(await request.json());
+ if (bodies.length === 1) {
+ await firstResponse;
+ firstFinished();
+ return new HttpResponse(null, { status: 503 });
+ }
+ return HttpResponse.json({ variant: "control" });
+ }),
+ );
+
+ const first = renderHook(() => useAssignment(), { wrapper: Wrapper });
+ await waitFor(() => expect(bodies).toHaveLength(1));
+ first.unmount();
+ renderHook(() => useAssignment(), { wrapper: Wrapper });
+ try {
+ await waitFor(() => expect(bodies).toHaveLength(2));
+ } finally {
+ releaseFirst();
+ }
+ await waitFor(() => expect(firstFinished).toHaveBeenCalledOnce());
+ await new Promise((resolve) => setTimeout(resolve, 30));
+
+ renderHook(() => useAssignment(), { wrapper: Wrapper });
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ expect(bodies).toHaveLength(2);
+ });
+
+ it("does not retry rejected input", async () => {
+ const attempts = vi.fn();
+ server.use(
+ http.post("*/api/experiments/assignments", () => {
+ attempts();
+ return new HttpResponse(null, { status: 422 });
+ }),
+ );
+ renderHook(() => useAssignment(), { wrapper: Wrapper });
+ await waitFor(() => expect(attempts).toHaveBeenCalledTimes(1));
+ await new Promise((resolve) => setTimeout(resolve, 1200));
+ expect(attempts).toHaveBeenCalledTimes(1);
+ });
+
+ it("cancels a scheduled retry on unmount", async () => {
+ const attempts = vi.fn();
+ server.use(
+ http.post("*/api/experiments/assignments", () => {
+ attempts();
+ return new HttpResponse(null, { status: 503 });
+ }),
+ );
+ const { unmount } = renderHook(() => useAssignment(), { wrapper: Wrapper });
+ await waitFor(() => expect(attempts).toHaveBeenCalledTimes(1));
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ unmount();
+ await new Promise((resolve) => setTimeout(resolve, 1200));
+ expect(attempts).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not retry the previous user's variant after an account switch", async () => {
+ const bodies: unknown[] = [];
+ server.use(
+ http.post("*/api/experiments/assignments", async ({ request }) => {
+ bodies.push(await request.json());
+ if (bodies.length === 1) return new HttpResponse(null, { status: 503 });
+ return HttpResponse.json({ variant: "treatment" });
+ }),
+ );
+ const { rerender } = renderHook(({ variant }) => useAssignment(variant), {
+ initialProps: { variant: "control" },
+ wrapper: Wrapper,
+ });
+ await waitFor(() => expect(bodies).toHaveLength(1));
+ await new Promise((resolve) => setTimeout(resolve, 20));
+
+ auth.user = { id: "user-2" };
+ rerender({ variant: "treatment" });
+ await waitFor(() => expect(bodies).toHaveLength(2));
+ await new Promise((resolve) => setTimeout(resolve, 1200));
+ expect(bodies).toEqual([
+ { experiment_key: "pricing-test", variant: "control", source: "posthog" },
+ {
+ experiment_key: "pricing-test",
+ variant: "treatment",
+ source: "posthog",
+ },
+ ]);
+ });
+});
diff --git a/autogpt_platform/frontend/src/services/experiments/useExperiment.ts b/autogpt_platform/frontend/src/services/experiments/useExperiment.ts
new file mode 100644
index 0000000000..43cdc0c9bb
--- /dev/null
+++ b/autogpt_platform/frontend/src/services/experiments/useExperiment.ts
@@ -0,0 +1,41 @@
+"use client";
+
+import { environment } from "@/services/environment";
+import { useFeatureFlagVariantKey } from "@posthog/react";
+import { useReportAssignment } from "./useReportAssignment";
+
+export { resetReportedAssignmentsForTests } from "./useReportAssignment";
+
+/**
+ * Read an A/B/C experiment arm bucketed by PostHog and make it durable.
+ *
+ * PostHog buckets the user (multivariate feature flag) and records the
+ * exposure for its own significance testing. The arm is also reported to
+ * the backend once per user and experiment, so the `analytics.*` views can
+ * split activation, retention and cost by variant.
+ *
+ * `variant` is the arm key (`"control"`, `"yearly-pro"`, ...) or `null` when
+ * the user is not enrolled. `isResolved` is false while PostHog is still
+ * loading flags: render the control experience and hold any one-shot
+ * side effects until it flips, so a late variant is never mis-recorded as
+ * control. When PostHog is disabled the experiment resolves immediately.
+ *
+ * For an experiment bucketed by a LaunchDarkly flag instead, use
+ * `useLaunchDarklyExperiment`; both report into the same table.
+ */
+export function useExperiment(experimentKey: string) {
+ const rawVariant = useFeatureFlagVariantKey(experimentKey);
+
+ const isResolved =
+ rawVariant !== undefined || !environment.isPostHogEnabled();
+ const variant = typeof rawVariant === "string" ? rawVariant : null;
+
+ useReportAssignment({
+ experimentKey,
+ variant,
+ isResolved,
+ source: "posthog",
+ });
+
+ return { variant, isResolved };
+}
diff --git a/autogpt_platform/frontend/src/services/experiments/useLaunchDarklyExperiment.ts b/autogpt_platform/frontend/src/services/experiments/useLaunchDarklyExperiment.ts
new file mode 100644
index 0000000000..7a9ba12d57
--- /dev/null
+++ b/autogpt_platform/frontend/src/services/experiments/useLaunchDarklyExperiment.ts
@@ -0,0 +1,91 @@
+"use client";
+
+import { useAuth } from "@/lib/auth/hooks/useAuth";
+import { environment } from "@/services/environment";
+import { useFlags, useLDClient } from "launchdarkly-react-client-sdk";
+import posthog from "posthog-js";
+import { useEffect, useState } from "react";
+import { useReportAssignment } from "./useReportAssignment";
+
+// `waitForInitialization` takes seconds, not milliseconds.
+const LD_READY_TIMEOUT_SECONDS = 5;
+
+/**
+ * Run an experiment whose arms are a LaunchDarkly string flag.
+ *
+ * The team gates with LaunchDarkly, but LaunchDarkly only knows a user after
+ * login and PostHog is where funnels and significance live. This hook is
+ * the bridge: LaunchDarkly picks the arm, and the hook (1) reports the
+ * assignment to the backend like `useExperiment` does, so Looker can split
+ * by arm, and (2) sends PostHog an `experiment_exposed` event carrying the
+ * arm as `$feature/<flag>`, so a PostHog experiment can use it as its
+ * exposure and measure the activation events against it.
+ *
+ * `variant` is the flag's string value, or `null` when the flag is off,
+ * boolean, or not yet known. `isResolved` flips once the LaunchDarkly client
+ * has initialised (or timed out) so a late arm is never recorded as control.
+ */
+export function useLaunchDarklyExperiment(flagKey: string) {
+ const { user } = useAuth();
+ const userID = user?.id;
+ const postHogEnabled = Boolean(environment.isPostHogEnabled());
+ const flags = useFlags<Record<string, unknown>>();
+ const client = useLDClient();
+ const [isClientReady, setIsClientReady] = useState(false);
+ const flagsEnabled = environment.areFeatureFlagsEnabled();
+
+ useEffect(() => {
+ if (!client) return;
+ let active = true;
+ client
+ .waitForInitialization(LD_READY_TIMEOUT_SECONDS)
+ .catch(() => undefined)
+ .finally(() => {
+ if (active) setIsClientReady(true);
+ });
+ return () => {
+ active = false;
+ };
+ }, [client]);
+
+ const isResolved = !flagsEnabled || isClientReady;
+ const raw = flags[flagKey];
+ const variant = flagsEnabled && typeof raw === "string" ? raw : null;
+
+ useReportAssignment({
+ experimentKey: flagKey,
+ variant,
+ isResolved,
+ source: "launchdarkly",
+ });
+
+ useEffect(() => {
+ if (!isResolved || !variant || !userID || !postHogEnabled) return;
+ if (!claimExposure(userID, flagKey, variant)) return;
+ try {
+ posthog.capture("experiment_exposed", {
+ experiment_key: flagKey,
+ variant,
+ provider: "launchdarkly",
+ [`$feature/${flagKey}`]: variant,
+ });
+ } catch {
+ // Analytics must never break the experience it measures.
+ }
+ }, [isResolved, variant, flagKey, userID, postHogEnabled]);
+
+ return { variant, isResolved };
+}
+
+const exposures = new Set<string>();
+
+function claimExposure(userID: string, flagKey: string, variant: string) {
+ const key = JSON.stringify([userID, flagKey, variant]);
+ if (exposures.has(key)) return false;
+ exposures.add(key);
+ return true;
+}
+
+export function resetExposuresForTests() {
+ exposures.clear();
+}
diff --git a/autogpt_platform/frontend/src/services/experiments/useReportAssignment.ts b/autogpt_platform/frontend/src/services/experiments/useReportAssignment.ts
new file mode 100644
index 0000000000..c5a62fa48f
--- /dev/null
+++ b/autogpt_platform/frontend/src/services/experiments/useReportAssignment.ts
@@ -0,0 +1,106 @@
+"use client";
+
+import { usePostExperimentsRecordExperimentAssignment } from "@/app/api/__generated__/endpoints/experiments/experiments";
+import { useAuth } from "@/lib/auth/hooks/useAuth";
+import { ApiError } from "@/lib/autogpt-server-api/helpers";
+import { useEffect } from "react";
+
+export type AssignmentSource = "posthog" | "launchdarkly";
+
+interface Args {
+ experimentKey: string;
+ variant: string | null;
+ isResolved: boolean;
+ source: AssignmentSource;
+}
+
+/**
+ * Report an experiment arm to the backend once per user and experiment, so
+ * the assignment is available to the `analytics.*` views whichever tool did
+ * the bucketing. Only string arms are reported: an unresolved or
+ * not-enrolled flag is not an assignment.
+ */
+export function useReportAssignment({
+ experimentKey,
+ variant,
+ isResolved,
+ source,
+}: Args) {
+ const { user } = useAuth();
+ const { mutateAsync: recordAssignment } =
+ usePostExperimentsRecordExperimentAssignment({
+ mutation: { retry: false },
+ });
+ const userID = user?.id ?? null;
+
+ useEffect(() => {
+ if (!isResolved || !variant || !userID) return;
+ const claim = claimAssignmentReport(userID, experimentKey);
+ if (!claim) return;
+ const reportUserID = userID;
+ const reportClaim = claim;
+ const data = { experiment_key: experimentKey, variant, source };
+ let active = true;
+ let succeeded = false;
+ let retryTimer: ReturnType<typeof setTimeout> | undefined;
+ let failures = 0;
+
+ async function report() {
+ retryTimer = undefined;
+ try {
+ await recordAssignment({ data });
+ succeeded = true;
+ } catch (error) {
+ failures += 1;
+ if (!active || failures > 2 || !isRetryable(error)) {
+ releaseAssignmentReport(reportUserID, experimentKey, reportClaim);
+ return;
+ }
+ retryTimer = setTimeout(report, 1000 * 2 ** (failures - 1));
+ }
+ }
+
+ retryTimer = setTimeout(report, 0);
+ return () => {
+ active = false;
+ if (retryTimer !== undefined) {
+ clearTimeout(retryTimer);
+ }
+ if (!succeeded) {
+ releaseAssignmentReport(userID, experimentKey, claim);
+ }
+ };
+ }, [isResolved, variant, userID, experimentKey, source, recordAssignment]);
+}
+
+function isRetryable(error: unknown) {
+ return (
+ !(error instanceof ApiError) ||
+ error.status === 408 ||
+ error.status === 429 ||
+ error.status >= 500
+ );
+}
+
+const reportedAssignments = new Map<string, symbol>();
+
+function claimAssignmentReport(userID: string, experimentKey: string) {
+ const key = `${userID}:${experimentKey}`;
+ if (reportedAssignments.has(key)) return;
+ const claim = Symbol();
+ reportedAssignments.set(key, claim);
+ return claim;
+}
+
+function releaseAssignmentReport(
+ userID: string,
+ experimentKey: string,
+ claim: symbol,
+) {
+ const key = `${userID}:${experimentKey}`;
+ if (reportedAssignments.get(key) === claim) reportedAssignments.delete(key);
+}
+
+export function resetReportedAssignmentsForTests() {
+ reportedAssignments.clear();
+}