Skip to content

Open source CapCut alternative.

TypeScript

feat(site): experiments and settings on su

David Le9996f42090aa
diff --git a/site/src/app/api/experiments/[id]/assignments/route.ts b/site/src/app/api/experiments/[id]/assignments/route.ts
new file mode 100644
index 00000000..a23e21cb
--- /dev/null
+++ b/site/src/app/api/experiments/[id]/assignments/route.ts
@@ -0,0 +1,52 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+
+import { findExperiment, invalidResponse, listExperiments } from "@/lib/config/experimentList";
+import { parseVariants } from "@/lib/config/effective";
+import { notFoundResponse, withSuperUser } from "@/lib/donkey-api-auth";
+import { prisma } from "@/lib/prisma";
+
+type Params = { params: Promise<{ id: string }> };
+
+// An assignment written by hand: the account reads this variant from now on,
+// whatever the audience and the hash say; a null variant holds the account
+// out of this experiment. Manual rows are never counted in the results.
+const putSchema = z
+ .object({
+ email: z.email(),
+ variant: z.string().min(1).nullable(),
+ })
+ .strict();
+
+export const PUT = withSuperUser(async (request, { params }: Params) => {
+ const { id } = await params;
+ const current = await findExperiment(id);
+ if (!current) return notFoundResponse();
+ const parsed = putSchema.safeParse(await request.json());
+ if (!parsed.success) return invalidResponse(parsed.error.issues);
+ const { email, variant } = parsed.data;
+ if (variant !== null && !parseVariants(current.variants).some((v) => v.key === variant)) {
+ return invalidResponse([{ path: ["variant"], message: `No variant "${variant}".` }]);
+ }
+ const user = await prisma.user.findUnique({ where: { email }, select: { id: true } });
+ if (!user) return invalidResponse([{ path: ["email"], message: "No account has this email." }]);
+ await prisma.experimentAssignment.upsert({
+ where: { experimentId_userId: { experimentId: id, userId: user.id } },
+ create: { experimentId: id, userId: user.id, variant, manual: true },
+ update: { variant, manual: true },
+ });
+ return NextResponse.json({ experiments: await listExperiments() });
+});
+
+const deleteSchema = z.object({ userId: z.string().min(1) }).strict();
+
+// Drops the row; the next read assigns the account afresh, if it is eligible.
+export const DELETE = withSuperUser(async (request, { params }: Params) => {
+ const { id } = await params;
+ const current = await findExperiment(id);
+ if (!current) return notFoundResponse();
+ const parsed = deleteSchema.safeParse(await request.json());
+ if (!parsed.success) return invalidResponse(parsed.error.issues);
+ await prisma.experimentAssignment.deleteMany({ where: { experimentId: id, userId: parsed.data.userId } });
+ return NextResponse.json({ experiments: await listExperiments() });
+});
diff --git a/site/src/app/api/experiments/[id]/route.ts b/site/src/app/api/experiments/[id]/route.ts
new file mode 100644
index 00000000..f2d16a67
--- /dev/null
+++ b/site/src/app/api/experiments/[id]/route.ts
@@ -0,0 +1,98 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+
+import { findExperiment, invalidResponse, listExperiments } from "@/lib/config/experimentList";
+import type { Prisma } from "@/generated/prisma/client";
+import {
+ canTransition,
+ experimentSchema,
+ removedAssignedVariants,
+ statusSchema,
+ type ExperimentStatus,
+} from "@/lib/config/experiment";
+import { notFoundResponse, withSuperUser } from "@/lib/donkey-api-auth";
+import { prisma } from "@/lib/prisma";
+
+type Params = { params: Promise<{ id: string }> };
+
+// The whole definition, replaced. Once accounts hold a variant its key stays;
+// new variants may join.
+export const PUT = withSuperUser(async (request, { params }: Params) => {
+ const { id } = await params;
+ const current = await findExperiment(id);
+ if (!current) return notFoundResponse();
+ const parsed = experimentSchema.safeParse(await request.json());
+ if (!parsed.success) return invalidResponse(parsed.error.issues);
+ const input = parsed.data;
+
+ if (input.key !== current.key) {
+ const taken = await prisma.experiment.findUnique({ where: { key: input.key }, select: { id: true } });
+ if (taken) return invalidResponse([{ path: ["key"], message: "An experiment with this key exists." }]);
+ }
+ const assigned = await prisma.experimentAssignment.groupBy({
+ by: ["variant"],
+ where: { experimentId: id },
+ });
+ const removed = removedAssignedVariants(
+ input,
+ assigned.flatMap((a) => (a.variant === null ? [] : [a.variant])),
+ );
+ if (removed.length > 0) {
+ return invalidResponse([
+ { path: ["variants"], message: `Accounts hold ${removed.join(", ")}; those variants must stay.` },
+ ]);
+ }
+
+ await prisma.experiment.update({
+ where: { id },
+ data: {
+ key: input.key,
+ name: input.name,
+ description: input.description,
+ variants: input.variants as Prisma.InputJsonValue,
+ audience: input.audience as Prisma.InputJsonValue,
+ percent: input.percent,
+ metrics: input.metrics as Prisma.InputJsonValue,
+ actorUserId: request.donkey.userId,
+ },
+ });
+ return NextResponse.json({ experiments: await listExperiments() });
+});
+
+const patchSchema = z.object({ status: statusSchema }).strict();
+
+export const PATCH = withSuperUser(async (request, { params }: Params) => {
+ const { id } = await params;
+ const current = await findExperiment(id);
+ if (!current) return notFoundResponse();
+ const parsed = patchSchema.safeParse(await request.json());
+ if (!parsed.success) return invalidResponse(parsed.error.issues);
+ const from = current.status as ExperimentStatus;
+ const to = parsed.data.status;
+ if (!canTransition(from, to)) {
+ return invalidResponse([{ path: ["status"], message: `Cannot go from ${from} to ${to}.` }]);
+ }
+ const now = new Date();
+ await prisma.experiment.update({
+ where: { id },
+ data: {
+ status: to,
+ actorUserId: request.donkey.userId,
+ ...(to === "running" && !current.startedAt ? { startedAt: now } : {}),
+ ...(to === "ended" ? { endedAt: now } : {}),
+ },
+ });
+ return NextResponse.json({ experiments: await listExperiments() });
+});
+
+// A draft nobody was assigned to can go; anything that ran keeps its record.
+export const DELETE = withSuperUser(async (_request, { params }: Params) => {
+ const { id } = await params;
+ const current = await findExperiment(id);
+ if (!current) return notFoundResponse();
+ if (current.status !== "draft") {
+ return invalidResponse([{ path: ["status"], message: "Only a draft can be deleted." }]);
+ }
+ await prisma.experiment.delete({ where: { id } });
+ return NextResponse.json({ experiments: await listExperiments() });
+});
diff --git a/site/src/app/api/experiments/holdout/route.ts b/site/src/app/api/experiments/holdout/route.ts
new file mode 100644
index 00000000..1e5c718b
--- /dev/null
+++ b/site/src/app/api/experiments/holdout/route.ts
@@ -0,0 +1,43 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+
+import { invalidResponse, listHoldout } from "@/lib/config/experimentList";
+import { withSuperUser } from "@/lib/donkey-api-auth";
+import { prisma } from "@/lib/prisma";
+
+// The accounts kept out of every experiment. A held-out account is never
+// assigned, reads the plain configuration even where it holds a row, and is
+// left out of every result.
+
+export const GET = withSuperUser(async () => {
+ return NextResponse.json({ holdout: await listHoldout() });
+});
+
+const postSchema = z
+ .object({
+ email: z.email(),
+ note: z.string().trim().max(200).nullable(),
+ })
+ .strict();
+
+export const POST = withSuperUser(async (request) => {
+ const parsed = postSchema.safeParse(await request.json());
+ if (!parsed.success) return invalidResponse(parsed.error.issues);
+ const user = await prisma.user.findUnique({ where: { email: parsed.data.email }, select: { id: true } });
+ if (!user) return invalidResponse([{ path: ["email"], message: "No account has this email." }]);
+ await prisma.experimentHoldout.upsert({
+ where: { userId: user.id },
+ create: { userId: user.id, note: parsed.data.note, actorUserId: request.donkey.userId },
+ update: { note: parsed.data.note, actorUserId: request.donkey.userId },
+ });
+ return NextResponse.json({ holdout: await listHoldout() });
+});
+
+const deleteSchema = z.object({ userId: z.string().min(1) }).strict();
+
+export const DELETE = withSuperUser(async (request) => {
+ const parsed = deleteSchema.safeParse(await request.json());
+ if (!parsed.success) return invalidResponse(parsed.error.issues);
+ await prisma.experimentHoldout.deleteMany({ where: { userId: parsed.data.userId } });
+ return NextResponse.json({ holdout: await listHoldout() });
+});
diff --git a/site/src/app/api/experiments/route.ts b/site/src/app/api/experiments/route.ts
new file mode 100644
index 00000000..ceaa6b8a
--- /dev/null
+++ b/site/src/app/api/experiments/route.ts
@@ -0,0 +1,35 @@
+import { NextResponse } from "next/server";
+
+import { invalidResponse, listExperiments } from "@/lib/config/experimentList";
+import { experimentSchema } from "@/lib/config/experiment";
+import { withSuperUser } from "@/lib/donkey-api-auth";
+import { prisma } from "@/lib/prisma";
+import type { Prisma } from "@/generated/prisma/client";
+
+export const GET = withSuperUser(async () => {
+ return NextResponse.json({ experiments: await listExperiments() });
+});
+
+// A new experiment starts as a draft; starting it is a status change.
+export const POST = withSuperUser(async (request) => {
+ const parsed = experimentSchema.safeParse(await request.json());
+ if (!parsed.success) return invalidResponse(parsed.error.issues);
+ const input = parsed.data;
+ const taken = await prisma.experiment.findUnique({ where: { key: input.key }, select: { id: true } });
+ if (taken) {
+ return invalidResponse([{ path: ["key"], message: "An experiment with this key exists." }]);
+ }
+ await prisma.experiment.create({
+ data: {
+ key: input.key,
+ name: input.name,
+ description: input.description,
+ variants: input.variants as Prisma.InputJsonValue,
+ audience: input.audience as Prisma.InputJsonValue,
+ percent: input.percent,
+ metrics: input.metrics as Prisma.InputJsonValue,
+ actorUserId: request.donkey.userId,
+ },
+ });
+ return NextResponse.json({ experiments: await listExperiments() });
+});
diff --git a/site/src/app/api/settings/route.ts b/site/src/app/api/settings/route.ts
new file mode 100644
index 00000000..9114a0f4
--- /dev/null
+++ b/site/src/app/api/settings/route.ts
@@ -0,0 +1,80 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+
+import type { Prisma } from "@/generated/prisma/client";
+import { SETTINGS, SETTING_KEYS, isSettingKey } from "@/lib/config/registry";
+import { notFoundResponse, withSuperUser } from "@/lib/donkey-api-auth";
+import { prisma } from "@/lib/prisma";
+
+// The settings registry as su sees it: every declared key with its default,
+// its override when one exists, and its schema for the form. Writes parse
+// against the key's own schema, so a stored override always reads back.
+
+const putSchema = z.object({ key: z.string(), value: z.unknown() }).strict();
+
+async function listSettings() {
+ const rows = await prisma.settingOverride.findMany();
+ const overrides = new Map(rows.map((row) => [row.key, row]));
+ return SETTING_KEYS.map((key) => {
+ const def = SETTINGS[key];
+ const row = overrides.get(key);
+ const parsed = row ? def.schema.safeParse(row.value) : null;
+ return {
+ key,
+ title: def.title,
+ description: def.description,
+ public: def.public,
+ default: def.default,
+ value: parsed?.success ? parsed.data : def.default,
+ overridden: row !== undefined,
+ invalid: parsed !== null && !parsed.success,
+ updatedAt: row?.updatedAt.toISOString() ?? null,
+ schema: z.toJSONSchema(def.schema),
+ };
+ });
+}
+
+export const GET = withSuperUser(async () => {
+ return NextResponse.json({ settings: await listSettings() });
+});
+
+export const PUT = withSuperUser(async (request) => {
+ const parsed = putSchema.safeParse(await request.json());
+ if (!parsed.success || !isSettingKey(parsed.data.key)) {
+ return NextResponse.json(
+ { error: "Invalid request", issues: [{ path: "key", message: "Unknown setting." }] },
+ { status: 400 },
+ );
+ }
+ const { key } = parsed.data;
+ const value = SETTINGS[key].schema.safeParse(parsed.data.value);
+ if (!value.success) {
+ return NextResponse.json(
+ {
+ error: "Invalid request",
+ issues: value.error.issues.map((issue) => ({
+ message: issue.message,
+ path: ["value", ...issue.path].join("."),
+ })),
+ },
+ { status: 400 },
+ );
+ }
+ const json = value.data as Prisma.InputJsonValue;
+ await prisma.settingOverride.upsert({
+ where: { key },
+ create: { key, value: json, actorUserId: request.donkey.userId },
+ update: { value: json, actorUserId: request.donkey.userId },
+ });
+ return NextResponse.json({ settings: await listSettings() });
+});
+
+export const DELETE = withSuperUser(async (request) => {
+ const key = new URL(request.url).searchParams.get("key");
+ if (!key || !isSettingKey(key)) {
+ return NextResponse.json({ error: "Missing key" }, { status: 400 });
+ }
+ const { count } = await prisma.settingOverride.deleteMany({ where: { key } });
+ if (count === 0) return notFoundResponse();
+ return NextResponse.json({ settings: await listSettings() });
+});
diff --git a/site/src/app/su/experiments/ExperimentDialog.tsx b/site/src/app/su/experiments/ExperimentDialog.tsx
new file mode 100644
index 00000000..54755f1d
--- /dev/null
+++ b/site/src/app/su/experiments/ExperimentDialog.tsx
@@ -0,0 +1,501 @@
+"use client";
+
+import { useState } from "react";
+import { Plus, X } from "lucide-react";
+
+import { SchemaField, type JsonSchema } from "@/app/su/experiments/SchemaField";
+import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
+import type { AudienceInput } from "@donkeycut/abexp";
+import { METRIC_SOURCES, experimentSchema, type ExperimentInput } from "@/lib/config/experiment";
+import { PUBLIC_SETTING_KEYS, SETTINGS, type SettingKey } from "@/lib/config/registry";
+import { useSettings } from "@/queries/settings";
+import {
+ useCreateExperiment,
+ useUpdateExperiment,
+ type ExperimentSummary,
+} from "@/queries/experiments";
+
+// One form for a new experiment and for editing one: who it enrols, the
+// variants they draw (the first is the control), and the metrics that read
+// it. A variant's settings are drawn with the same fields the settings tab
+// uses.
+
+type Draft = {
+ key: string;
+ name: string;
+ description: string;
+ percent: string;
+ audience: {
+ countries: string;
+ createdAfter: string;
+ createdBefore: string;
+ plan: "any" | "free" | "pro";
+ paid: "any" | "yes" | "no";
+ activeWithinDays: string;
+ storageUsedPercentAtLeast: string;
+ creditsUsedPercentAtLeast: string;
+ };
+ variants: { key: string; name: string; weight: string; config: Record<string, unknown> }[];
+ metrics: { key: string; name: string; source: (typeof METRIC_SOURCES)[number]; event: string }[];
+};
+
+const blank = (): Draft => ({
+ key: "",
+ name: "",
+ description: "",
+ percent: "100",
+ audience: {
+ countries: "",
+ createdAfter: "",
+ createdBefore: "",
+ plan: "any",
+ paid: "any",
+ activeWithinDays: "",
+ storageUsedPercentAtLeast: "",
+ creditsUsedPercentAtLeast: "",
+ },
+ variants: [
+ { key: "control", name: "Control", weight: "50", config: {} },
+ { key: "treatment", name: "Treatment", weight: "50", config: {} },
+ ],
+ metrics: [{ key: "purchase", name: "Purchase", source: "purchase", event: "" }],
+});
+
+const day = (iso: string | null) => (iso ? iso.slice(0, 10) : "");
+const numberOrBlank = (n: number | null) => (n === null ? "" : String(n));
+
+const fromSummary = (e: ExperimentSummary): Draft => ({
+ key: e.key,
+ name: e.name,
+ description: e.description ?? "",
+ percent: String(e.percent),
+ audience: {
+ countries: e.audience.countries.join(", "),
+ createdAfter: day(e.audience.createdAfter),
+ createdBefore: day(e.audience.createdBefore),
+ plan: e.audience.plan,
+ paid: e.audience.paid,
+ activeWithinDays: numberOrBlank(e.audience.activeWithinDays),
+ storageUsedPercentAtLeast: numberOrBlank(e.audience.storageUsedPercentAtLeast),
+ creditsUsedPercentAtLeast: numberOrBlank(e.audience.creditsUsedPercentAtLeast),
+ },
+ variants: e.variants.map((v) => ({ ...v, weight: String(v.weight) })),
+ metrics: e.metrics.map((m) => ({ ...m, event: m.event ?? "" })),
+});
+
+const dayStart = (d: string) => (d ? new Date(`${d}T00:00:00Z`).toISOString() : null);
+const numberOrNull = (s: string) => (s.trim() === "" ? null : Number(s));
+
+function toInput(draft: Draft): unknown {
+ const audience: AudienceInput = {
+ countries: draft.audience.countries
+ .split(",")
+ .map((s) => s.trim().toUpperCase())
+ .filter(Boolean),
+ createdAfter: dayStart(draft.audience.createdAfter),
+ createdBefore: dayStart(draft.audience.createdBefore),
+ plan: draft.audience.plan,
+ paid: draft.audience.paid,
+ activeWithinDays: numberOrNull(draft.audience.activeWithinDays),
+ storageUsedPercentAtLeast: numberOrNull(draft.audience.storageUsedPercentAtLeast),
+ creditsUsedPercentAtLeast: numberOrNull(draft.audience.creditsUsedPercentAtLeast),
+ };
+ return {
+ key: draft.key.trim(),
+ name: draft.name.trim(),
+ description: draft.description.trim() || null,
+ audience,
+ percent: Number(draft.percent),
+ variants: draft.variants.map((v) => ({
+ key: v.key.trim(),
+ name: v.name.trim(),
+ weight: Number(v.weight),
+ config: v.config,
+ })),
+ metrics: draft.metrics.map((m) => ({
+ key: m.key.trim(),
+ name: m.name.trim(),
+ source: m.source,
+ event: m.source === "event" ? m.event.trim() || null : null,
+ })),
+ };
+}
+
+export function ExperimentDialog({
+ existing,
+ open,
+ onOpenChange,
+}: {
+ existing: ExperimentSummary | null;
+ open: boolean;
+ onOpenChange: (open: boolean) => void;
+}) {
+ const [draft, setDraft] = useState<Draft>(() => (existing ? fromSummary(existing) : blank()));
+ const [issues, setIssues] = useState<string[]>([]);
+ const create = useCreateExperiment();
+ const update = useUpdateExperiment();
+ const settings = useSettings();
+ const schemas = new Map(settings.data?.settings.map((s) => [s.key, s.schema as JsonSchema]));
+ // Anything an account holds, drawn or written by hand: the route rejects a
+ // save that drops either.
+ const assignedKeys = new Set([
+ ...(existing?.stats.filter((s) => s.assigned > 0).map((s) => s.key) ?? []),
+ ...(existing?.overrides.flatMap((o) => (o.variant === null ? [] : [o.variant])) ?? []),
+ ]);
+ const pending = create.isPending || update.isPending;
+
+ const submit = () => {
+ const parsed = experimentSchema.safeParse(toInput(draft));
+ if (!parsed.success) {
+ setIssues(parsed.error.issues.map((i) => `${i.path.join(".") || "form"}: ${i.message}`));
+ return;
+ }
+ setIssues([]);
+ const input: ExperimentInput = parsed.data;
+ const done = { onSuccess: () => onOpenChange(false), onError: (e: Error) => setIssues([e.message]) };
+ if (existing) update.mutate({ id: existing.id, ...input }, done);
+ else create.mutate(input, done);
+ };
+
+ const setAudience = (patch: Partial<Draft["audience"]>) =>
+ setDraft((d) => ({ ...d, audience: { ...d.audience, ...patch } }));
+ const setVariant = (i: number, patch: Partial<Draft["variants"][number]>) =>
+ setDraft((d) => ({ ...d, variants: d.variants.map((v, j) => (j === i ? { ...v, ...patch } : v)) }));
+ const setMetric = (i: number, patch: Partial<Draft["metrics"][number]>) =>
+ setDraft((d) => ({ ...d, metrics: d.metrics.map((m, j) => (j === i ? { ...m, ...patch } : m)) }));
+
+ return (
+ <Dialog open={open} onOpenChange={onOpenChange}>
+ <DialogContent className="max-h-[90vh] overflow-y-auto sm:max-w-3xl">
+ <DialogHeader>
+ <DialogTitle>{existing ? "Edit experiment" : "New experiment"}</DialogTitle>
+ <DialogDescription>
+ Accounts in the audience draw a variant by weight the first time they load
+ the app, and keep it. The first variant is the control.
+ </DialogDescription>
+ </DialogHeader>
+
+ <div className="grid gap-5">
+ <div className="grid grid-cols-2 gap-3">
+ <Field label="Key" htmlFor="exp-key">
+ <Input
+ id="exp-key"
+ placeholder="welcome_copy_v1"
+ value={draft.key}
+ onChange={(e) => setDraft({ ...draft, key: e.target.value })}
+ />
+ </Field>
+ <Field label="Name" htmlFor="exp-name">
+ <Input id="exp-name" value={draft.name} onChange={(e) => setDraft({ ...draft, name: e.target.value })} />
+ </Field>
+ </div>
+ <Field label="Description" htmlFor="exp-desc">
+ <Textarea
+ id="exp-desc"
+ rows={2}
+ value={draft.description}
+ onChange={(e) => setDraft({ ...draft, description: e.target.value })}
+ />
+ </Field>
+
+ <div className="space-y-3">
+ <Label>Audience</Label>
+ <div className="grid grid-cols-3 gap-3">
+ <Field label="Countries (empty = any)" htmlFor="aud-countries">
+ <Input
+ id="aud-countries"
+ placeholder="US, GB, CA"
+ value={draft.audience.countries}
+ onChange={(e) => setAudience({ countries: e.target.value })}
+ />
+ </Field>
+ <Field label="Created on or after" htmlFor="aud-after">
+ <Input
+ id="aud-after"
+ type="date"
+ value={draft.audience.createdAfter}
+ onChange={(e) => setAudience({ createdAfter: e.target.value })}
+ />
+ </Field>
+ <Field label="Created before" htmlFor="aud-before">
+ <Input
+ id="aud-before"
+ type="date"
+ value={draft.audience.createdBefore}
+ onChange={(e) => setAudience({ createdBefore: e.target.value })}
+ />
+ </Field>
+ <Field label="Plan" htmlFor="aud-plan">
+ <Select value={draft.audience.plan} onValueChange={(v) => setAudience({ plan: v as Draft["audience"]["plan"] })}>
+ <SelectTrigger id="aud-plan">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="any">Any</SelectItem>
+ <SelectItem value="free">No Pro</SelectItem>
+ <SelectItem value="pro">Pro</SelectItem>
+ </SelectContent>
+ </Select>
+ </Field>
+ <Field label="Has paid" htmlFor="aud-paid">
+ <Select value={draft.audience.paid} onValueChange={(v) => setAudience({ paid: v as Draft["audience"]["paid"] })}>
+ <SelectTrigger id="aud-paid">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="any">Any</SelectItem>
+ <SelectItem value="yes">Yes</SelectItem>
+ <SelectItem value="no">No</SelectItem>
+ </SelectContent>
+ </Select>
+ </Field>
+ <Field label="Active within days" htmlFor="aud-active">
+ <Input
+ id="aud-active"
+ type="number"
+ min={1}
+ max={365}
+ value={draft.audience.activeWithinDays}
+ onChange={(e) => setAudience({ activeWithinDays: e.target.value })}
+ />
+ </Field>
+ <Field label="Storage used ≥ %" htmlFor="aud-storage">
+ <Input
+ id="aud-storage"
+ type="number"
+ min={0}
+ max={100}
+ value={draft.audience.storageUsedPercentAtLeast}
+ onChange={(e) => setAudience({ storageUsedPercentAtLeast: e.target.value })}
+ />
+ </Field>
+ <Field label="Credits spent ≥ %" htmlFor="aud-credits">
+ <Input
+ id="aud-credits"
+ type="number"
+ min={0}
+ max={100}
+ value={draft.audience.creditsUsedPercentAtLeast}
+ onChange={(e) => setAudience({ creditsUsedPercentAtLeast: e.target.value })}
+ />
+ </Field>
+ <Field label="Percent enrolled" htmlFor="exp-percent">
+ <Input
+ id="exp-percent"
+ type="number"
+ min={0}
+ max={100}
+ value={draft.percent}
+ onChange={(e) => setDraft({ ...draft, percent: e.target.value })}
+ />
+ </Field>
+ </div>
+ </div>
+
+ <div className="space-y-3">
+ <div className="flex items-center justify-between">
+ <Label>Variants</Label>
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ onClick={() =>
+ setDraft({
+ ...draft,
+ variants: [...draft.variants, { key: "", name: "", weight: "50", config: {} }],
+ })
+ }
+ >
+ <Plus /> Add variant
+ </Button>
+ </div>
+ {draft.variants.map((variant, i) => (
+ <div key={i} className="space-y-3 rounded-lg border p-3">
+ <div className="grid grid-cols-[1fr_1fr_100px_auto] items-end gap-2">
+ <Field label={i === 0 ? "Key (control)" : "Key"} htmlFor={`v-${i}-key`}>
+ <Input
+ id={`v-${i}-key`}
+ disabled={assignedKeys.has(variant.key)}
+ value={variant.key}
+ onChange={(e) => setVariant(i, { key: e.target.value })}
+ />
+ </Field>
+ <Field label="Name" htmlFor={`v-${i}-name`}>
+ <Input id={`v-${i}-name`} value={variant.name} onChange={(e) => setVariant(i, { name: e.target.value })} />
+ </Field>
+ <Field label="Weight" htmlFor={`v-${i}-weight`}>
+ <Input
+ id={`v-${i}-weight`}
+ type="number"
+ min={0}
+ value={variant.weight}
+ onChange={(e) => setVariant(i, { weight: e.target.value })}
+ />
+ </Field>
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon"
+ aria-label="Remove variant"
+ disabled={assignedKeys.has(variant.key) || draft.variants.length <= 1}
+ onClick={() =>
+ setDraft({ ...draft, variants: draft.variants.filter((_, j) => j !== i) })
+ }
+ >
+ <X />
+ </Button>
+ </div>
+ {PUBLIC_SETTING_KEYS.length === 0 ? (
+ <p className="text-xs text-muted-foreground">
+ No public setting is registered yet; the app reads the variant key itself.
+ </p>
+ ) : null}
+ <div className="grid gap-2">
+ {PUBLIC_SETTING_KEYS.map((key: SettingKey) => {
+ const carries = key in variant.config;
+ return (
+ <div key={key} className="space-y-2">
+ <div className="flex items-center gap-2">
+ <Switch
+ checked={carries}
+ onCheckedChange={(on) => {
+ const config = { ...variant.config };
+ if (on) config[key] = structuredClone(SETTINGS[key].default);
+ else delete config[key];
+ setVariant(i, { config });
+ }}
+ />
+ <span className="text-sm">{SETTINGS[key].title}</span>
+ <span className="text-xs text-muted-foreground">{key}</span>
+ </div>
+ {carries && schemas.get(key) ? (
+ <div className="pl-10">
+ <SchemaField
+ id={`v-${i}-${key}`}
+ schema={schemas.get(key)!}
+ value={variant.config[key]}
+ onChange={(next) =>
+ setVariant(i, { config: { ...variant.config, [key]: next } })
+ }
+ />
+ </div>
+ ) : null}
+ </div>
+ );
+ })}
+ </div>
+ </div>
+ ))}
+ </div>
+
+ <div className="space-y-3">
+ <div className="flex items-center justify-between">
+ <div>
+ <Label>Metrics</Label>
+ <p className="text-xs text-muted-foreground">
+ Counted per exposed account, after its exposure. The first decides the verdict.
+ </p>
+ </div>
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ onClick={() =>
+ setDraft({
+ ...draft,
+ metrics: [...draft.metrics, { key: "", name: "", source: "event", event: "" }],
+ })
+ }
+ >
+ <Plus /> Add metric
+ </Button>
+ </div>
+ {draft.metrics.map((metric, i) => (
+ <div key={i} className="grid grid-cols-[1fr_1fr_140px_1fr_auto] items-end gap-2">
+ <Field label={i === 0 ? "Key (primary)" : "Key"} htmlFor={`m-${i}-key`}>
+ <Input id={`m-${i}-key`} value={metric.key} onChange={(e) => setMetric(i, { key: e.target.value })} />
+ </Field>
+ <Field label="Name" htmlFor={`m-${i}-name`}>
+ <Input id={`m-${i}-name`} value={metric.name} onChange={(e) => setMetric(i, { name: e.target.value })} />
+ </Field>
+ <Field label="Source" htmlFor={`m-${i}-source`}>
+ <Select
+ value={metric.source}
+ onValueChange={(v) => setMetric(i, { source: v as Draft["metrics"][number]["source"] })}
+ >
+ <SelectTrigger id={`m-${i}-source`}>
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="event">PostHog event</SelectItem>
+ <SelectItem value="purchase">Purchase</SelectItem>
+ </SelectContent>
+ </Select>
+ </Field>
+ <Field label="Event" htmlFor={`m-${i}-event`}>
+ <Input
+ id={`m-${i}-event`}
+ disabled={metric.source !== "event"}
+ placeholder="export_completed"
+ value={metric.event}
+ onChange={(e) => setMetric(i, { event: e.target.value })}
+ />
+ </Field>
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon"
+ aria-label="Remove metric"
+ onClick={() => setDraft({ ...draft, metrics: draft.metrics.filter((_, j) => j !== i) })}
+ >
+ <X />
+ </Button>
+ </div>
+ ))}
+ </div>
+
+ {issues.length > 0 ? (
+ <ul className="space-y-1 text-sm text-destructive">
+ {issues.map((issue) => (
+ <li key={issue}>{issue}</li>
+ ))}
+ </ul>
+ ) : null}
+ </div>
+
+ <DialogFooter>
+ <Button variant="outline" onClick={() => onOpenChange(false)}>
+ Cancel
+ </Button>
+ <Button disabled={pending} onClick={submit}>
+ {pending ? "Saving…" : existing ? "Save" : "Create draft"}
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+ );
+}
+
+function Field({ label, htmlFor, children }: { label: string; htmlFor: string; children: React.ReactNode }) {
+ return (
+ <div className="space-y-1.5">
+ <Label htmlFor={htmlFor} className="text-xs text-muted-foreground">
+ {label}
+ </Label>
+ {children}
+ </div>
+ );
+}
diff --git a/site/src/app/su/experiments/SchemaField.tsx b/site/src/app/su/experiments/SchemaField.tsx
new file mode 100644
index 00000000..c94c6068
--- /dev/null
+++ b/site/src/app/su/experiments/SchemaField.tsx
@@ -0,0 +1,187 @@
+"use client";
+
+import { useState } from "react";
+
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
+
+// A value editor drawn from a setting's JSON schema (the registry's zod schema,
+// converted on the server). One level of object nests as a fieldset; anything
+// deeper or stranger falls back to JSON text, which the route still validates.
+
+export type JsonSchema = {
+ type?: string | string[];
+ enum?: unknown[];
+ anyOf?: JsonSchema[];
+ properties?: Record<string, JsonSchema>;
+ required?: string[];
+ items?: JsonSchema;
+ minimum?: number;
+ maximum?: number;
+ description?: string;
+};
+
+type Props = {
+ id: string;
+ schema: JsonSchema;
+ value: unknown;
+ onChange: (value: unknown) => void;
+ depth?: number;
+};
+
+// A nullable scalar arrives as anyOf [scalar, null]; the form shows the
+// scalar's control with "off" standing for null.
+function nullable(schema: JsonSchema): { inner: JsonSchema; nullable: boolean } {
+ if (!schema.anyOf) return { inner: schema, nullable: false };
+ const inner = schema.anyOf.find((s) => s.type !== "null");
+ const hasNull = schema.anyOf.some((s) => s.type === "null");
+ return inner && hasNull && schema.anyOf.length === 2
+ ? { inner, nullable: true }
+ : { inner: schema, nullable: false };
+}
+
+export function SchemaField({ id, schema: raw, value, onChange, depth = 0 }: Props) {
+ const { inner, nullable: isNullable } = nullable(raw);
+ const type = Array.isArray(inner.type) ? inner.type[0] : inner.type;
+
+ if (inner.enum) {
+ return (
+ <Select value={String(value)} onValueChange={(v) => onChange(v)}>
+ <SelectTrigger id={id} className="w-48">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {inner.enum.map((option) => (
+ <SelectItem key={String(option)} value={String(option)}>
+ {String(option)}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ );
+ }
+
+ if (type === "boolean") {
+ return <Switch id={id} checked={value === true} onCheckedChange={(checked) => onChange(checked)} />;
+ }
+
+ if (type === "number" || type === "integer") {
+ return (
+ <div className="flex items-center gap-2">
+ {isNullable ? (
+ <Switch
+ aria-label="Rule on"
+ checked={value !== null}
+ onCheckedChange={(on) => onChange(on ? (inner.minimum ?? 0) : null)}
+ />
+ ) : null}
+ <Input
+ id={id}
+ className="w-28"
+ type="number"
+ disabled={isNullable && value === null}
+ min={inner.minimum}
+ max={inner.maximum}
+ step={type === "integer" ? 1 : "any"}
+ value={value === null || value === undefined ? "" : String(value)}
+ onChange={(event) => {
+ const next = event.target.value;
+ onChange(next === "" ? (isNullable ? null : 0) : Number(next));
+ }}
+ />
+ {isNullable && value === null ? (
+ <span className="text-xs text-muted-foreground">off</span>
+ ) : null}
+ </div>
+ );
+ }
+
+ if (type === "string") {
+ return (
+ <Input
+ id={id}
+ className="max-w-md"
+ value={typeof value === "string" ? value : ""}
+ onChange={(event) => onChange(event.target.value)}
+ />
+ );
+ }
+
+ if (type === "array" && inner.items && inner.items.type === "string") {
+ const list = Array.isArray(value) ? (value as string[]) : [];
+ return (
+ <Input
+ id={id}
+ className="max-w-md"
+ placeholder="comma separated"
+ value={list.join(", ")}
+ onChange={(event) =>
+ onChange(
+ event.target.value
+ .split(",")
+ .map((s) => s.trim())
+ .filter(Boolean),
+ )
+ }
+ />
+ );
+ }
+
+ if (type === "object" && inner.properties && depth === 0) {
+ const record = (value && typeof value === "object" ? value : {}) as Record<string, unknown>;
+ return (
+ <div className="grid gap-3">
+ {Object.entries(inner.properties).map(([name, property]) => (
+ <div key={name} className="grid grid-cols-[180px_1fr] items-center gap-3">
+ <Label htmlFor={`${id}-${name}`} className="text-muted-foreground">
+ {name}
+ </Label>
+ <SchemaField
+ id={`${id}-${name}`}
+ schema={property}
+ value={record[name]}
+ onChange={(next) => onChange({ ...record, [name]: next })}
+ depth={depth + 1}
+ />
+ </div>
+ ))}
+ </div>
+ );
+ }
+
+ return <JsonField id={id} value={value} onChange={onChange} />;
+}
+
+function JsonField({ id, value, onChange }: { id: string; value: unknown; onChange: (v: unknown) => void }) {
+ const [text, setText] = useState(() => JSON.stringify(value, null, 2));
+ const [bad, setBad] = useState(false);
+ return (
+ <div className="space-y-1">
+ <Textarea
+ id={id}
+ className="font-mono text-xs"
+ rows={6}
+ value={text}
+ onChange={(event) => {
+ setText(event.target.value);
+ try {
+ onChange(JSON.parse(event.target.value));
+ setBad(false);
+ } catch {
+ setBad(true);
+ }
+ }}
+ />
+ {bad ? <p className="text-xs text-destructive">Not valid JSON.</p> : null}
+ </div>
+ );
+}
diff --git a/site/src/app/su/experiments/list/page.tsx b/site/src/app/su/experiments/list/page.tsx
new file mode 100644
index 00000000..c6e338a9
--- /dev/null
+++ b/site/src/app/su/experiments/list/page.tsx
@@ -0,0 +1,399 @@
+"use client";
+
+import { useState } from "react";
+
+import { ExperimentDialog } from "@/app/su/experiments/ExperimentDialog";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import { Skeleton } from "@/components/ui/skeleton";
+import { describeAudience } from "@donkeycut/abexp";
+import type { ExperimentStatus } from "@/lib/config/experiment";
+import type { MetricResult, VerdictState } from "@donkeycut/abexp";
+import {
+ useAddHoldout,
+ useComputeResults,
+ useDeleteExperiment,
+ useExperiments,
+ useHoldout,
+ useRemoveAssignment,
+ useRemoveHoldout,
+ useSetAssignment,
+ useSetExperimentStatus,
+ type ExperimentSummary,
+} from "@/queries/experiments";
+
+type BadgeVariant = "default" | "secondary" | "outline" | "destructive";
+
+const STATUS_VARIANT: Record<ExperimentStatus, BadgeVariant> = {
+ draft: "outline",
+ running: "default",
+ paused: "secondary",
+ ended: "destructive",
+};
+
+const VERDICT: Record<VerdictState, { label: string; variant: BadgeVariant }> = {
+ insufficient: { label: "Too early", variant: "outline" },
+ keep_running: { label: "Keep running", variant: "secondary" },
+ ship: { label: "Ship it", variant: "default" },
+ stop: { label: "Stop it", variant: "destructive" },
+};
+
+
+const HOLD_OUT_KEY = "__hold_out__";
+
+const pct = (x: number) => `${(100 * x).toFixed(1)}%`;
+const when = (iso: string) => new Date(iso).toLocaleString();
+
+export default function SuExperimentsListPage() {
+ const experiments = useExperiments();
+ const [editing, setEditing] = useState<ExperimentSummary | null>(null);
+ const [dialogOpen, setDialogOpen] = useState(false);
+ // The dialog mounts fresh per open so its draft starts from the row.
+ const [dialogKey, setDialogKey] = useState(0);
+ const open = (e: ExperimentSummary | null) => {
+ setEditing(e);
+ setDialogKey((k) => k + 1);
+ setDialogOpen(true);
+ };
+
+ if (!experiments.data) return <Skeleton className="h-40 w-full" />;
+
+ return (
+ <div className="space-y-6 pb-9">
+ <div className="flex justify-end">
+ <Button onClick={() => open(null)}>New experiment</Button>
+ </div>
+ {experiments.data.experiments.length === 0 ? (
+ <p className="text-sm text-muted-foreground">No experiments yet.</p>
+ ) : null}
+ {experiments.data.experiments.map((e) => (
+ <ExperimentRow key={e.id} experiment={e} onEdit={() => open(e)} />
+ ))}
+ <Holdout />
+ {dialogOpen ? (
+ <ExperimentDialog key={dialogKey} existing={editing} open={dialogOpen} onOpenChange={setDialogOpen} />
+ ) : null}
+ </div>
+ );
+}
+
+function ExperimentRow({ experiment: e, onEdit }: { experiment: ExperimentSummary; onEdit: () => void }) {
+ const setStatus = useSetExperimentStatus();
+ const remove = useDeleteExperiment();
+ const compute = useComputeResults();
+ const status = e.status as ExperimentStatus;
+ const go = (next: ExperimentStatus) => setStatus.mutate({ id: e.id, status: next });
+ const busy = setStatus.isPending || remove.isPending;
+
+ return (
+ <div className="rounded-lg border p-4">
+ <div className="flex items-start justify-between gap-4">
+ <div className="min-w-0">
+ <div className="flex items-center gap-2">
+ <span className="font-medium">{e.name}</span>
+ <span className="font-mono text-xs text-muted-foreground">{e.key}</span>
+ <Badge variant={STATUS_VARIANT[status]}>{status}</Badge>
+ </div>
+ {e.description ? <p className="mt-1 text-sm text-muted-foreground">{e.description}</p> : null}
+ <p className="mt-1 text-xs text-muted-foreground">
+ {describeAudience(e.audience)} · {e.percent}% enrolled
+ </p>
+ </div>
+ <div className="flex shrink-0 items-center gap-2">
+ {status === "draft" ? (
+ <Button size="sm" disabled={busy} onClick={() => go("running")}>
+ Start
+ </Button>
+ ) : null}
+ {status === "running" ? (
+ <Button size="sm" variant="outline" disabled={busy} onClick={() => go("paused")}>
+ Pause
+ </Button>
+ ) : null}
+ {status === "paused" ? (
+ <Button size="sm" disabled={busy} onClick={() => go("running")}>
+ Resume
+ </Button>
+ ) : null}
+ {status === "running" || status === "paused" ? (
+ <Button size="sm" variant="destructive" disabled={busy} onClick={() => go("ended")}>
+ End
+ </Button>
+ ) : null}
+ {status !== "ended" ? (
+ <Button size="sm" variant="ghost" onClick={onEdit}>
+ Edit
+ </Button>
+ ) : null}
+ {status === "draft" ? (
+ <Button size="sm" variant="ghost" disabled={busy} onClick={() => remove.mutate(e.id)}>
+ Delete
+ </Button>
+ ) : null}
+ </div>
+ </div>
+
+ <table className="mt-3 w-full text-sm">
+ <thead className="text-left text-xs text-muted-foreground">
+ <tr>
+ <th className="py-1 font-normal">Variant</th>
+ <th className="py-1 font-normal">Weight</th>
+ <th className="py-1 font-normal">Changes</th>
+ <th className="py-1 text-right font-normal">Assigned</th>
+ <th className="py-1 text-right font-normal">Exposed</th>
+ </tr>
+ </thead>
+ <tbody>
+ {e.variants.map((v, i) => {
+ const stat = e.stats.find((s) => s.key === v.key);
+ return (
+ <tr key={v.key} className="border-t">
+ <td className="py-1.5">
+ {v.name} <span className="font-mono text-xs text-muted-foreground">{v.key}</span>
+ {i === 0 ? <span className="ml-1 text-xs text-muted-foreground">control</span> : null}
+ </td>
+ <td className="py-1.5 tabular-nums">{v.weight}</td>
+ <td className="py-1.5 text-xs text-muted-foreground">
+ {Object.keys(v.config).join(", ") || "nothing"}
+ </td>
+ <td className="py-1.5 text-right tabular-nums">{stat?.assigned ?? 0}</td>
+ <td className="py-1.5 text-right tabular-nums">{stat?.exposed ?? 0}</td>
+ </tr>
+ );
+ })}
+ </tbody>
+ </table>
+
+ {status !== "draft" ? (
+ <div className="mt-4 space-y-2">
+ <div className="flex items-center justify-between">
+ <div className="flex items-center gap-2">
+ <span className="text-sm font-medium">Results</span>
+ {e.results ? (
+ <Badge variant={VERDICT[e.results.verdict.state].variant}>
+ {VERDICT[e.results.verdict.state].label}
+ </Badge>
+ ) : null}
+ <span className="text-xs text-muted-foreground">
+ {e.resultsAt ? `computed ${when(e.resultsAt)}` : "not computed yet; the nightly run will"}
+ </span>
+ </div>
+ <Button size="sm" variant="outline" disabled={compute.isPending} onClick={() => compute.mutate(e.id)}>
+ {compute.isPending ? "Computing…" : "Compute now"}
+ </Button>
+ </div>
+ {compute.isError ? <p className="text-sm text-destructive">{compute.error.message}</p> : null}
+ {e.results ? (
+ <>
+ <p className="text-sm">
+ {e.results.verdict.reason}
+ {e.results.verdict.neededPerArm !== null && e.results.verdict.state !== "ship" ? (
+ <span className="text-muted-foreground">
+ {" "}
+ About {e.results.verdict.neededPerArm.toLocaleString()} exposed per arm reads a{" "}
+ {Math.round(100 * plannedLift(e.results.metrics[0]))}% lift.
+ </span>
+ ) : null}
+ </p>
+ {e.results.metrics.map((m) => (
+ <MetricTable key={m.key} metric={m} control={e.results!.control} />
+ ))}
+ {e.results.notes.map((note) => (
+ <p key={note} className="text-xs text-muted-foreground">
+ {note}
+ </p>
+ ))}
+ </>
+ ) : null}
+ </div>
+ ) : null}
+
+ <Overrides experiment={e} />
+ </div>
+ );
+}
+
+// The lift the sample-size estimate planned for: the larger of the standard
+// one and what the arms have shown so far.
+function plannedLift(metric: MetricResult | undefined): number {
+ const observed = metric?.comparisons.map((c) => Math.abs(c.relativeLift ?? 0)) ?? [];
+ return Math.max(0.2, ...observed);
+}
+
+function MetricTable({ metric, control }: { metric: MetricResult; control: string }) {
+ return (
+ <table className="w-full text-sm">
+ <thead className="text-left text-xs text-muted-foreground">
+ <tr>
+ <th className="py-1 font-normal">
+ {metric.name}
+ {metric.event ? <span className="ml-1 font-mono">{metric.event}</span> : null}
+ </th>
+ <th className="py-1 text-right font-normal">Exposed</th>
+ <th className="py-1 text-right font-normal">Converted</th>
+ <th className="py-1 text-right font-normal">Rate</th>
+ <th className="py-1 text-right font-normal">Lift</th>
+ <th className="py-1 text-right font-normal">p</th>
+ <th className="py-1 text-right font-normal">Beats control</th>
+ </tr>
+ </thead>
+ <tbody>
+ {metric.variants.map((v) => {
+ const cmp = metric.comparisons.find((c) => c.variant === v.key);
+ return (
+ <tr key={v.key} className="border-t">
+ <td className="py-1 font-mono text-xs">{v.key}</td>
+ <td className="py-1 text-right tabular-nums">{v.exposed}</td>
+ <td className="py-1 text-right tabular-nums">{v.converted}</td>
+ <td className="py-1 text-right tabular-nums">{pct(v.rate)}</td>
+ <td className="py-1 text-right tabular-nums">
+ {v.key === control ? "—" : cmp?.relativeLift === null || cmp === undefined ? "n/a" : pct(cmp.relativeLift)}
+ </td>
+ <td className="py-1 text-right tabular-nums">
+ {v.key === control ? "—" : cmp?.pValue == null ? "n/a" : cmp.pValue.toFixed(3)}
+ </td>
+ <td className="py-1 text-right tabular-nums">
+ {v.key === control || !cmp ? "—" : pct(cmp.probabilityBeatsControl)}
+ </td>
+ </tr>
+ );
+ })}
+ </tbody>
+ </table>
+ );
+}
+
+// Assignments written by hand: an account put into a variant, or held out of
+// this one experiment.
+function Overrides({ experiment: e }: { experiment: ExperimentSummary }) {
+ const set = useSetAssignment();
+ const remove = useRemoveAssignment();
+ const [email, setEmail] = useState("");
+ const [variant, setVariant] = useState<string>(e.variants[0]?.key ?? HOLD_OUT_KEY);
+ const submit = () => {
+ if (!email.trim()) return;
+ set.mutate(
+ { id: e.id, email: email.trim(), variant: variant === HOLD_OUT_KEY ? null : variant },
+ { onSuccess: () => setEmail("") },
+ );
+ };
+ return (
+ <div className="mt-4 space-y-2">
+ <span className="text-sm font-medium">Overrides</span>
+ {e.overrides.length > 0 ? (
+ <ul className="space-y-1 text-sm">
+ {e.overrides.map((o) => (
+ <li key={o.userId} className="flex items-center gap-2">
+ <span>{o.email}</span>
+ <span className="font-mono text-xs text-muted-foreground">{o.variant ?? "held out"}</span>
+ <Button
+ size="sm"
+ variant="ghost"
+ disabled={remove.isPending}
+ onClick={() => remove.mutate({ id: e.id, userId: o.userId })}
+ >
+ Remove
+ </Button>
+ </li>
+ ))}
+ </ul>
+ ) : null}
+ <div className="flex items-center gap-2">
+ <Input
+ className="max-w-xs"
+ placeholder="account email"
+ value={email}
+ onChange={(ev) => setEmail(ev.target.value)}
+ onKeyDown={(ev) => {
+ if (ev.key === "Enter") submit();
+ }}
+ />
+ <Select value={variant} onValueChange={(v) => setVariant(v ?? HOLD_OUT_KEY)}>
+ <SelectTrigger className="w-44">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {e.variants.map((v) => (
+ <SelectItem key={v.key} value={v.key}>
+ {v.name}
+ </SelectItem>
+ ))}
+ <SelectItem value={HOLD_OUT_KEY}>Hold out</SelectItem>
+ </SelectContent>
+ </Select>
+ <Button size="sm" variant="outline" disabled={set.isPending || !email.trim()} onClick={submit}>
+ Set
+ </Button>
+ </div>
+ {set.isError ? <p className="text-sm text-destructive">{set.error.message}</p> : null}
+ </div>
+ );
+}
+
+// Accounts kept out of every experiment.
+function Holdout() {
+ const holdout = useHoldout();
+ const add = useAddHoldout();
+ const remove = useRemoveHoldout();
+ const [email, setEmail] = useState("");
+ const [note, setNote] = useState("");
+ const submit = () => {
+ if (!email.trim()) return;
+ add.mutate(
+ { email: email.trim(), note: note.trim() || null },
+ {
+ onSuccess: () => {
+ setEmail("");
+ setNote("");
+ },
+ },
+ );
+ };
+ return (
+ <div className="rounded-lg border p-4">
+ <div className="space-y-1">
+ <span className="font-medium">Holdout</span>
+ <p className="text-sm text-muted-foreground">
+ Accounts kept out of every experiment: they are never assigned, read the plain
+ configuration, and never count in a result.
+ </p>
+ </div>
+ {holdout.data && holdout.data.holdout.length > 0 ? (
+ <ul className="mt-3 space-y-1 text-sm">
+ {holdout.data.holdout.map((h) => (
+ <li key={h.userId} className="flex items-center gap-2">
+ <span>{h.email}</span>
+ {h.note ? <span className="text-xs text-muted-foreground">{h.note}</span> : null}
+ <Button size="sm" variant="ghost" disabled={remove.isPending} onClick={() => remove.mutate(h.userId)}>
+ Remove
+ </Button>
+ </li>
+ ))}
+ </ul>
+ ) : null}
+ <div className="mt-3 flex items-center gap-2">
+ <Input
+ className="max-w-xs"
+ placeholder="account email"
+ value={email}
+ onChange={(ev) => setEmail(ev.target.value)}
+ />
+ <Input
+ className="max-w-xs"
+ placeholder="note"
+ value={note}
+ onChange={(ev) => setNote(ev.target.value)}
+ onKeyDown={(ev) => {
+ if (ev.key === "Enter") submit();
+ }}
+ />
+ <Button size="sm" variant="outline" disabled={add.isPending || !email.trim()} onClick={submit}>
+ Hold out
+ </Button>
+ </div>
+ {add.isError ? <p className="mt-2 text-sm text-destructive">{add.error.message}</p> : null}
+ </div>
+ );
+}
diff --git a/site/src/app/su/experiments/page.tsx b/site/src/app/su/experiments/page.tsx
new file mode 100644
index 00000000..478a85b6
--- /dev/null
+++ b/site/src/app/su/experiments/page.tsx
@@ -0,0 +1,9 @@
+import { redirect } from "next/navigation";
+
+import { SU_NAV } from "@/app/su/nav";
+
+// The section has no surface of its own; it opens its first tab.
+export default function SuExperimentsPage() {
+ const section = SU_NAV.find((s) => s.href === "/experiments")!;
+ redirect(section.tabs![0].href);
+}
diff --git a/site/src/app/su/experiments/settings/page.tsx b/site/src/app/su/experiments/settings/page.tsx
new file mode 100644
index 00000000..93ec653e
--- /dev/null
+++ b/site/src/app/su/experiments/settings/page.tsx
@@ -0,0 +1,102 @@
+"use client";
+
+import { useState } from "react";
+
+import { SchemaField, type JsonSchema } from "@/app/su/experiments/SchemaField";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import {
+ Card,
+ CardContent,
+ CardDescription,
+ CardHeader,
+ CardTitle,
+} from "@/components/ui/card";
+import { Skeleton } from "@/components/ui/skeleton";
+import { useResetSetting, useSaveSetting, useSettings, type SettingRow } from "@/queries/settings";
+
+// Every registered setting, editable. The value on screen is what the product
+// reads for everyone outside an experiment; Reset returns it to the default
+// declared in code.
+export default function SuSettingsPage() {
+ const settings = useSettings();
+ if (!settings.data) {
+ return <Skeleton className="h-40 w-full max-w-2xl" />;
+ }
+ if (settings.data.settings.length === 0) {
+ return (
+ <p className="max-w-2xl text-sm text-muted-foreground">
+ No setting is registered yet. A feature declares one in the settings registry and it
+ appears here, with a form drawn from its schema.
+ </p>
+ );
+ }
+ return (
+ <div className="max-w-2xl space-y-6 pb-9">
+ {settings.data.settings.map((row) => (
+ <SettingCard key={row.key} row={row} />
+ ))}
+ </div>
+ );
+}
+
+function SettingCard({ row }: { row: SettingRow }) {
+ const save = useSaveSetting();
+ const reset = useResetSetting();
+ const [draft, setDraft] = useState<unknown>(row.value);
+ const [savedFrom, setSavedFrom] = useState(() => JSON.stringify(row.value));
+ // A save or reset that lands replaces the row; the draft follows it. The
+ // comparison is by content: a refetch hands back an equal object under a
+ // new reference, and that must not throw away what is being typed.
+ const incoming = JSON.stringify(row.value);
+ if (savedFrom !== incoming) {
+ setSavedFrom(incoming);
+ setDraft(row.value);
+ }
+ const dirty = JSON.stringify(draft) !== JSON.stringify(row.value);
+
+ return (
+ <Card>
+ <CardHeader>
+ <CardTitle className="flex items-center gap-2">
+ {row.title}
+ <Badge variant="outline">{row.public ? "public" : "server"}</Badge>
+ {row.overridden ? <Badge variant="secondary">overridden</Badge> : null}
+ {row.invalid ? <Badge variant="destructive">invalid, reset</Badge> : null}
+ </CardTitle>
+ <CardDescription>{row.description}</CardDescription>
+ </CardHeader>
+ <CardContent className="space-y-4">
+ <SchemaField
+ id={`setting-${row.key}`}
+ schema={row.schema as JsonSchema}
+ value={draft}
+ onChange={setDraft}
+ />
+ <div className="flex items-center gap-2">
+ <Button
+ disabled={!dirty || save.isPending}
+ onClick={() => save.mutate({ key: row.key, value: draft })}
+ >
+ {save.isPending ? "Saving…" : "Save"}
+ </Button>
+ <Button
+ variant="outline"
+ disabled={!row.overridden || reset.isPending}
+ onClick={() => reset.mutate(row.key)}
+ >
+ Reset to default
+ </Button>
+ {row.updatedAt ? (
+ <span className="text-xs text-muted-foreground">
+ changed {new Date(row.updatedAt).toLocaleString()}
+ </span>
+ ) : null}
+ </div>
+ {save.isError ? (
+ <p className="text-sm text-destructive">The value was rejected. Check each field.</p>
+ ) : null}
+ </CardContent>
+ </Card>
+ );
+}
diff --git a/site/src/app/su/nav.ts b/site/src/app/su/nav.ts
index 8a0cfcd9..67a08a36 100644
--- a/site/src/app/su/nav.ts
+++ b/site/src/app/su/nav.ts
@@ -1,4 +1,4 @@
-import { ChartColumn, CreditCard, ListChecks, Mail, UserRound } from "lucide-react";
+import { ChartColumn, CreditCard, FlaskConical, ListChecks, Mail, UserRound } from "lucide-react";
import type { ComponentType } from "react";
import { RunAnalyticsButton } from "@/app/su/analytics/RunAnalyticsButton";
@@ -58,6 +58,24 @@ export const SU_NAV: SuSurface[] = [
title: "Outreach",
Action: ScanOutreachButton,
},
+ {
+ href: "/experiments",
+ label: "Experiments",
+ icon: FlaskConical,
+ title: "Experiments",
+ tabs: [
+ {
+ href: "/experiments/list",
+ label: "Experiments",
+ description: "Variants over settings, assigned once per account and kept.",
+ },
+ {
+ href: "/experiments/settings",
+ label: "Settings",
+ description: "Every runtime setting, with its default from code and its override here.",
+ },
+ ],
+ },
{
href: "/jobs",
label: "Jobs",
diff --git a/site/src/queries/experiments.ts b/site/src/queries/experiments.ts
new file mode 100644
index 00000000..3818b8a4
--- /dev/null
+++ b/site/src/queries/experiments.ts
@@ -0,0 +1,138 @@
+"use client";
+
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+
+import type { ExperimentSummary, HoldoutRow } from "@/lib/config/experimentList";
+import type { ExperimentInput, ExperimentStatus } from "@/lib/config/experiment";
+import { apiFetch } from "@/queries/apiClient";
+
+export type { ExperimentSummary, HoldoutRow };
+
+export const experimentsQueryKey = ["su", "experiments"] as const;
+
+type Body = { experiments: ExperimentSummary[] };
+
+// Super-user only: every experiment with per-variant counts.
+export function useExperiments() {
+ return useQuery({
+ queryFn: () => apiFetch<Body>("/api/experiments"),
+ queryKey: experimentsQueryKey,
+ });
+}
+
+export function useCreateExperiment() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: (input: ExperimentInput) =>
+ apiFetch<Body>("/api/experiments", { body: JSON.stringify(input), method: "POST" }),
+ onSuccess: (body) => queryClient.setQueryData(experimentsQueryKey, body),
+ });
+}
+
+export function useUpdateExperiment() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: ({ id, ...input }: ExperimentInput & { id: string }) =>
+ apiFetch<Body>(`/api/experiments/${encodeURIComponent(id)}`, {
+ body: JSON.stringify(input),
+ method: "PUT",
+ }),
+ onSuccess: (body) => queryClient.setQueryData(experimentsQueryKey, body),
+ });
+}
+
+export function useSetExperimentStatus() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: ({ id, status }: { id: string; status: ExperimentStatus }) =>
+ apiFetch<Body>(`/api/experiments/${encodeURIComponent(id)}`, {
+ body: JSON.stringify({ status }),
+ method: "PATCH",
+ }),
+ onSuccess: (body) => queryClient.setQueryData(experimentsQueryKey, body),
+ });
+}
+
+export function useDeleteExperiment() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: (id: string) =>
+ apiFetch<Body>(`/api/experiments/${encodeURIComponent(id)}`, { method: "DELETE" }),
+ onSuccess: (body) => queryClient.setQueryData(experimentsQueryKey, body),
+ });
+}
+
+// A row written by hand: the account reads this variant (null holds it out).
+export function useSetAssignment() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: ({ id, email, variant }: { id: string; email: string; variant: string | null }) =>
+ apiFetch<Body>(`/api/experiments/${encodeURIComponent(id)}/assignments`, {
+ body: JSON.stringify({ email, variant }),
+ method: "PUT",
+ }),
+ onSuccess: (body) => queryClient.setQueryData(experimentsQueryKey, body),
+ });
+}
+
+export function useRemoveAssignment() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: ({ id, userId }: { id: string; userId: string }) =>
+ apiFetch<Body>(`/api/experiments/${encodeURIComponent(id)}/assignments`, {
+ body: JSON.stringify({ userId }),
+ method: "DELETE",
+ }),
+ onSuccess: (body) => queryClient.setQueryData(experimentsQueryKey, body),
+ });
+}
+
+// Runs the results job for one experiment and follows it to completion, then
+// refetches the list so the new read shows.
+export function useComputeResults() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: async (experimentId: string) => {
+ const { jobId } = await apiFetch<{ jobId: string }>("/api/jobs", {
+ body: JSON.stringify({ kind: "experiment-results", payload: { experimentId } }),
+ method: "POST",
+ });
+ for (;;) {
+ await new Promise((resolve) => setTimeout(resolve, 2000));
+ const job = await apiFetch<{ state: string; error?: string }>(`/api/jobs/${jobId}`);
+ if (job.state === "done") return job;
+ if (job.state === "error") throw new Error(job.error ?? "The run failed.");
+ }
+ },
+ onSettled: () => queryClient.invalidateQueries({ queryKey: experimentsQueryKey }),
+ });
+}
+
+export const holdoutQueryKey = ["su", "holdout"] as const;
+
+type HoldoutBody = { holdout: HoldoutRow[] };
+
+export function useHoldout() {
+ return useQuery({
+ queryFn: () => apiFetch<HoldoutBody>("/api/experiments/holdout"),
+ queryKey: holdoutQueryKey,
+ });
+}
+
+export function useAddHoldout() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: (input: { email: string; note: string | null }) =>
+ apiFetch<HoldoutBody>("/api/experiments/holdout", { body: JSON.stringify(input), method: "POST" }),
+ onSuccess: (body) => queryClient.setQueryData(holdoutQueryKey, body),
+ });
+}
+
+export function useRemoveHoldout() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: (userId: string) =>
+ apiFetch<HoldoutBody>("/api/experiments/holdout", { body: JSON.stringify({ userId }), method: "DELETE" }),
+ onSuccess: (body) => queryClient.setQueryData(holdoutQueryKey, body),
+ });
+}
diff --git a/site/src/queries/settings.ts b/site/src/queries/settings.ts
new file mode 100644
index 00000000..98d1c5b0
--- /dev/null
+++ b/site/src/queries/settings.ts
@@ -0,0 +1,49 @@
+"use client";
+
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+
+import type { SettingKey } from "@/lib/config/registry";
+import { apiFetch } from "@/queries/apiClient";
+
+export type SettingRow = {
+ key: SettingKey;
+ title: string;
+ description: string;
+ public: boolean;
+ default: unknown;
+ value: unknown;
+ overridden: boolean;
+ invalid: boolean;
+ updatedAt: string | null;
+ schema: Record<string, unknown>;
+};
+
+export const settingsQueryKey = ["su", "settings"] as const;
+
+type Body = { settings: SettingRow[] };
+
+// Super-user only: every registered setting with its current override.
+export function useSettings() {
+ return useQuery({
+ queryFn: () => apiFetch<Body>("/api/settings"),
+ queryKey: settingsQueryKey,
+ });
+}
+
+export function useSaveSetting() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: (input: { key: SettingKey; value: unknown }) =>
+ apiFetch<Body>("/api/settings", { body: JSON.stringify(input), method: "PUT" }),
+ onSuccess: (body) => queryClient.setQueryData(settingsQueryKey, body),
+ });
+}
+
+export function useResetSetting() {
+ const queryClient = useQueryClient();
+ return useMutation({
+ mutationFn: (key: SettingKey) =>
+ apiFetch<Body>(`/api/settings?key=${encodeURIComponent(key)}`, { method: "DELETE" }),
+ onSuccess: (body) => queryClient.setQueryData(settingsQueryKey, body),
+ });
+}