Open source CapCut alternative.
TypeScript
feat(site): analytics serves a summary and a paged user list
David Lefe606834c10c
diff --git a/site/scripts/analytics-repair-credit-activity.ts b/site/scripts/analytics-repair-credit-activity.tsindex 360f5c67..cec1f3da 100644--- a/site/scripts/analytics-repair-credit-activity.ts+++ b/site/scripts/analytics-repair-credit-activity.ts@@ -5,8 +5,13 @@ import { mkdtemp, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";-import { dayDbKey, extractCreditActivity, ROLLUP_KEY } from "../src/lib/analytics/pipeline";-import { analyticsDbDayFileSchema, analyticsRollupSchema } from "../src/lib/analytics/schema";+import { extractCreditActivity } from "../src/lib/analytics/pipeline";+import {+ analyticsDbDayFileSchema,+ analyticsRollupSchema,+ dayDbKey,+ ROLLUP_KEY,+} from "../src/lib/analytics/schema";import { prisma } from "../src/lib/prisma";const apply = process.argv.includes("--apply");diff --git a/site/src/app/api/analytics/rollup/route.ts b/site/src/app/api/analytics/rollup/route.tsdeleted file mode 100644index 1e99739e..00000000--- a/site/src/app/api/analytics/rollup/route.ts+++ /dev/null@@ -1,24 +0,0 @@-import { NextResponse } from "next/server";--import { getObject, R2NotConfiguredError } from "@/cut/server/cloud/r2";-import { ROLLUP_KEY } from "@/lib/analytics/pipeline";-import { withSuperUser } from "@/lib/donkey-api-auth";--// The consolidated analytics JSON, passed through exactly as the nightly job-// wrote it. It carries emails and balances, so it stays behind the super-user-// gate; the dashboard reads this and never the database.-export const GET = withSuperUser(async () => {- let rollup;- try {- rollup = await getObject(ROLLUP_KEY);- } catch (e) {- if (e instanceof R2NotConfiguredError) {- return NextResponse.json({ error: "Storage is not configured." }, { status: 503 });- }- throw e;- }- if (!rollup) return NextResponse.json({ error: "No rollup yet." }, { status: 404 });- return new NextResponse(new Uint8Array(rollup.bytes), {- headers: { "Cache-Control": "no-store", "Content-Type": "application/json" },- });-});diff --git a/site/src/app/api/analytics/summary/route.ts b/site/src/app/api/analytics/summary/route.tsnew file mode 100644index 00000000..b0a39718--- /dev/null+++ b/site/src/app/api/analytics/summary/route.ts@@ -0,0 +1,21 @@+import { NextResponse } from "next/server";++import { noRollupResponse, readRollup, storageProblemResponse } from "@/lib/analytics/read";+import { summarizeRollup } from "@/lib/analytics/summarize";+import { withSuperUser } from "@/lib/donkey-api-auth";++// The nightly rollup folded to the numbers a dashboard draws: one point per+// day, the headline counts, and the billing and referral blocks. Tens of+// kilobytes whatever the account table does, and no account's email or credit+// balance rides along — people come from /api/analytics/users, a page at a+// time. Super users only, like everything under /api/analytics.+export const GET = withSuperUser(async () => {+ let rollup;+ try {+ rollup = await readRollup();+ } catch (e) {+ return storageProblemResponse(e);+ }+ if (!rollup) return noRollupResponse();+ return NextResponse.json(summarizeRollup(rollup), { headers: { "Cache-Control": "no-store" } });+});diff --git a/site/src/app/api/analytics/users/route.ts b/site/src/app/api/analytics/users/route.tsnew file mode 100644index 00000000..3a7dd326--- /dev/null+++ b/site/src/app/api/analytics/users/route.ts@@ -0,0 +1,62 @@+import { NextResponse } from "next/server";+import { z } from "zod";++import { noRollupResponse, readRollup, storageProblemResponse } from "@/lib/analytics/read";+import {+ rankUsers,+ USER_SORTS,+ USERS_PAGE_SIZE,+ type AnalyticsUsersPage,+ type UserSort,+} from "@/lib/analytics/rank";+import { withSuperUser } from "@/lib/donkey-api-auth";++// One page of accounts out of the nightly rollup, ranked by the sort the+// caller asked for. The ranking runs here, over the whole window, so a page is+// the top of a real order rather than a slice of an arbitrary one, and the+// dashboard holds the rows it shows instead of every account there is.+//+// `cursor` is the rank offset of the next page. Ranks are only stable within+// one rollup, so the answer carries `generatedAt` and a caller keys its pages+// on it: a new rollup starts a new list.+const sortIds = USER_SORTS.map((sort) => sort.id) as [UserSort, ...UserSort[]];++const querySchema = z.object({+ sort: z.enum(sortIds).default("active"),+ cursor: z.coerce.number().int().min(0).default(0),+ limit: z.coerce.number().int().min(1).max(200).default(USERS_PAGE_SIZE),+ // A specific account, for a panel that needs one person's row.+ id: z.string().min(1).optional(),+});++export const GET = withSuperUser(async (request) => {+ const params = querySchema.safeParse(Object.fromEntries(request.nextUrl.searchParams));+ if (!params.success) {+ return NextResponse.json({ error: "Bad query." }, { status: 400 });+ }+ const { cursor, id, limit, sort } = params.data;++ let rollup;+ try {+ rollup = await readRollup();+ } catch (e) {+ return storageProblemResponse(e);+ }+ if (!rollup) return noRollupResponse();++ const ranked = rankUsers(rollup, sort);+ const page = id+ ? ranked.filter((user) => user.id === id)+ : ranked.slice(cursor, cursor + limit);+ const end = cursor + limit;++ const body: AnalyticsUsersPage = {+ generatedAt: rollup.generatedAt,+ nextCursor: id || end >= ranked.length ? null : end,+ pageSize: limit,+ sorts: USER_SORTS,+ total: ranked.length,+ users: page,+ };+ return NextResponse.json(body, { headers: { "Cache-Control": "no-store" } });+});diff --git a/site/src/app/su/SuShell.tsx b/site/src/app/su/SuShell.tsxindex 24aaf9c8..1e1e0efa 100644--- a/site/src/app/su/SuShell.tsx+++ b/site/src/app/su/SuShell.tsx@@ -7,7 +7,7 @@ import { SuCrumbProvider } from "@/app/su/SuCrumb";import { SuHeader } from "@/app/su/SuHeader";import { SuSidebar } from "@/app/su/SuSidebar";import { SidebarProvider, useSidebar } from "@/components/ui/sidebar";-import { useWarmAnalyticsRollup } from "@/queries/analytics";+import { useWarmAnalyticsSummary } from "@/queries/analytics";// The section's two-pane shell. It mounts only for a super user: the proxy// (src/proxy.ts) has already checked the session and the role before the@@ -17,7 +17,7 @@ import { useWarmAnalyticsRollup } from "@/queries/analytics";// and the rail share one open state: on a phone the rail is a sheet the// header opens, on a wider screen it is the fixed left column.export function SuShell({ children }: { children: ReactNode }) {- useWarmAnalyticsRollup();+ useWarmAnalyticsSummary();// The section paints the white product surface. `app-surface` is what// repoints the --background token for the whole document (see globals.css),// so it belongs on the outermost element here.diff --git a/site/src/app/su/analytics/product/page.tsx b/site/src/app/su/analytics/product/page.tsxindex 541ec982..169883ba 100644--- a/site/src/app/su/analytics/product/page.tsx+++ b/site/src/app/su/analytics/product/page.tsx@@ -1,7 +1,7 @@"use client";import { ArrowUpRight, Check, Mail } from "lucide-react";-import { useMemo, useRef, useState } from "react";+import { useEffect, useMemo, useRef, useState } from "react";import {Area,AreaChart,@@ -23,12 +23,9 @@ import {ChartTooltipContent,type ChartConfig,} from "@/components/ui/chart";-import type {- AnalyticsBilling,- AnalyticsReferrals,- AnalyticsRollup,- AnalyticsRollupUser,-} from "@/lib/analytics/schema";+import type { AnalyticsBilling, AnalyticsReferrals } from "@/lib/analytics/schema";+import { USER_SORTS, type RankedUser, type UserSort } from "@/lib/analytics/rank";+import { workBitsOf, type AnalyticsSummary } from "@/lib/analytics/summarize";import { REFERRAL_SOURCES } from "@/lib/onboarding/sequence";import { useLocalPref } from "@/cut/lib/uiState";import { cn } from "@/lib/utils";@@ -37,14 +34,16 @@ import { OutreachComposeDialog } from "@/app/su/outreach/ComposeDialog";import { Button } from "@/components/ui/button";import { SuStandIn } from "@/app/su/SuStandIn";import { useRowWindow } from "@/app/su/analytics/rowWindow";-import { useAnalyticsRollup } from "@/queries/analytics";+import { useAnalyticsSummary, useAnalyticsUsers } from "@/queries/analytics";import { ApiError } from "@/queries/apiClient";import { useOutreachAction, type OutreachRow } from "@/queries/outreach";-// Everything here renders the nightly rollup (analytics/rollup.json via-// /api/analytics/rollup) — stale until the next job run by design. "Active" is-// any source bit for the day; "working" narrows to the DB event sources, i.e.-// the user did something beyond opening the app.+// Everything here renders the nightly rollup — stale until the next job run by+// design. The charts read the folded numbers (/api/analytics/summary) and the+// grid pages through the accounts (/api/analytics/users), so the page never+// holds the whole account table. "Active" is any source bit for the day;+// "working" narrows to the DB event sources, i.e. the user did something+// beyond opening the app.// Formatting a day goes through the locale machinery, which is slow enough// that a label per grid cell held the page for over a second. The rollup@@ -87,102 +86,6 @@ function storageTone(bytes: number, quotaBytes: number | null | undefined): strireturn "";}-// Activity is null for a day the pipeline never extracted: the masks are-// empty because there was nothing to read, which is not the same as a day-// nobody worked. Those days leave a gap in the charts and an unknown dot in-// the grid. Signups come from the user snapshot, so they are always known.-type DayPoint = {- day: string;- active: number | null;- working: number | null;- signups: number;- totalRegistered: number;-};--type RollupView = {- series: DayPoint[];- workBits: number;- missingDays: Set<string>;- registered: number;- signups7d: number;- signupsWindow: number;- activeYesterday: number | null;- active7d: number | null;- activePrior7d: number | null;- totalBalanceMicros: bigint;-};--function deriveView(rollup: AnalyticsRollup): RollupView {- const workBits = rollup.sources.reduce(- (mask, source, i) => (source === "posthog" ? mask : mask | (1 << i)),- 0,- );-- const signupsByDay = new Map<string, number>();- for (const user of rollup.users) {- const day = user.registeredAt.slice(0, 10);- signupsByDay.set(day, (signupsByDay.get(day) ?? 0) + 1);- }-- // Cumulative registrations start from everyone who signed up before the- // window, so the total line carries the real base, not zero.- const firstDay = rollup.days[0] ?? "";- let totalRegistered = rollup.users.filter(- (user) => user.registeredAt.slice(0, 10) < firstDay,- ).length;-- // Any day the consolidation could not read a source for is undercounted at- // best, and indistinguishable from a quiet day — so it reports as unknown- // rather than as a number the reader would trust.- const missingDays = new Set(rollup.missing.map((entry) => entry.day));-- const series = rollup.days.map((day, i) => {- const signups = signupsByDay.get(day) ?? 0;- totalRegistered += signups;- if (missingDays.has(day)) {- return { active: null, day, signups, totalRegistered, working: null };- }- let active = 0;- let working = 0;- for (const user of rollup.users) {- const mask = user.activity[i] ?? 0;- if (mask !== 0) active++;- if ((mask & workBits) !== 0) working++;- }- return { active, day, signups, totalRegistered, working };- });-- // Null when the whole range went unextracted; otherwise it counts over the- // days there is data for, so one missing day doesn't drag the number down.- const activeInRange = (from: number, to: number): number | null => {- const known: number[] = [];- for (let i = Math.max(0, from); i < to; i++) {- if (!missingDays.has(rollup.days[i])) known.push(i);- }- if (known.length === 0) return null;- let count = 0;- for (const user of rollup.users) {- if (known.some((i) => (user.activity[i] ?? 0) !== 0)) count++;- }- return count;- };-- const len = rollup.days.length;- const last7 = rollup.days.slice(-7);- return {- active7d: activeInRange(len - 7, len),- activePrior7d: activeInRange(len - 14, len - 7),- activeYesterday: series[len - 1]?.active ?? null,- missingDays,- registered: rollup.users.length,- series,- signups7d: last7.reduce((sum, day) => sum + (signupsByDay.get(day) ?? 0), 0),- signupsWindow: series.reduce((sum, point) => sum + point.signups, 0),- totalBalanceMicros: rollup.users.reduce((sum, u) => sum + BigInt(u.balanceMicros), BigInt(0)),- workBits,- };-}-// One chart point per day, twice over: `series` holds the per-source answer// counts of that day (the stacked bars), `cumulative` the running totals per// source plus the running total of users who answered (the trend lines).@@ -637,63 +540,16 @@ function ActivityDot({);}-// Ranking weight decays with age — a working day two weeks back counts half-// of yesterday's — so the list leads with who is active now. A day the user-// only visited counts a fraction of one they worked.-const HALF_LIFE_DAYS = 14;-const VISIT_WEIGHT = 0.35;--const USER_SORTS = [- { id: "active", label: "Most active" },- { id: "recent", label: "Recently active" },- { id: "joined", label: "Newest" },- { id: "paid", label: "Top paid" },-] as const;-type UserSort = (typeof USER_SORTS)[number]["id"];--type RankedUser = AnalyticsRollupUser & {- score: number;- activeDays: number;- /** Index into rollup.days of the last day with any activity; -1 for never. */- lastActive: number;-};--function rankUsers(rollup: AnalyticsRollup, workBits: number, sort: UserSort): RankedUser[] {- const len = rollup.days.length;- const ranked = rollup.users.map((user) => {- let score = 0;- let activeDays = 0;- let lastActive = -1;- for (let i = 0; i < len; i++) {- const mask = user.activity[i] ?? 0;- if (mask === 0) continue;- activeDays++;- lastActive = i;- score +=- ((mask & workBits) !== 0 ? 1 : VISIT_WEIGHT) * 0.5 ** ((len - 1 - i) / HALF_LIFE_DAYS);- }- return { ...user, activeDays, lastActive, score };- });- const funded = (user: RankedUser) => Number(user.fundedMicros ?? "0");- const by: Record<UserSort, (a: RankedUser, b: RankedUser) => number> = {- active: (a, b) => b.score - a.score,- joined: (a, b) => (a.registeredAt < b.registeredAt ? 1 : -1),- paid: (a, b) => funded(b) - funded(a) || b.score - a.score,- recent: (a, b) => b.lastActive - a.lastActive || b.score - a.score,- };- // Our own accounts sink to the bottom under every sort: they are active- // every day and would otherwise own the top of the list.- return ranked.sort(- (a, b) => Number(a.superUser === true) - Number(b.superUser === true) || by[sort](a, b),- );-}+// How close to the end of the loaded rows the window may come before the next+// page is asked for, so scrolling meets rows that are already there.+const LOAD_AHEAD_ROWS = 20;function ActivityGrid({- rollup,+ summary,workBits,missingDays,}: {- rollup: AnalyticsRollup;+ summary: AnalyticsSummary;workBits: number;missingDays: Set<string>;}) {@@ -702,7 +558,14 @@ function ActivityGrid({"active",(v) => USER_SORTS.some((option) => option.id === v),);- const users = useMemo(() => rankUsers(rollup, workBits, sort), [rollup, workBits, sort]);+ // Ranked on the server, a page at a time: the browser holds the rows it has+ // scrolled to, never every account.+ const paged = useAnalyticsUsers(sort, summary.generatedAt);+ const users: RankedUser[] = useMemo(+ () => paged.data?.pages.flatMap((page) => page.users) ?? [],+ [paged.data],+ );+ const total = paged.data?.pages[0]?.total ?? 0;// A row marks itself processed on click and stays marked across visits, so// a pass through the list can pick up where it left off.const [processedIds, setProcessedIds] = useLocalPref<string[]>(@@ -715,9 +578,16 @@ function ActivityGrid({setProcessedIds((prev) => (prev.includes(id) ? prev.filter((x) => x !== id) : [...prev, id]));const markProcessed = (id: string) =>setProcessedIds((prev) => (prev.includes(id) ? prev : [...prev, id]));- const processedCount = users.reduce((n, user) => n + (processed.has(user.id) ? 1 : 0), 0);+ const processedCount = processedIds.length;const body = useRef<HTMLTableSectionElement>(null);const rows = useRowWindow(users.length, body);+ // Scrolling near the end of what is loaded pulls the next page.+ const { fetchNextPage, hasNextPage, isFetchingNextPage } = paged;+ useEffect(() => {+ if (hasNextPage && !isFetchingNextPage && rows.end >= users.length - LOAD_AHEAD_ROWS) {+ void fetchNextPage();+ }+ }, [fetchNextPage, hasNextPage, isFetchingNextPage, rows.end, users.length]);// A row's email button puts the account on the outreach list and opens the// same note the Outreach tab writes, so a busy user gets a word or a deal// from here.@@ -746,8 +616,8 @@ function ActivityGrid({// before any horizontal scroll; each column keeps its index into the// activity masks.const columns = useMemo(- () => rollup.days.map((day, i) => ({ day, i, unknown: missingDays.has(day) })).reverse(),- [rollup.days, missingDays],+ () => summary.days.map((day, i) => ({ day, i, unknown: missingDays.has(day) })).reverse(),+ [summary.days, missingDays],);const columnCount = columns.length + 1;@@ -765,7 +635,7 @@ function ActivityGrid({? `${user.email} — ${when}: no data`: mask === 0? `${user.email} — ${when}: inactive`- : `${user.email} — ${when}: ${rollup.sources.filter((_, i) => (mask & (1 << i)) !== 0).join(", ")}`;+ : `${user.email} — ${when}: ${summary.sources.filter((_, i) => (mask & (1 << i)) !== 0).join(", ")}`;};return (@@ -774,8 +644,9 @@ function ActivityGrid({<div><p className="font-medium">User activity</p><p className="text-sm text-muted-foreground">- One dot per user per day, last {rollup.days.length} days+ One dot per user per day, last {summary.days.length} days{missingDays.size > 0 && ` · ${missingDays.size} without data`}+ {total > 0 && ` · ${users.length.toLocaleString("en-US")} of ${total.toLocaleString("en-US")} accounts loaded`}{processedCount > 0 && (<>{" "}@@ -893,7 +764,7 @@ function ActivityGrid({<span className="block text-xs text-muted-foreground">{user.activeDays === 0? "no activity"- : `${user.activeDays} active ${user.activeDays === 1 ? "day" : "days"} · last ${formatDay(rollup.days[user.lastActive])}`}{" "}+ : `${user.activeDays} active ${user.activeDays === 1 ? "day" : "days"} · last ${formatDay(summary.days[user.lastActive])}`}{" "}· joined {formatDay(user.registeredAt.slice(0, 10))} ·{" "}{formatMicros(user.balanceMicros)}{user.fundedMicros !== undefined && (@@ -934,6 +805,16 @@ function ActivityGrid({<td colSpan={columnCount} className="p-0" style={{ height: rows.below }} /></tr>)}+ {(paged.isPending || paged.isFetchingNextPage || paged.isError) && (+ <tr>+ <td+ className="sticky left-0 z-10 bg-card py-2 pl-6 text-sm text-muted-foreground"+ colSpan={columnCount}+ >+ {paged.isError ? "Couldn\u2019t load accounts." : "Loading accounts\u2026"}+ </td>+ </tr>+ )}</tbody></table></div>@@ -968,16 +849,12 @@ function ActivityGrid({}export default function SuAnalyticsPage() {- const rollup = useAnalyticsRollup();- const view = useMemo(- () => (rollup.data ? deriveView(rollup.data) : null),- [rollup.data],- );+ const summaryQuery = useAnalyticsSummary();const referrals = useMemo(- () => (rollup.data?.referrals ? deriveReferrals(rollup.data.referrals) : null),- [rollup.data],+ () => (summaryQuery.data?.referrals ? deriveReferrals(summaryQuery.data.referrals) : null),+ [summaryQuery.data],);- const billingSection = rollup.data?.billing;+ const billingSection = summaryQuery.data?.billing;const revenue = useMemo(() => (billingSection && "days" in billingSection ? deriveRevenue(billingSection) : null),[billingSection],@@ -993,10 +870,10 @@ export default function SuAnalyticsPage() {const tiles = useReorder("su-analytics-tile-order", TILE_IDS);const cards = useReorder("su-analytics-card-order", CARD_IDS);- if (rollup.isPending) return <SuStandIn />;+ if (summaryQuery.isPending) return <SuStandIn />;- if (rollup.error || !view || !rollup.data) {- const noData = rollup.error instanceof ApiError && rollup.error.status === 404;+ if (summaryQuery.error || !summaryQuery.data) {+ const noData = summaryQuery.error instanceof ApiError && summaryQuery.error.status === 404;return (<div className="rounded-xl border bg-card p-5 text-sm text-muted-foreground">{noData@@ -1006,16 +883,18 @@ export default function SuAnalyticsPage() {);}- const data = rollup.data;+ const summary = summaryQuery.data;+ const missingDays = new Set(summary.missing);+ const workBits = workBitsOf(summary.sources);const deltaPct =- view.active7d !== null && view.activePrior7d !== null && view.activePrior7d > 0- ? `${(((view.active7d - view.activePrior7d) / view.activePrior7d) * 100).toFixed(1)}%`+ summary.active7d !== null && summary.activePrior7d !== null && summary.activePrior7d > 0+ ? `${(((summary.active7d - summary.activePrior7d) / summary.activePrior7d) * 100).toFixed(1)}%`: null;const count = (n: number | null) => (n === null ? "—" : n.toLocaleString("en-US"));- const lastDay = data.days[data.days.length - 1];+ const lastDay = summary.days[summary.days.length - 1];// Billing is absent from rollups written before it shipped (or before it// took this shape); the next run fills it in.- const billing = data.billing && "days" in data.billing ? data.billing : undefined;+ const billing = summary.billing && "days" in summary.billing ? summary.billing : undefined;const churnBase = billing ? billing.subscribers + billing.churned : 0;const staleBilling = "not in this rollup yet — run analytics";const links = billing ? stripeLinks(billing.dashboardUrl) : undefined;@@ -1025,27 +904,27 @@ export default function SuAnalyticsPage() {registered: (<StatTilelabel="Registered users"- value={view.registered.toLocaleString("en-US")}- sub={`+${view.signups7d} in the last 7 days`}+ value={summary.registered.toLocaleString("en-US")}+ sub={`+${summary.signups7d} in the last 7 days`}/>),activeYesterday: (<StatTilelabel="Active yesterday"- value={count(view.activeYesterday)}+ value={count(summary.activeYesterday)}sub={- view.activeYesterday === null+ summary.activeYesterday === null? `no extract for ${formatDay(lastDay)} yet`- : `of ${view.registered.toLocaleString("en-US")} registered`+ : `of ${summary.registered.toLocaleString("en-US")} registered`}/>),active7d: (<StatTilelabel="Active last 7 days"- value={count(view.active7d)}+ value={count(summary.active7d)}sub={- view.active7d === null ? (+ summary.active7d === null ? ("no extracts for the last 7 days") : deltaPct === null ? ("no prior-week baseline"@@ -1069,7 +948,7 @@ export default function SuAnalyticsPage() {balance: (<StatTilelabel="Outstanding balance"- value={formatMicros(view.totalBalanceMicros)}+ value={formatMicros(summary.balanceMicros)}sub="credits across all accounts"/>),@@ -1154,13 +1033,13 @@ export default function SuAnalyticsPage() {<ChartCardtitle="Active users"subtitle={- view.missingDays.size > 0- ? `Daily actives, last 60 days · ${view.missingDays.size} days without data are left blank`+ missingDays.size > 0+ ? `Daily actives, last 60 days · ${missingDays.size} days without data are left blank`: "Daily actives, last 60 days"}><ChartContainer className="w-full" config={activesConfig}>- <AreaChart accessibilityLayer data={view.series} margin={{ left: -16 }}>+ <AreaChart accessibilityLayer data={summary.series} margin={{ left: -16 }}><CartesianGrid vertical={false} /><XAxisaxisLine={false}@@ -1207,10 +1086,10 @@ export default function SuAnalyticsPage() {signups: (<ChartCardtitle="Signups"- subtitle={`New registrations per day · ${view.signupsWindow.toLocaleString("en-US")} in the last 60 days`}+ subtitle={`New registrations per day · ${summary.signupsWindow.toLocaleString("en-US")} in the last 60 days`}><ChartContainer className="w-full" config={signupsConfig}>- <BarChart accessibilityLayer data={view.series} margin={{ left: -16 }}>+ <BarChart accessibilityLayer data={summary.series} margin={{ left: -16 }}><CartesianGrid vertical={false} /><XAxisaxisLine={false}@@ -1235,7 +1114,7 @@ export default function SuAnalyticsPage() {totalRegistered: (<ChartCard title="Total registered" subtitle="Cumulative registrations, last 60 days"><ChartContainer className="max-h-56 w-full" config={totalRegisteredConfig}>- <AreaChart accessibilityLayer data={view.series} margin={{ left: -16 }}>+ <AreaChart accessibilityLayer data={summary.series} margin={{ left: -16 }}><CartesianGrid vertical={false} /><XAxisaxisLine={false}@@ -1537,11 +1416,11 @@ export default function SuAnalyticsPage() {))}</div>- <ActivityGrid missingDays={view.missingDays} rollup={data} workBits={view.workBits} />+ <ActivityGrid missingDays={missingDays} summary={summary} workBits={workBits} /><p className="text-xs text-muted-foreground">From the nightly rollup generated{" "}- {new Date(data.generatedAt).toLocaleString("en-US", { timeZoneName: "short" })}.+ {new Date(summary.generatedAt).toLocaleString("en-US", { timeZoneName: "short" })}.</p></div>);diff --git a/site/src/app/su/outreach/UserHistory.tsx b/site/src/app/su/outreach/UserHistory.tsxindex d4f4d75f..4bb4c066 100644--- a/site/src/app/su/outreach/UserHistory.tsx+++ b/site/src/app/su/outreach/UserHistory.tsx@@ -4,7 +4,7 @@ import { useState } from "react";import { formatBytes } from "@/lib/bytes";import { formatUsdPlain } from "@/lib/credits/format-usd";-import { useAnalyticsRollup } from "@/queries/analytics";+import { useAnalyticsSummary, useAnalyticsUser } from "@/queries/analytics";import { type OutreachRow } from "@/queries/outreach";import { useUserHistory, type UserHistory, type UserHistoryEmail } from "@/queries/userHistory";@@ -137,11 +137,12 @@ const TONE: Record<Event["tone"], string> = {export function UserHistoryPanel({ target }: { target: OutreachRow }) {const history = useUserHistory(target.userId);- const rollup = useAnalyticsRollup();- const rolled = rollup.data?.users.find((user) => user.id === target.userId);- // The rollup's billing events name the payer by email; the window's charges- // for this person are the ones with theirs.- const charges = (rollup.data?.billing?.events ?? []).filter(+ // This one account's row out of the nightly rollup, and the billing events+ // the summary carries. The rollup's events name the payer by email; the+ // window's charges for this person are the ones with theirs.+ const rolled = useAnalyticsUser(target.userId).data;+ const summary = useAnalyticsSummary();+ const charges = (summary.data?.billing?.events ?? []).filter((event) => event.email === target.email && event.kind !== "canceled" && event.amountMicros !== null,);// The moment the panel opened; offers and grants are read against it.diff --git a/site/src/lib/analytics/pipeline.ts b/site/src/lib/analytics/pipeline.tsindex ab275291..594623b2 100644--- a/site/src/lib/analytics/pipeline.ts+++ b/site/src/lib/analytics/pipeline.ts@@ -20,6 +20,11 @@ import {analyticsPosthogDayFileSchema,analyticsRollupSchema,analyticsSnapshotFileSchema,+ DAYS_PREFIX,+ dayDbKey,+ dayPosthogKey,+ ROLLUP_KEY,+ SNAPSHOT_KEY,type AnalyticsDbDayFile,type AnalyticsPosthogDayFile,type AnalyticsRollup,@@ -34,13 +39,6 @@ import { lapsedCreditByUser } from "@/lib/credits/inference";import { REFERRAL_SOURCES } from "@/lib/onboarding/sequence";import { prisma } from "@/lib/prisma";-const ANALYTICS_PREFIX = "analytics/";-const DAYS_PREFIX = `${ANALYTICS_PREFIX}days/`;-export const dayDbKey = (day: string) => `${DAYS_PREFIX}${day}/db.json`;-export const dayPosthogKey = (day: string) => `${DAYS_PREFIX}${day}/posthog.json`;-export const SNAPSHOT_KEY = `${ANALYTICS_PREFIX}snapshot.json`;-export const ROLLUP_KEY = `${ANALYTICS_PREFIX}rollup.json`;-const WINDOW_DAYS = 60;// Backfill after an outage converges over nights instead of one heavy run.const MAX_EXTRACT_DAYS_PER_RUN = 5;@@ -336,7 +334,7 @@ async function consolidate(// day at a time, so a 60-day window holds one day's ids at once; nothing here// touches storage or the clock. Every rollup in storage came out of this// function, so the fixture the phone's model is tested against comes out of-// it too (rollup-fixture.ts): a change to what the dashboard is served+// it too (summary-fixture.ts): a change to what the dashboard is served// reaches that test in the same change.export async function buildRollup(input: {snapshot: AnalyticsSnapshotFile;diff --git a/site/src/lib/analytics/rank.ts b/site/src/lib/analytics/rank.tsnew file mode 100644index 00000000..6d1958bb--- /dev/null+++ b/site/src/lib/analytics/rank.ts@@ -0,0 +1,73 @@+// How the user list is ordered, and the per-user numbers that ordering needs.+// It runs on the server so a dashboard reads a page of people instead of the+// whole account table.+import { workBitsOf } from "@/lib/analytics/summarize";+import type { AnalyticsRollup, AnalyticsRollupUser } from "@/lib/analytics/schema";++// Ranking weight decays with age — a working day two weeks back counts half of+// yesterday's — so the list leads with who is active now. A day the user only+// visited counts a fraction of one they worked.+const HALF_LIFE_DAYS = 14;+const VISIT_WEIGHT = 0.35;++/** How many accounts one page carries. The server owns it: a client asks for+ * the first page, then follows `nextCursor` until it is null. */+export const USERS_PAGE_SIZE = 50;++export const USER_SORTS = [+ { id: "active", label: "Most active" },+ { id: "recent", label: "Recently active" },+ { id: "joined", label: "Newest" },+ { id: "paid", label: "Top paid" },+] as const;+export type UserSort = (typeof USER_SORTS)[number]["id"];++export type RankedUser = AnalyticsRollupUser & {+ score: number;+ activeDays: number;+ /** Index into the rollup's days of the last day with any activity; -1 for never. */+ lastActive: number;+};++/** One page of ranked accounts, as /api/analytics/users answers it. Ranks+ * hold only within one rollup, so a caller keys its pages on `generatedAt`. */+export type AnalyticsUsersPage = {+ generatedAt: string;+ users: RankedUser[];+ total: number;+ nextCursor: number | null;+ /** The orders this API ranks by, so every client offers the same ones. */+ sorts: readonly { id: UserSort; label: string }[];+ pageSize: number;+};++export function rankUsers(rollup: AnalyticsRollup, sort: UserSort): RankedUser[] {+ const workBits = workBitsOf(rollup.sources);+ const len = rollup.days.length;+ const ranked = rollup.users.map((user) => {+ let score = 0;+ let activeDays = 0;+ let lastActive = -1;+ for (let i = 0; i < len; i++) {+ const mask = user.activity[i] ?? 0;+ if (mask === 0) continue;+ activeDays++;+ lastActive = i;+ score +=+ ((mask & workBits) !== 0 ? 1 : VISIT_WEIGHT) * 0.5 ** ((len - 1 - i) / HALF_LIFE_DAYS);+ }+ return { ...user, activeDays, lastActive, score };+ });+ const funded = (user: RankedUser) => Number(user.fundedMicros ?? "0");+ const by: Record<UserSort, (a: RankedUser, b: RankedUser) => number> = {+ active: (a, b) => b.score - a.score,+ joined: (a, b) => (a.registeredAt < b.registeredAt ? 1 : -1),+ paid: (a, b) => funded(b) - funded(a) || b.score - a.score,+ recent: (a, b) => b.lastActive - a.lastActive || b.score - a.score,+ };+ // Our own accounts sink to the bottom under every sort: they are active+ // every day and would otherwise own the top of the list.+ return ranked.sort(+ (a, b) => Number(a.superUser === true) - Number(b.superUser === true) || by[sort](a, b),+ );+}diff --git a/site/src/lib/analytics/read.ts b/site/src/lib/analytics/read.tsnew file mode 100644index 00000000..156ef45e--- /dev/null+++ b/site/src/lib/analytics/read.ts@@ -0,0 +1,26 @@+// Server-side access to the stored rollup. The su routes fold it into what a+// caller asked for — a summary, or one page of people — so the file itself+// never goes over the wire.+import { NextResponse } from "next/server";++import { getObject, R2NotConfiguredError } from "@/cut/server/cloud/r2";+import { analyticsRollupSchema, ROLLUP_KEY, type AnalyticsRollup } from "@/lib/analytics/schema";++/** The stored rollup, or null when the nightly job hasn't written one. */+export async function readRollup(): Promise<AnalyticsRollup | null> {+ const stored = await getObject(ROLLUP_KEY);+ if (!stored) return null;+ return analyticsRollupSchema.parse(JSON.parse(stored.bytes.toString("utf8")));+}++/** The answer when there is no rollup to read: a deploy without storage, or a+ * job that has not run yet. */+export const noRollupResponse = () =>+ NextResponse.json({ error: "No rollup yet." }, { status: 404 });++export function storageProblemResponse(error: unknown): NextResponse {+ if (error instanceof R2NotConfiguredError) {+ return NextResponse.json({ error: "Storage is not configured." }, { status: 503 });+ }+ throw error;+}diff --git a/site/src/lib/analytics/schema.ts b/site/src/lib/analytics/schema.tsindex 51c8d192..e47dd57b 100644--- a/site/src/lib/analytics/schema.ts+++ b/site/src/lib/analytics/schema.ts@@ -5,6 +5,16 @@ import { z } from "zod";export const ANALYTICS_ROLLUP_VERSION = 1;+// Where the pipeline's files live in R2. Here rather than beside the pipeline+// so a reader of one file — the rollup route — names it without loading the+// extraction code.+const ANALYTICS_PREFIX = "analytics/";+export const DAYS_PREFIX = `${ANALYTICS_PREFIX}days/`;+export const dayDbKey = (day: string) => `${DAYS_PREFIX}${day}/db.json`;+export const dayPosthogKey = (day: string) => `${DAYS_PREFIX}${day}/posthog.json`;+export const SNAPSHOT_KEY = `${ANALYTICS_PREFIX}snapshot.json`;+export const ROLLUP_KEY = `${ANALYTICS_PREFIX}rollup.json`;+// Bit order for the per-day activity masks. Append-only: a new source takes// the next bit so already-written rollups keep decoding.export const ANALYTICS_DB_SOURCES = [diff --git a/site/src/lib/analytics/summarize.ts b/site/src/lib/analytics/summarize.tsnew file mode 100644index 00000000..f68af0ff--- /dev/null+++ b/site/src/lib/analytics/summarize.ts@@ -0,0 +1,114 @@+// The rollup folded to what a dashboard draws: one point per day plus the+// headline numbers. Pure, and the only place those numbers are computed — the+// su charts and the phone both read the result, so neither walks the user+// list to draw a line.+import type { AnalyticsRollup } from "@/lib/analytics/schema";++/** Activity is null for a day the pipeline never extracted: nothing was read,+ * which is not the same as a day nobody worked. Signups come from the user+ * snapshot, so they are always known. */+export type AnalyticsDayPoint = {+ day: string;+ active: number | null;+ working: number | null;+ signups: number;+ totalRegistered: number;+};++export type AnalyticsSummary = {+ version: number;+ generatedAt: string;+ /** Oldest → newest. */+ days: string[];+ /** Bit order of the activity masks the users page carries. */+ sources: string[];+ /** Days the consolidation could not read. */+ missing: string[];+ series: AnalyticsDayPoint[];+ registered: number;+ signups7d: number;+ signupsWindow: number;+ activeYesterday: number | null;+ active7d: number | null;+ activePrior7d: number | null;+ /** Credit held across every account, BigInt micros as a decimal string. */+ balanceMicros: string;+ billing?: AnalyticsRollup["billing"];+ referrals?: AnalyticsRollup["referrals"];+};++/** The mask bits that mean the person did something, not just opened the app. */+export function workBitsOf(sources: string[]): number {+ return sources.reduce((mask, source, i) => (source === "posthog" ? mask : mask | (1 << i)), 0);+}++export function summarizeRollup(rollup: AnalyticsRollup): AnalyticsSummary {+ const workBits = workBitsOf(rollup.sources);+ const missing = rollup.missing.map((entry) => entry.day);+ const missingDays = new Set(missing);++ const signupsByDay = new Map<string, number>();+ for (const user of rollup.users) {+ const day = user.registeredAt.slice(0, 10);+ signupsByDay.set(day, (signupsByDay.get(day) ?? 0) + 1);+ }++ // Cumulative registrations start from everyone who signed up before the+ // window, so the total line carries the real base, not zero.+ const firstDay = rollup.days[0] ?? "";+ let totalRegistered = rollup.users.filter(+ (user) => user.registeredAt.slice(0, 10) < firstDay,+ ).length;++ const series = rollup.days.map((day, i) => {+ const signups = signupsByDay.get(day) ?? 0;+ totalRegistered += signups;+ if (missingDays.has(day)) {+ return { active: null, day, signups, totalRegistered, working: null };+ }+ let active = 0;+ let working = 0;+ for (const user of rollup.users) {+ const mask = user.activity[i] ?? 0;+ if (mask !== 0) active++;+ if ((mask & workBits) !== 0) working++;+ }+ return { active, day, signups, totalRegistered, working };+ });++ // Null when the whole range went unextracted; otherwise it counts over the+ // days there is data for, so one missing day doesn't drag the number down.+ const activeInRange = (from: number, to: number): number | null => {+ const known: number[] = [];+ for (let i = Math.max(0, from); i < to; i++) {+ if (!missingDays.has(rollup.days[i])) known.push(i);+ }+ if (known.length === 0) return null;+ let count = 0;+ for (const user of rollup.users) {+ if (known.some((i) => (user.activity[i] ?? 0) !== 0)) count++;+ }+ return count;+ };++ const len = rollup.days.length;+ return {+ active7d: activeInRange(len - 7, len),+ activePrior7d: activeInRange(len - 14, len - 7),+ activeYesterday: series[len - 1]?.active ?? null,+ balanceMicros: rollup.users+ .reduce((sum, user) => sum + BigInt(user.balanceMicros), BigInt(0))+ .toString(),+ billing: rollup.billing,+ days: rollup.days,+ generatedAt: rollup.generatedAt,+ missing,+ referrals: rollup.referrals,+ registered: rollup.users.length,+ series,+ signups7d: rollup.days.slice(-7).reduce((sum, day) => sum + (signupsByDay.get(day) ?? 0), 0),+ signupsWindow: series.reduce((sum, point) => sum + point.signups, 0),+ sources: rollup.sources,+ version: rollup.version,+ };+}diff --git a/site/src/queries/analytics.ts b/site/src/queries/analytics.tsindex 39618f0c..10ccb447 100644--- a/site/src/queries/analytics.ts+++ b/site/src/queries/analytics.ts@@ -1,38 +1,82 @@"use client";-import { queryOptions, useMutation, useQuery, useQueryClient } from "@tanstack/react-query";+import {+ infiniteQueryOptions,+ queryOptions,+ useInfiniteQuery,+ useMutation,+ useQuery,+ useQueryClient,+} from "@tanstack/react-query";import { useEffect } from "react";-import type { AnalyticsRollup } from "@/lib/analytics/schema";+import type { AnalyticsUsersPage, UserSort } from "@/lib/analytics/rank";+import type { AnalyticsSummary } from "@/lib/analytics/summarize";import { apiFetch } from "@/queries/apiClient";import type { AsyncJobStatus } from "@/queries/jobs";-export const analyticsRollupQueryKey = ["analytics", "rollup"] as const;+export const analyticsQueryKey = ["analytics"] as const;+export const analyticsSummaryQueryKey = [...analyticsQueryKey, "summary"] as const;-// Super-user only: the nightly job's consolidated rollup. It changes once a-// night, and a manual run invalidates it, so a copy in memory stays good for-// the session.-const analyticsRollupQuery = queryOptions({- queryFn: () => apiFetch<AnalyticsRollup>("/api/analytics/rollup"),- queryKey: analyticsRollupQueryKey,+// Super-user only: the nightly job's numbers, without the account table. It+// changes once a night, and a manual run invalidates it, so a copy in memory+// stays good for the session.+const analyticsSummaryQuery = queryOptions({+ queryFn: () => apiFetch<AnalyticsSummary>("/api/analytics/summary"),+ queryKey: analyticsSummaryQueryKey,staleTime: 60 * 60_000,});-export function useAnalyticsRollup() {- return useQuery(analyticsRollupQuery);+export function useAnalyticsSummary() {+ return useQuery(analyticsSummaryQuery);}-// Pulls the rollup as soon as the super-user shell mounts, so the Product tab+// Pulls the summary as soon as the super-user shell mounts, so the Product tab// paints from memory when it opens.-export function useWarmAnalyticsRollup() {+export function useWarmAnalyticsSummary() {const queryClient = useQueryClient();useEffect(() => {- void queryClient.prefetchQuery(analyticsRollupQuery);+ void queryClient.prefetchQuery(analyticsSummaryQuery);}, [queryClient]);}+// Accounts in rank order, a page at a time: ask for the first page, then+// follow the cursor the server hands back — the phone reads the same endpoint+// the same way, and the page size stays the server's to set. Ranks hold only+// within one rollup, so the rollup's stamp is part of the key: a fresh rollup+// is a fresh list rather than pages from two different orders stitched+// together.+function analyticsUsersQuery(sort: UserSort, generatedAt: string) {+ return infiniteQueryOptions({+ getNextPageParam: (page: AnalyticsUsersPage) => page.nextCursor,+ initialPageParam: 0,+ queryFn: ({ pageParam }) =>+ apiFetch<AnalyticsUsersPage>(`/api/analytics/users?sort=${sort}&cursor=${pageParam}`),+ queryKey: [...analyticsQueryKey, "users", generatedAt, sort] as const,+ staleTime: 60 * 60_000,+ });+}++export function useAnalyticsUsers(sort: UserSort, generatedAt: string) {+ return useInfiniteQuery(analyticsUsersQuery(sort, generatedAt));+}++/** One account's row, for a panel that opens on a single person. */+export function useAnalyticsUser(userId: string) {+ return useQuery({+ queryFn: async () => {+ const page = await apiFetch<AnalyticsUsersPage>(+ `/api/analytics/users?id=${encodeURIComponent(userId)}`,+ );+ return page.users[0] ?? null;+ },+ queryKey: [...analyticsQueryKey, "user", userId] as const,+ staleTime: 60 * 60_000,+ });+}+// Super-user only: run the analytics-daily job now and follow it to-// completion, then refetch the rollup so the dashboard updates in place.+// completion, then refetch what the dashboard shows.export function useRunAnalytics() {const queryClient = useQueryClient();return useMutation({@@ -48,7 +92,6 @@ export function useRunAnalytics() {if (job.state === "error") throw new Error(job.error ?? "The run failed.");}},- onSuccess: () =>- queryClient.invalidateQueries({ queryKey: analyticsRollupQueryKey }),+ onSuccess: () => queryClient.invalidateQueries({ queryKey: analyticsQueryKey }),});}