Skip to content

Open source CapCut alternative.

TypeScript

feat(site): analytics serves a summary and a paged user list

David Lefe606834c10c
diff --git a/site/scripts/analytics-repair-credit-activity.ts b/site/scripts/analytics-repair-credit-activity.ts
index 360f5c67..cec1f3da 100644
--- a/site/scripts/analytics-repair-credit-activity.ts
+++ b/site/scripts/analytics-repair-credit-activity.ts
@@ -5,8 +5,13 @@ import { mkdtemp, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { dayDbKey, extractCreditActivity, ROLLUP_KEY } from "../src/lib/analytics/pipeline";
-import { analyticsDbDayFileSchema, analyticsRollupSchema } from "../src/lib/analytics/schema";
+import { extractCreditActivity } from "../src/lib/analytics/pipeline";
+import {
+ analyticsDbDayFileSchema,
+ analyticsRollupSchema,
+ dayDbKey,
+ ROLLUP_KEY,
+} from "../src/lib/analytics/schema";
import { prisma } from "../src/lib/prisma";
const apply = process.argv.includes("--apply");
diff --git a/site/src/app/api/analytics/rollup/route.ts b/site/src/app/api/analytics/rollup/route.ts
deleted file mode 100644
index 1e99739e..00000000
--- a/site/src/app/api/analytics/rollup/route.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-import { NextResponse } from "next/server";
-
-import { getObject, R2NotConfiguredError } from "@/cut/server/cloud/r2";
-import { ROLLUP_KEY } from "@/lib/analytics/pipeline";
-import { withSuperUser } from "@/lib/donkey-api-auth";
-
-// The consolidated analytics JSON, passed through exactly as the nightly job
-// wrote it. It carries emails and balances, so it stays behind the super-user
-// gate; the dashboard reads this and never the database.
-export const GET = withSuperUser(async () => {
- let rollup;
- try {
- rollup = await getObject(ROLLUP_KEY);
- } catch (e) {
- if (e instanceof R2NotConfiguredError) {
- return NextResponse.json({ error: "Storage is not configured." }, { status: 503 });
- }
- throw e;
- }
- if (!rollup) return NextResponse.json({ error: "No rollup yet." }, { status: 404 });
- return new NextResponse(new Uint8Array(rollup.bytes), {
- headers: { "Cache-Control": "no-store", "Content-Type": "application/json" },
- });
-});
diff --git a/site/src/app/api/analytics/summary/route.ts b/site/src/app/api/analytics/summary/route.ts
new file mode 100644
index 00000000..b0a39718
--- /dev/null
+++ b/site/src/app/api/analytics/summary/route.ts
@@ -0,0 +1,21 @@
+import { NextResponse } from "next/server";
+
+import { noRollupResponse, readRollup, storageProblemResponse } from "@/lib/analytics/read";
+import { summarizeRollup } from "@/lib/analytics/summarize";
+import { withSuperUser } from "@/lib/donkey-api-auth";
+
+// The nightly rollup folded to the numbers a dashboard draws: one point per
+// day, the headline counts, and the billing and referral blocks. Tens of
+// kilobytes whatever the account table does, and no account's email or credit
+// balance rides along — people come from /api/analytics/users, a page at a
+// time. Super users only, like everything under /api/analytics.
+export const GET = withSuperUser(async () => {
+ let rollup;
+ try {
+ rollup = await readRollup();
+ } catch (e) {
+ return storageProblemResponse(e);
+ }
+ if (!rollup) return noRollupResponse();
+ return NextResponse.json(summarizeRollup(rollup), { headers: { "Cache-Control": "no-store" } });
+});
diff --git a/site/src/app/api/analytics/users/route.ts b/site/src/app/api/analytics/users/route.ts
new file mode 100644
index 00000000..3a7dd326
--- /dev/null
+++ b/site/src/app/api/analytics/users/route.ts
@@ -0,0 +1,62 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+
+import { noRollupResponse, readRollup, storageProblemResponse } from "@/lib/analytics/read";
+import {
+ rankUsers,
+ USER_SORTS,
+ USERS_PAGE_SIZE,
+ type AnalyticsUsersPage,
+ type UserSort,
+} from "@/lib/analytics/rank";
+import { withSuperUser } from "@/lib/donkey-api-auth";
+
+// One page of accounts out of the nightly rollup, ranked by the sort the
+// caller asked for. The ranking runs here, over the whole window, so a page is
+// the top of a real order rather than a slice of an arbitrary one, and the
+// dashboard holds the rows it shows instead of every account there is.
+//
+// `cursor` is the rank offset of the next page. Ranks are only stable within
+// one rollup, so the answer carries `generatedAt` and a caller keys its pages
+// on it: a new rollup starts a new list.
+const sortIds = USER_SORTS.map((sort) => sort.id) as [UserSort, ...UserSort[]];
+
+const querySchema = z.object({
+ sort: z.enum(sortIds).default("active"),
+ cursor: z.coerce.number().int().min(0).default(0),
+ limit: z.coerce.number().int().min(1).max(200).default(USERS_PAGE_SIZE),
+ // A specific account, for a panel that needs one person's row.
+ id: z.string().min(1).optional(),
+});
+
+export const GET = withSuperUser(async (request) => {
+ const params = querySchema.safeParse(Object.fromEntries(request.nextUrl.searchParams));
+ if (!params.success) {
+ return NextResponse.json({ error: "Bad query." }, { status: 400 });
+ }
+ const { cursor, id, limit, sort } = params.data;
+
+ let rollup;
+ try {
+ rollup = await readRollup();
+ } catch (e) {
+ return storageProblemResponse(e);
+ }
+ if (!rollup) return noRollupResponse();
+
+ const ranked = rankUsers(rollup, sort);
+ const page = id
+ ? ranked.filter((user) => user.id === id)
+ : ranked.slice(cursor, cursor + limit);
+ const end = cursor + limit;
+
+ const body: AnalyticsUsersPage = {
+ generatedAt: rollup.generatedAt,
+ nextCursor: id || end >= ranked.length ? null : end,
+ pageSize: limit,
+ sorts: USER_SORTS,
+ total: ranked.length,
+ users: page,
+ };
+ return NextResponse.json(body, { headers: { "Cache-Control": "no-store" } });
+});
diff --git a/site/src/app/su/SuShell.tsx b/site/src/app/su/SuShell.tsx
index 24aaf9c8..1e1e0efa 100644
--- a/site/src/app/su/SuShell.tsx
+++ b/site/src/app/su/SuShell.tsx
@@ -7,7 +7,7 @@ import { SuCrumbProvider } from "@/app/su/SuCrumb";
import { SuHeader } from "@/app/su/SuHeader";
import { SuSidebar } from "@/app/su/SuSidebar";
import { SidebarProvider, useSidebar } from "@/components/ui/sidebar";
-import { useWarmAnalyticsRollup } from "@/queries/analytics";
+import { useWarmAnalyticsSummary } from "@/queries/analytics";
// The section's two-pane shell. It mounts only for a super user: the proxy
// (src/proxy.ts) has already checked the session and the role before the
@@ -17,7 +17,7 @@ import { useWarmAnalyticsRollup } from "@/queries/analytics";
// and the rail share one open state: on a phone the rail is a sheet the
// header opens, on a wider screen it is the fixed left column.
export function SuShell({ children }: { children: ReactNode }) {
- useWarmAnalyticsRollup();
+ useWarmAnalyticsSummary();
// The section paints the white product surface. `app-surface` is what
// repoints the --background token for the whole document (see globals.css),
// so it belongs on the outermost element here.
diff --git a/site/src/app/su/analytics/product/page.tsx b/site/src/app/su/analytics/product/page.tsx
index 541ec982..169883ba 100644
--- a/site/src/app/su/analytics/product/page.tsx
+++ b/site/src/app/su/analytics/product/page.tsx
@@ -1,7 +1,7 @@
"use client";
import { ArrowUpRight, Check, Mail } from "lucide-react";
-import { useMemo, useRef, useState } from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
import {
Area,
AreaChart,
@@ -23,12 +23,9 @@ import {
ChartTooltipContent,
type ChartConfig,
} from "@/components/ui/chart";
-import type {
- AnalyticsBilling,
- AnalyticsReferrals,
- AnalyticsRollup,
- AnalyticsRollupUser,
-} from "@/lib/analytics/schema";
+import type { AnalyticsBilling, AnalyticsReferrals } from "@/lib/analytics/schema";
+import { USER_SORTS, type RankedUser, type UserSort } from "@/lib/analytics/rank";
+import { workBitsOf, type AnalyticsSummary } from "@/lib/analytics/summarize";
import { REFERRAL_SOURCES } from "@/lib/onboarding/sequence";
import { useLocalPref } from "@/cut/lib/uiState";
import { cn } from "@/lib/utils";
@@ -37,14 +34,16 @@ import { OutreachComposeDialog } from "@/app/su/outreach/ComposeDialog";
import { Button } from "@/components/ui/button";
import { SuStandIn } from "@/app/su/SuStandIn";
import { useRowWindow } from "@/app/su/analytics/rowWindow";
-import { useAnalyticsRollup } from "@/queries/analytics";
+import { useAnalyticsSummary, useAnalyticsUsers } from "@/queries/analytics";
import { ApiError } from "@/queries/apiClient";
import { useOutreachAction, type OutreachRow } from "@/queries/outreach";
-// Everything here renders the nightly rollup (analytics/rollup.json via
-// /api/analytics/rollup) — stale until the next job run by design. "Active" is
-// any source bit for the day; "working" narrows to the DB event sources, i.e.
-// the user did something beyond opening the app.
+// Everything here renders the nightly rollup — stale until the next job run by
+// design. The charts read the folded numbers (/api/analytics/summary) and the
+// grid pages through the accounts (/api/analytics/users), so the page never
+// holds the whole account table. "Active" is any source bit for the day;
+// "working" narrows to the DB event sources, i.e. the user did something
+// beyond opening the app.
// Formatting a day goes through the locale machinery, which is slow enough
// that a label per grid cell held the page for over a second. The rollup
@@ -87,102 +86,6 @@ function storageTone(bytes: number, quotaBytes: number | null | undefined): stri
return "";
}
-// Activity is null for a day the pipeline never extracted: the masks are
-// empty because there was nothing to read, which is not the same as a day
-// nobody worked. Those days leave a gap in the charts and an unknown dot in
-// the grid. Signups come from the user snapshot, so they are always known.
-type DayPoint = {
- day: string;
- active: number | null;
- working: number | null;
- signups: number;
- totalRegistered: number;
-};
-
-type RollupView = {
- series: DayPoint[];
- workBits: number;
- missingDays: Set<string>;
- registered: number;
- signups7d: number;
- signupsWindow: number;
- activeYesterday: number | null;
- active7d: number | null;
- activePrior7d: number | null;
- totalBalanceMicros: bigint;
-};
-
-function deriveView(rollup: AnalyticsRollup): RollupView {
- const workBits = rollup.sources.reduce(
- (mask, source, i) => (source === "posthog" ? mask : mask | (1 << i)),
- 0,
- );
-
- const signupsByDay = new Map<string, number>();
- for (const user of rollup.users) {
- const day = user.registeredAt.slice(0, 10);
- signupsByDay.set(day, (signupsByDay.get(day) ?? 0) + 1);
- }
-
- // Cumulative registrations start from everyone who signed up before the
- // window, so the total line carries the real base, not zero.
- const firstDay = rollup.days[0] ?? "";
- let totalRegistered = rollup.users.filter(
- (user) => user.registeredAt.slice(0, 10) < firstDay,
- ).length;
-
- // Any day the consolidation could not read a source for is undercounted at
- // best, and indistinguishable from a quiet day — so it reports as unknown
- // rather than as a number the reader would trust.
- const missingDays = new Set(rollup.missing.map((entry) => entry.day));
-
- const series = rollup.days.map((day, i) => {
- const signups = signupsByDay.get(day) ?? 0;
- totalRegistered += signups;
- if (missingDays.has(day)) {
- return { active: null, day, signups, totalRegistered, working: null };
- }
- let active = 0;
- let working = 0;
- for (const user of rollup.users) {
- const mask = user.activity[i] ?? 0;
- if (mask !== 0) active++;
- if ((mask & workBits) !== 0) working++;
- }
- return { active, day, signups, totalRegistered, working };
- });
-
- // Null when the whole range went unextracted; otherwise it counts over the
- // days there is data for, so one missing day doesn't drag the number down.
- const activeInRange = (from: number, to: number): number | null => {
- const known: number[] = [];
- for (let i = Math.max(0, from); i < to; i++) {
- if (!missingDays.has(rollup.days[i])) known.push(i);
- }
- if (known.length === 0) return null;
- let count = 0;
- for (const user of rollup.users) {
- if (known.some((i) => (user.activity[i] ?? 0) !== 0)) count++;
- }
- return count;
- };
-
- const len = rollup.days.length;
- const last7 = rollup.days.slice(-7);
- return {
- active7d: activeInRange(len - 7, len),
- activePrior7d: activeInRange(len - 14, len - 7),
- activeYesterday: series[len - 1]?.active ?? null,
- missingDays,
- registered: rollup.users.length,
- series,
- signups7d: last7.reduce((sum, day) => sum + (signupsByDay.get(day) ?? 0), 0),
- signupsWindow: series.reduce((sum, point) => sum + point.signups, 0),
- totalBalanceMicros: rollup.users.reduce((sum, u) => sum + BigInt(u.balanceMicros), BigInt(0)),
- workBits,
- };
-}
-
// One chart point per day, twice over: `series` holds the per-source answer
// counts of that day (the stacked bars), `cumulative` the running totals per
// source plus the running total of users who answered (the trend lines).
@@ -637,63 +540,16 @@ function ActivityDot({
);
}
-// Ranking weight decays with age — a working day two weeks back counts half
-// of yesterday's — so the list leads with who is active now. A day the user
-// only visited counts a fraction of one they worked.
-const HALF_LIFE_DAYS = 14;
-const VISIT_WEIGHT = 0.35;
-
-const USER_SORTS = [
- { id: "active", label: "Most active" },
- { id: "recent", label: "Recently active" },
- { id: "joined", label: "Newest" },
- { id: "paid", label: "Top paid" },
-] as const;
-type UserSort = (typeof USER_SORTS)[number]["id"];
-
-type RankedUser = AnalyticsRollupUser & {
- score: number;
- activeDays: number;
- /** Index into rollup.days of the last day with any activity; -1 for never. */
- lastActive: number;
-};
-
-function rankUsers(rollup: AnalyticsRollup, workBits: number, sort: UserSort): RankedUser[] {
- const len = rollup.days.length;
- const ranked = rollup.users.map((user) => {
- let score = 0;
- let activeDays = 0;
- let lastActive = -1;
- for (let i = 0; i < len; i++) {
- const mask = user.activity[i] ?? 0;
- if (mask === 0) continue;
- activeDays++;
- lastActive = i;
- score +=
- ((mask & workBits) !== 0 ? 1 : VISIT_WEIGHT) * 0.5 ** ((len - 1 - i) / HALF_LIFE_DAYS);
- }
- return { ...user, activeDays, lastActive, score };
- });
- const funded = (user: RankedUser) => Number(user.fundedMicros ?? "0");
- const by: Record<UserSort, (a: RankedUser, b: RankedUser) => number> = {
- active: (a, b) => b.score - a.score,
- joined: (a, b) => (a.registeredAt < b.registeredAt ? 1 : -1),
- paid: (a, b) => funded(b) - funded(a) || b.score - a.score,
- recent: (a, b) => b.lastActive - a.lastActive || b.score - a.score,
- };
- // Our own accounts sink to the bottom under every sort: they are active
- // every day and would otherwise own the top of the list.
- return ranked.sort(
- (a, b) => Number(a.superUser === true) - Number(b.superUser === true) || by[sort](a, b),
- );
-}
+// How close to the end of the loaded rows the window may come before the next
+// page is asked for, so scrolling meets rows that are already there.
+const LOAD_AHEAD_ROWS = 20;
function ActivityGrid({
- rollup,
+ summary,
workBits,
missingDays,
}: {
- rollup: AnalyticsRollup;
+ summary: AnalyticsSummary;
workBits: number;
missingDays: Set<string>;
}) {
@@ -702,7 +558,14 @@ function ActivityGrid({
"active",
(v) => USER_SORTS.some((option) => option.id === v),
);
- const users = useMemo(() => rankUsers(rollup, workBits, sort), [rollup, workBits, sort]);
+ // Ranked on the server, a page at a time: the browser holds the rows it has
+ // scrolled to, never every account.
+ const paged = useAnalyticsUsers(sort, summary.generatedAt);
+ const users: RankedUser[] = useMemo(
+ () => paged.data?.pages.flatMap((page) => page.users) ?? [],
+ [paged.data],
+ );
+ const total = paged.data?.pages[0]?.total ?? 0;
// A row marks itself processed on click and stays marked across visits, so
// a pass through the list can pick up where it left off.
const [processedIds, setProcessedIds] = useLocalPref<string[]>(
@@ -715,9 +578,16 @@ function ActivityGrid({
setProcessedIds((prev) => (prev.includes(id) ? prev.filter((x) => x !== id) : [...prev, id]));
const markProcessed = (id: string) =>
setProcessedIds((prev) => (prev.includes(id) ? prev : [...prev, id]));
- const processedCount = users.reduce((n, user) => n + (processed.has(user.id) ? 1 : 0), 0);
+ const processedCount = processedIds.length;
const body = useRef<HTMLTableSectionElement>(null);
const rows = useRowWindow(users.length, body);
+ // Scrolling near the end of what is loaded pulls the next page.
+ const { fetchNextPage, hasNextPage, isFetchingNextPage } = paged;
+ useEffect(() => {
+ if (hasNextPage && !isFetchingNextPage && rows.end >= users.length - LOAD_AHEAD_ROWS) {
+ void fetchNextPage();
+ }
+ }, [fetchNextPage, hasNextPage, isFetchingNextPage, rows.end, users.length]);
// A row's email button puts the account on the outreach list and opens the
// same note the Outreach tab writes, so a busy user gets a word or a deal
// from here.
@@ -746,8 +616,8 @@ function ActivityGrid({
// before any horizontal scroll; each column keeps its index into the
// activity masks.
const columns = useMemo(
- () => rollup.days.map((day, i) => ({ day, i, unknown: missingDays.has(day) })).reverse(),
- [rollup.days, missingDays],
+ () => summary.days.map((day, i) => ({ day, i, unknown: missingDays.has(day) })).reverse(),
+ [summary.days, missingDays],
);
const columnCount = columns.length + 1;
@@ -765,7 +635,7 @@ function ActivityGrid({
? `${user.email} — ${when}: no data`
: mask === 0
? `${user.email} — ${when}: inactive`
- : `${user.email} — ${when}: ${rollup.sources.filter((_, i) => (mask & (1 << i)) !== 0).join(", ")}`;
+ : `${user.email} — ${when}: ${summary.sources.filter((_, i) => (mask & (1 << i)) !== 0).join(", ")}`;
};
return (
@@ -774,8 +644,9 @@ function ActivityGrid({
<div>
<p className="font-medium">User activity</p>
<p className="text-sm text-muted-foreground">
- One dot per user per day, last {rollup.days.length} days
+ One dot per user per day, last {summary.days.length} days
{missingDays.size > 0 && ` · ${missingDays.size} without data`}
+ {total > 0 && ` · ${users.length.toLocaleString("en-US")} of ${total.toLocaleString("en-US")} accounts loaded`}
{processedCount > 0 && (
<>
{" "}
@@ -893,7 +764,7 @@ function ActivityGrid({
<span className="block text-xs text-muted-foreground">
{user.activeDays === 0
? "no activity"
- : `${user.activeDays} active ${user.activeDays === 1 ? "day" : "days"} · last ${formatDay(rollup.days[user.lastActive])}`}{" "}
+ : `${user.activeDays} active ${user.activeDays === 1 ? "day" : "days"} · last ${formatDay(summary.days[user.lastActive])}`}{" "}
· joined {formatDay(user.registeredAt.slice(0, 10))} ·{" "}
{formatMicros(user.balanceMicros)}
{user.fundedMicros !== undefined && (
@@ -934,6 +805,16 @@ function ActivityGrid({
<td colSpan={columnCount} className="p-0" style={{ height: rows.below }} />
</tr>
)}
+ {(paged.isPending || paged.isFetchingNextPage || paged.isError) && (
+ <tr>
+ <td
+ className="sticky left-0 z-10 bg-card py-2 pl-6 text-sm text-muted-foreground"
+ colSpan={columnCount}
+ >
+ {paged.isError ? "Couldn\u2019t load accounts." : "Loading accounts\u2026"}
+ </td>
+ </tr>
+ )}
</tbody>
</table>
</div>
@@ -968,16 +849,12 @@ function ActivityGrid({
}
export default function SuAnalyticsPage() {
- const rollup = useAnalyticsRollup();
- const view = useMemo(
- () => (rollup.data ? deriveView(rollup.data) : null),
- [rollup.data],
- );
+ const summaryQuery = useAnalyticsSummary();
const referrals = useMemo(
- () => (rollup.data?.referrals ? deriveReferrals(rollup.data.referrals) : null),
- [rollup.data],
+ () => (summaryQuery.data?.referrals ? deriveReferrals(summaryQuery.data.referrals) : null),
+ [summaryQuery.data],
);
- const billingSection = rollup.data?.billing;
+ const billingSection = summaryQuery.data?.billing;
const revenue = useMemo(
() => (billingSection && "days" in billingSection ? deriveRevenue(billingSection) : null),
[billingSection],
@@ -993,10 +870,10 @@ export default function SuAnalyticsPage() {
const tiles = useReorder("su-analytics-tile-order", TILE_IDS);
const cards = useReorder("su-analytics-card-order", CARD_IDS);
- if (rollup.isPending) return <SuStandIn />;
+ if (summaryQuery.isPending) return <SuStandIn />;
- if (rollup.error || !view || !rollup.data) {
- const noData = rollup.error instanceof ApiError && rollup.error.status === 404;
+ if (summaryQuery.error || !summaryQuery.data) {
+ const noData = summaryQuery.error instanceof ApiError && summaryQuery.error.status === 404;
return (
<div className="rounded-xl border bg-card p-5 text-sm text-muted-foreground">
{noData
@@ -1006,16 +883,18 @@ export default function SuAnalyticsPage() {
);
}
- const data = rollup.data;
+ const summary = summaryQuery.data;
+ const missingDays = new Set(summary.missing);
+ const workBits = workBitsOf(summary.sources);
const deltaPct =
- view.active7d !== null && view.activePrior7d !== null && view.activePrior7d > 0
- ? `${(((view.active7d - view.activePrior7d) / view.activePrior7d) * 100).toFixed(1)}%`
+ summary.active7d !== null && summary.activePrior7d !== null && summary.activePrior7d > 0
+ ? `${(((summary.active7d - summary.activePrior7d) / summary.activePrior7d) * 100).toFixed(1)}%`
: null;
const count = (n: number | null) => (n === null ? "—" : n.toLocaleString("en-US"));
- const lastDay = data.days[data.days.length - 1];
+ const lastDay = summary.days[summary.days.length - 1];
// Billing is absent from rollups written before it shipped (or before it
// took this shape); the next run fills it in.
- const billing = data.billing && "days" in data.billing ? data.billing : undefined;
+ const billing = summary.billing && "days" in summary.billing ? summary.billing : undefined;
const churnBase = billing ? billing.subscribers + billing.churned : 0;
const staleBilling = "not in this rollup yet — run analytics";
const links = billing ? stripeLinks(billing.dashboardUrl) : undefined;
@@ -1025,27 +904,27 @@ export default function SuAnalyticsPage() {
registered: (
<StatTile
label="Registered users"
- value={view.registered.toLocaleString("en-US")}
- sub={`+${view.signups7d} in the last 7 days`}
+ value={summary.registered.toLocaleString("en-US")}
+ sub={`+${summary.signups7d} in the last 7 days`}
/>
),
activeYesterday: (
<StatTile
label="Active yesterday"
- value={count(view.activeYesterday)}
+ value={count(summary.activeYesterday)}
sub={
- view.activeYesterday === null
+ summary.activeYesterday === null
? `no extract for ${formatDay(lastDay)} yet`
- : `of ${view.registered.toLocaleString("en-US")} registered`
+ : `of ${summary.registered.toLocaleString("en-US")} registered`
}
/>
),
active7d: (
<StatTile
label="Active last 7 days"
- value={count(view.active7d)}
+ value={count(summary.active7d)}
sub={
- view.active7d === null ? (
+ summary.active7d === null ? (
"no extracts for the last 7 days"
) : deltaPct === null ? (
"no prior-week baseline"
@@ -1069,7 +948,7 @@ export default function SuAnalyticsPage() {
balance: (
<StatTile
label="Outstanding balance"
- value={formatMicros(view.totalBalanceMicros)}
+ value={formatMicros(summary.balanceMicros)}
sub="credits across all accounts"
/>
),
@@ -1154,13 +1033,13 @@ export default function SuAnalyticsPage() {
<ChartCard
title="Active users"
subtitle={
- view.missingDays.size > 0
- ? `Daily actives, last 60 days · ${view.missingDays.size} days without data are left blank`
+ missingDays.size > 0
+ ? `Daily actives, last 60 days · ${missingDays.size} days without data are left blank`
: "Daily actives, last 60 days"
}
>
<ChartContainer className="w-full" config={activesConfig}>
- <AreaChart accessibilityLayer data={view.series} margin={{ left: -16 }}>
+ <AreaChart accessibilityLayer data={summary.series} margin={{ left: -16 }}>
<CartesianGrid vertical={false} />
<XAxis
axisLine={false}
@@ -1207,10 +1086,10 @@ export default function SuAnalyticsPage() {
signups: (
<ChartCard
title="Signups"
- subtitle={`New registrations per day · ${view.signupsWindow.toLocaleString("en-US")} in the last 60 days`}
+ subtitle={`New registrations per day · ${summary.signupsWindow.toLocaleString("en-US")} in the last 60 days`}
>
<ChartContainer className="w-full" config={signupsConfig}>
- <BarChart accessibilityLayer data={view.series} margin={{ left: -16 }}>
+ <BarChart accessibilityLayer data={summary.series} margin={{ left: -16 }}>
<CartesianGrid vertical={false} />
<XAxis
axisLine={false}
@@ -1235,7 +1114,7 @@ export default function SuAnalyticsPage() {
totalRegistered: (
<ChartCard title="Total registered" subtitle="Cumulative registrations, last 60 days">
<ChartContainer className="max-h-56 w-full" config={totalRegisteredConfig}>
- <AreaChart accessibilityLayer data={view.series} margin={{ left: -16 }}>
+ <AreaChart accessibilityLayer data={summary.series} margin={{ left: -16 }}>
<CartesianGrid vertical={false} />
<XAxis
axisLine={false}
@@ -1537,11 +1416,11 @@ export default function SuAnalyticsPage() {
))}
</div>
- <ActivityGrid missingDays={view.missingDays} rollup={data} workBits={view.workBits} />
+ <ActivityGrid missingDays={missingDays} summary={summary} workBits={workBits} />
<p className="text-xs text-muted-foreground">
From the nightly rollup generated{" "}
- {new Date(data.generatedAt).toLocaleString("en-US", { timeZoneName: "short" })}.
+ {new Date(summary.generatedAt).toLocaleString("en-US", { timeZoneName: "short" })}.
</p>
</div>
);
diff --git a/site/src/app/su/outreach/UserHistory.tsx b/site/src/app/su/outreach/UserHistory.tsx
index d4f4d75f..4bb4c066 100644
--- a/site/src/app/su/outreach/UserHistory.tsx
+++ b/site/src/app/su/outreach/UserHistory.tsx
@@ -4,7 +4,7 @@ import { useState } from "react";
import { formatBytes } from "@/lib/bytes";
import { formatUsdPlain } from "@/lib/credits/format-usd";
-import { useAnalyticsRollup } from "@/queries/analytics";
+import { useAnalyticsSummary, useAnalyticsUser } from "@/queries/analytics";
import { type OutreachRow } from "@/queries/outreach";
import { useUserHistory, type UserHistory, type UserHistoryEmail } from "@/queries/userHistory";
@@ -137,11 +137,12 @@ const TONE: Record<Event["tone"], string> = {
export function UserHistoryPanel({ target }: { target: OutreachRow }) {
const history = useUserHistory(target.userId);
- const rollup = useAnalyticsRollup();
- const rolled = rollup.data?.users.find((user) => user.id === target.userId);
- // The rollup's billing events name the payer by email; the window's charges
- // for this person are the ones with theirs.
- const charges = (rollup.data?.billing?.events ?? []).filter(
+ // This one account's row out of the nightly rollup, and the billing events
+ // the summary carries. The rollup's events name the payer by email; the
+ // window's charges for this person are the ones with theirs.
+ const rolled = useAnalyticsUser(target.userId).data;
+ const summary = useAnalyticsSummary();
+ const charges = (summary.data?.billing?.events ?? []).filter(
(event) => event.email === target.email && event.kind !== "canceled" && event.amountMicros !== null,
);
// The moment the panel opened; offers and grants are read against it.
diff --git a/site/src/lib/analytics/pipeline.ts b/site/src/lib/analytics/pipeline.ts
index ab275291..594623b2 100644
--- a/site/src/lib/analytics/pipeline.ts
+++ b/site/src/lib/analytics/pipeline.ts
@@ -20,6 +20,11 @@ import {
analyticsPosthogDayFileSchema,
analyticsRollupSchema,
analyticsSnapshotFileSchema,
+ DAYS_PREFIX,
+ dayDbKey,
+ dayPosthogKey,
+ ROLLUP_KEY,
+ SNAPSHOT_KEY,
type AnalyticsDbDayFile,
type AnalyticsPosthogDayFile,
type AnalyticsRollup,
@@ -34,13 +39,6 @@ import { lapsedCreditByUser } from "@/lib/credits/inference";
import { REFERRAL_SOURCES } from "@/lib/onboarding/sequence";
import { prisma } from "@/lib/prisma";
-const ANALYTICS_PREFIX = "analytics/";
-const DAYS_PREFIX = `${ANALYTICS_PREFIX}days/`;
-export const dayDbKey = (day: string) => `${DAYS_PREFIX}${day}/db.json`;
-export const dayPosthogKey = (day: string) => `${DAYS_PREFIX}${day}/posthog.json`;
-export const SNAPSHOT_KEY = `${ANALYTICS_PREFIX}snapshot.json`;
-export const ROLLUP_KEY = `${ANALYTICS_PREFIX}rollup.json`;
-
const WINDOW_DAYS = 60;
// Backfill after an outage converges over nights instead of one heavy run.
const MAX_EXTRACT_DAYS_PER_RUN = 5;
@@ -336,7 +334,7 @@ async function consolidate(
// day at a time, so a 60-day window holds one day's ids at once; nothing here
// touches storage or the clock. Every rollup in storage came out of this
// function, so the fixture the phone's model is tested against comes out of
-// it too (rollup-fixture.ts): a change to what the dashboard is served
+// it too (summary-fixture.ts): a change to what the dashboard is served
// reaches that test in the same change.
export async function buildRollup(input: {
snapshot: AnalyticsSnapshotFile;
diff --git a/site/src/lib/analytics/rank.ts b/site/src/lib/analytics/rank.ts
new file mode 100644
index 00000000..6d1958bb
--- /dev/null
+++ b/site/src/lib/analytics/rank.ts
@@ -0,0 +1,73 @@
+// How the user list is ordered, and the per-user numbers that ordering needs.
+// It runs on the server so a dashboard reads a page of people instead of the
+// whole account table.
+import { workBitsOf } from "@/lib/analytics/summarize";
+import type { AnalyticsRollup, AnalyticsRollupUser } from "@/lib/analytics/schema";
+
+// Ranking weight decays with age — a working day two weeks back counts half of
+// yesterday's — so the list leads with who is active now. A day the user only
+// visited counts a fraction of one they worked.
+const HALF_LIFE_DAYS = 14;
+const VISIT_WEIGHT = 0.35;
+
+/** How many accounts one page carries. The server owns it: a client asks for
+ * the first page, then follows `nextCursor` until it is null. */
+export const USERS_PAGE_SIZE = 50;
+
+export const USER_SORTS = [
+ { id: "active", label: "Most active" },
+ { id: "recent", label: "Recently active" },
+ { id: "joined", label: "Newest" },
+ { id: "paid", label: "Top paid" },
+] as const;
+export type UserSort = (typeof USER_SORTS)[number]["id"];
+
+export type RankedUser = AnalyticsRollupUser & {
+ score: number;
+ activeDays: number;
+ /** Index into the rollup's days of the last day with any activity; -1 for never. */
+ lastActive: number;
+};
+
+/** One page of ranked accounts, as /api/analytics/users answers it. Ranks
+ * hold only within one rollup, so a caller keys its pages on `generatedAt`. */
+export type AnalyticsUsersPage = {
+ generatedAt: string;
+ users: RankedUser[];
+ total: number;
+ nextCursor: number | null;
+ /** The orders this API ranks by, so every client offers the same ones. */
+ sorts: readonly { id: UserSort; label: string }[];
+ pageSize: number;
+};
+
+export function rankUsers(rollup: AnalyticsRollup, sort: UserSort): RankedUser[] {
+ const workBits = workBitsOf(rollup.sources);
+ const len = rollup.days.length;
+ const ranked = rollup.users.map((user) => {
+ let score = 0;
+ let activeDays = 0;
+ let lastActive = -1;
+ for (let i = 0; i < len; i++) {
+ const mask = user.activity[i] ?? 0;
+ if (mask === 0) continue;
+ activeDays++;
+ lastActive = i;
+ score +=
+ ((mask & workBits) !== 0 ? 1 : VISIT_WEIGHT) * 0.5 ** ((len - 1 - i) / HALF_LIFE_DAYS);
+ }
+ return { ...user, activeDays, lastActive, score };
+ });
+ const funded = (user: RankedUser) => Number(user.fundedMicros ?? "0");
+ const by: Record<UserSort, (a: RankedUser, b: RankedUser) => number> = {
+ active: (a, b) => b.score - a.score,
+ joined: (a, b) => (a.registeredAt < b.registeredAt ? 1 : -1),
+ paid: (a, b) => funded(b) - funded(a) || b.score - a.score,
+ recent: (a, b) => b.lastActive - a.lastActive || b.score - a.score,
+ };
+ // Our own accounts sink to the bottom under every sort: they are active
+ // every day and would otherwise own the top of the list.
+ return ranked.sort(
+ (a, b) => Number(a.superUser === true) - Number(b.superUser === true) || by[sort](a, b),
+ );
+}
diff --git a/site/src/lib/analytics/read.ts b/site/src/lib/analytics/read.ts
new file mode 100644
index 00000000..156ef45e
--- /dev/null
+++ b/site/src/lib/analytics/read.ts
@@ -0,0 +1,26 @@
+// Server-side access to the stored rollup. The su routes fold it into what a
+// caller asked for — a summary, or one page of people — so the file itself
+// never goes over the wire.
+import { NextResponse } from "next/server";
+
+import { getObject, R2NotConfiguredError } from "@/cut/server/cloud/r2";
+import { analyticsRollupSchema, ROLLUP_KEY, type AnalyticsRollup } from "@/lib/analytics/schema";
+
+/** The stored rollup, or null when the nightly job hasn't written one. */
+export async function readRollup(): Promise<AnalyticsRollup | null> {
+ const stored = await getObject(ROLLUP_KEY);
+ if (!stored) return null;
+ return analyticsRollupSchema.parse(JSON.parse(stored.bytes.toString("utf8")));
+}
+
+/** The answer when there is no rollup to read: a deploy without storage, or a
+ * job that has not run yet. */
+export const noRollupResponse = () =>
+ NextResponse.json({ error: "No rollup yet." }, { status: 404 });
+
+export function storageProblemResponse(error: unknown): NextResponse {
+ if (error instanceof R2NotConfiguredError) {
+ return NextResponse.json({ error: "Storage is not configured." }, { status: 503 });
+ }
+ throw error;
+}
diff --git a/site/src/lib/analytics/schema.ts b/site/src/lib/analytics/schema.ts
index 51c8d192..e47dd57b 100644
--- a/site/src/lib/analytics/schema.ts
+++ b/site/src/lib/analytics/schema.ts
@@ -5,6 +5,16 @@ import { z } from "zod";
export const ANALYTICS_ROLLUP_VERSION = 1;
+// Where the pipeline's files live in R2. Here rather than beside the pipeline
+// so a reader of one file — the rollup route — names it without loading the
+// extraction code.
+const ANALYTICS_PREFIX = "analytics/";
+export const DAYS_PREFIX = `${ANALYTICS_PREFIX}days/`;
+export const dayDbKey = (day: string) => `${DAYS_PREFIX}${day}/db.json`;
+export const dayPosthogKey = (day: string) => `${DAYS_PREFIX}${day}/posthog.json`;
+export const SNAPSHOT_KEY = `${ANALYTICS_PREFIX}snapshot.json`;
+export const ROLLUP_KEY = `${ANALYTICS_PREFIX}rollup.json`;
+
// Bit order for the per-day activity masks. Append-only: a new source takes
// the next bit so already-written rollups keep decoding.
export const ANALYTICS_DB_SOURCES = [
diff --git a/site/src/lib/analytics/summarize.ts b/site/src/lib/analytics/summarize.ts
new file mode 100644
index 00000000..f68af0ff
--- /dev/null
+++ b/site/src/lib/analytics/summarize.ts
@@ -0,0 +1,114 @@
+// The rollup folded to what a dashboard draws: one point per day plus the
+// headline numbers. Pure, and the only place those numbers are computed — the
+// su charts and the phone both read the result, so neither walks the user
+// list to draw a line.
+import type { AnalyticsRollup } from "@/lib/analytics/schema";
+
+/** Activity is null for a day the pipeline never extracted: nothing was read,
+ * which is not the same as a day nobody worked. Signups come from the user
+ * snapshot, so they are always known. */
+export type AnalyticsDayPoint = {
+ day: string;
+ active: number | null;
+ working: number | null;
+ signups: number;
+ totalRegistered: number;
+};
+
+export type AnalyticsSummary = {
+ version: number;
+ generatedAt: string;
+ /** Oldest → newest. */
+ days: string[];
+ /** Bit order of the activity masks the users page carries. */
+ sources: string[];
+ /** Days the consolidation could not read. */
+ missing: string[];
+ series: AnalyticsDayPoint[];
+ registered: number;
+ signups7d: number;
+ signupsWindow: number;
+ activeYesterday: number | null;
+ active7d: number | null;
+ activePrior7d: number | null;
+ /** Credit held across every account, BigInt micros as a decimal string. */
+ balanceMicros: string;
+ billing?: AnalyticsRollup["billing"];
+ referrals?: AnalyticsRollup["referrals"];
+};
+
+/** The mask bits that mean the person did something, not just opened the app. */
+export function workBitsOf(sources: string[]): number {
+ return sources.reduce((mask, source, i) => (source === "posthog" ? mask : mask | (1 << i)), 0);
+}
+
+export function summarizeRollup(rollup: AnalyticsRollup): AnalyticsSummary {
+ const workBits = workBitsOf(rollup.sources);
+ const missing = rollup.missing.map((entry) => entry.day);
+ const missingDays = new Set(missing);
+
+ const signupsByDay = new Map<string, number>();
+ for (const user of rollup.users) {
+ const day = user.registeredAt.slice(0, 10);
+ signupsByDay.set(day, (signupsByDay.get(day) ?? 0) + 1);
+ }
+
+ // Cumulative registrations start from everyone who signed up before the
+ // window, so the total line carries the real base, not zero.
+ const firstDay = rollup.days[0] ?? "";
+ let totalRegistered = rollup.users.filter(
+ (user) => user.registeredAt.slice(0, 10) < firstDay,
+ ).length;
+
+ const series = rollup.days.map((day, i) => {
+ const signups = signupsByDay.get(day) ?? 0;
+ totalRegistered += signups;
+ if (missingDays.has(day)) {
+ return { active: null, day, signups, totalRegistered, working: null };
+ }
+ let active = 0;
+ let working = 0;
+ for (const user of rollup.users) {
+ const mask = user.activity[i] ?? 0;
+ if (mask !== 0) active++;
+ if ((mask & workBits) !== 0) working++;
+ }
+ return { active, day, signups, totalRegistered, working };
+ });
+
+ // Null when the whole range went unextracted; otherwise it counts over the
+ // days there is data for, so one missing day doesn't drag the number down.
+ const activeInRange = (from: number, to: number): number | null => {
+ const known: number[] = [];
+ for (let i = Math.max(0, from); i < to; i++) {
+ if (!missingDays.has(rollup.days[i])) known.push(i);
+ }
+ if (known.length === 0) return null;
+ let count = 0;
+ for (const user of rollup.users) {
+ if (known.some((i) => (user.activity[i] ?? 0) !== 0)) count++;
+ }
+ return count;
+ };
+
+ const len = rollup.days.length;
+ return {
+ active7d: activeInRange(len - 7, len),
+ activePrior7d: activeInRange(len - 14, len - 7),
+ activeYesterday: series[len - 1]?.active ?? null,
+ balanceMicros: rollup.users
+ .reduce((sum, user) => sum + BigInt(user.balanceMicros), BigInt(0))
+ .toString(),
+ billing: rollup.billing,
+ days: rollup.days,
+ generatedAt: rollup.generatedAt,
+ missing,
+ referrals: rollup.referrals,
+ registered: rollup.users.length,
+ series,
+ signups7d: rollup.days.slice(-7).reduce((sum, day) => sum + (signupsByDay.get(day) ?? 0), 0),
+ signupsWindow: series.reduce((sum, point) => sum + point.signups, 0),
+ sources: rollup.sources,
+ version: rollup.version,
+ };
+}
diff --git a/site/src/queries/analytics.ts b/site/src/queries/analytics.ts
index 39618f0c..10ccb447 100644
--- a/site/src/queries/analytics.ts
+++ b/site/src/queries/analytics.ts
@@ -1,38 +1,82 @@
"use client";
-import { queryOptions, useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import {
+ infiniteQueryOptions,
+ queryOptions,
+ useInfiniteQuery,
+ useMutation,
+ useQuery,
+ useQueryClient,
+} from "@tanstack/react-query";
import { useEffect } from "react";
-import type { AnalyticsRollup } from "@/lib/analytics/schema";
+import type { AnalyticsUsersPage, UserSort } from "@/lib/analytics/rank";
+import type { AnalyticsSummary } from "@/lib/analytics/summarize";
import { apiFetch } from "@/queries/apiClient";
import type { AsyncJobStatus } from "@/queries/jobs";
-export const analyticsRollupQueryKey = ["analytics", "rollup"] as const;
+export const analyticsQueryKey = ["analytics"] as const;
+export const analyticsSummaryQueryKey = [...analyticsQueryKey, "summary"] as const;
-// Super-user only: the nightly job's consolidated rollup. It changes once a
-// night, and a manual run invalidates it, so a copy in memory stays good for
-// the session.
-const analyticsRollupQuery = queryOptions({
- queryFn: () => apiFetch<AnalyticsRollup>("/api/analytics/rollup"),
- queryKey: analyticsRollupQueryKey,
+// Super-user only: the nightly job's numbers, without the account table. It
+// changes once a night, and a manual run invalidates it, so a copy in memory
+// stays good for the session.
+const analyticsSummaryQuery = queryOptions({
+ queryFn: () => apiFetch<AnalyticsSummary>("/api/analytics/summary"),
+ queryKey: analyticsSummaryQueryKey,
staleTime: 60 * 60_000,
});
-export function useAnalyticsRollup() {
- return useQuery(analyticsRollupQuery);
+export function useAnalyticsSummary() {
+ return useQuery(analyticsSummaryQuery);
}
-// Pulls the rollup as soon as the super-user shell mounts, so the Product tab
+// Pulls the summary as soon as the super-user shell mounts, so the Product tab
// paints from memory when it opens.
-export function useWarmAnalyticsRollup() {
+export function useWarmAnalyticsSummary() {
const queryClient = useQueryClient();
useEffect(() => {
- void queryClient.prefetchQuery(analyticsRollupQuery);
+ void queryClient.prefetchQuery(analyticsSummaryQuery);
}, [queryClient]);
}
+// Accounts in rank order, a page at a time: ask for the first page, then
+// follow the cursor the server hands back — the phone reads the same endpoint
+// the same way, and the page size stays the server's to set. Ranks hold only
+// within one rollup, so the rollup's stamp is part of the key: a fresh rollup
+// is a fresh list rather than pages from two different orders stitched
+// together.
+function analyticsUsersQuery(sort: UserSort, generatedAt: string) {
+ return infiniteQueryOptions({
+ getNextPageParam: (page: AnalyticsUsersPage) => page.nextCursor,
+ initialPageParam: 0,
+ queryFn: ({ pageParam }) =>
+ apiFetch<AnalyticsUsersPage>(`/api/analytics/users?sort=${sort}&cursor=${pageParam}`),
+ queryKey: [...analyticsQueryKey, "users", generatedAt, sort] as const,
+ staleTime: 60 * 60_000,
+ });
+}
+
+export function useAnalyticsUsers(sort: UserSort, generatedAt: string) {
+ return useInfiniteQuery(analyticsUsersQuery(sort, generatedAt));
+}
+
+/** One account's row, for a panel that opens on a single person. */
+export function useAnalyticsUser(userId: string) {
+ return useQuery({
+ queryFn: async () => {
+ const page = await apiFetch<AnalyticsUsersPage>(
+ `/api/analytics/users?id=${encodeURIComponent(userId)}`,
+ );
+ return page.users[0] ?? null;
+ },
+ queryKey: [...analyticsQueryKey, "user", userId] as const,
+ staleTime: 60 * 60_000,
+ });
+}
+
// Super-user only: run the analytics-daily job now and follow it to
-// completion, then refetch the rollup so the dashboard updates in place.
+// completion, then refetch what the dashboard shows.
export function useRunAnalytics() {
const queryClient = useQueryClient();
return useMutation({
@@ -48,7 +92,6 @@ export function useRunAnalytics() {
if (job.state === "error") throw new Error(job.error ?? "The run failed.");
}
},
- onSuccess: () =>
- queryClient.invalidateQueries({ queryKey: analyticsRollupQueryKey }),
+ onSuccess: () => queryClient.invalidateQueries({ queryKey: analyticsQueryKey }),
});
}