Skip to content

Open source CapCut alternative.

TypeScript

refactor(site): keep only the analytics the site serves

David Le38a8882fedc9
diff --git a/.github/workflows/analytics-contract.yml b/.github/workflows/analytics-contract.yml
index 121c1fed..2c60d9be 100644
--- a/.github/workflows/analytics-contract.yml
+++ b/.github/workflows/analytics-contract.yml
@@ -1,13 +1,11 @@
name: Analytics Contract
-# The phone decodes the analytics summary the site folds out of its nightly
-# rollup. The site test rebuilds that summary from the pipeline and fails when
-# the checked-in fixture is behind it; the DonkeyKit tests decode that fixture.
-# Together they fail any change that moves the summary's shape without the
-# phone's model moving with it.
-#
-# The fixture's bytes follow whatever the pipeline imports (the referral
-# source list, the tier limits), so the site job runs on every site change.
+# The phone decodes the analytics summary the site folds out of the back
+# office's nightly rollup. The site test folds a checked-in rollup fixture,
+# a copy of the one the back office's own test builds with its pipeline, and
+# fails when the phone's fixtures are behind it; the DonkeyKit tests decode
+# those. Together they fail any change that moves the summary's shape without
+# the phone's model moving with it.
on:
workflow_dispatch:
@@ -50,12 +48,7 @@ jobs:
- name: Install dependencies
run: npm ci
- # The pipeline module constructs its database client on import; the
- # tests never open a connection, so a placeholder address is enough.
- name: Test
- env:
- DATABASE_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder
- DIRECT_URL: postgresql://placeholder:placeholder@localhost:5432/placeholder
run: ./node_modules/.bin/bun test src/lib/analytics
donkeykit:
diff --git a/apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-summary.json b/apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-summary.json
index 87d532fa..01f83391 100644
--- a/apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-summary.json
+++ b/apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-summary.json
@@ -4,18 +4,6 @@
"activeYesterday": 3,
"balanceMicros": "3500000",
"billing": {
- "canceling": [
- {
- "comment": "Export broke.",
- "customerId": "cus_p",
- "email": "[email protected]",
- "endsAt": "2026-09-08T00:00:00.000Z",
- "feedback": "switched_service",
- "requestedAt": "2026-08-31T10:00:00.000Z",
- "subscriptionId": "sub_1"
- }
- ],
- "churned": 1,
"dashboardUrl": "https://dashboard.stripe.com/test",
"days": [
"2026-08-30",
@@ -23,98 +11,110 @@
"2026-09-01",
"2026-09-02"
],
- "events": [
- {
- "amountMicros": "5000000",
- "customerId": "cus_n",
- "day": "2026-09-01",
- "detail": "topup",
- "email": "[email protected]",
- "ended": false,
- "endsAt": null,
- "kind": "paid",
- "objectId": "pi_2"
- },
+ "subscribers": 2,
+ "canceling": [
{
- "amountMicros": null,
+ "subscriptionId": "sub_1",
"customerId": "cus_p",
- "day": "2026-08-31",
- "detail": "switched_service · Export broke.",
"email": "[email protected]",
- "ended": false,
"endsAt": "2026-09-08T00:00:00.000Z",
- "kind": "canceled",
- "objectId": "sub_1"
- },
- {
- "amountMicros": "20000000",
- "customerId": "cus_d",
- "day": "2026-08-31",
- "detail": "Your card has insufficient funds.",
- "email": "[email protected]",
- "ended": false,
- "endsAt": null,
- "kind": "declined",
- "objectId": "pi_3"
- },
- {
- "amountMicros": "20000000",
- "customerId": "cus_p",
- "day": "2026-08-30",
- "detail": "pro",
- "email": "[email protected]",
- "ended": false,
- "endsAt": null,
- "kind": "paid",
- "objectId": "pi_1"
+ "requestedAt": "2026-08-31T10:00:00.000Z",
+ "feedback": "switched_service",
+ "comment": "Export broke."
}
],
+ "churned": 1,
"funded": 2,
"fundedMicros": "25000000",
+ "window": {
+ "paidMicros": "25000000",
+ "refundedMicros": "1000000",
+ "netMicros": "24000000",
+ "declinedMicros": "20000000",
+ "declinedCustomers": 1,
+ "abandonedCheckouts": 1,
+ "cancels": 1
+ },
"revenue": [
{
- "cancels": 0,
- "declinedMicros": "0",
- "otherMicros": "0",
"proMicros": "20000000",
+ "topupMicros": "0",
+ "otherMicros": "0",
"refundedMicros": "0",
- "topupMicros": "0"
+ "declinedMicros": "0",
+ "cancels": 0
},
{
- "cancels": 1,
- "declinedMicros": "20000000",
- "otherMicros": "0",
"proMicros": "0",
+ "topupMicros": "0",
+ "otherMicros": "0",
"refundedMicros": "0",
- "topupMicros": "0"
+ "declinedMicros": "20000000",
+ "cancels": 1
},
{
- "cancels": 0,
- "declinedMicros": "0",
- "otherMicros": "0",
"proMicros": "0",
+ "topupMicros": "5000000",
+ "otherMicros": "0",
"refundedMicros": "0",
- "topupMicros": "5000000"
+ "declinedMicros": "0",
+ "cancels": 0
},
{
- "cancels": 0,
- "declinedMicros": "0",
- "otherMicros": "0",
"proMicros": "0",
+ "topupMicros": "0",
+ "otherMicros": "0",
"refundedMicros": "1000000",
- "topupMicros": "0"
+ "declinedMicros": "0",
+ "cancels": 0
}
],
- "subscribers": 2,
- "window": {
- "abandonedCheckouts": 1,
- "cancels": 1,
- "declinedCustomers": 1,
- "declinedMicros": "20000000",
- "netMicros": "24000000",
- "paidMicros": "25000000",
- "refundedMicros": "1000000"
- }
+ "events": [
+ {
+ "day": "2026-09-01",
+ "kind": "paid",
+ "email": "[email protected]",
+ "customerId": "cus_n",
+ "objectId": "pi_2",
+ "amountMicros": "5000000",
+ "detail": "topup",
+ "endsAt": null,
+ "ended": false
+ },
+ {
+ "day": "2026-08-31",
+ "kind": "canceled",
+ "email": "[email protected]",
+ "customerId": "cus_p",
+ "objectId": "sub_1",
+ "amountMicros": null,
+ "detail": "switched_service · Export broke.",
+ "endsAt": "2026-09-08T00:00:00.000Z",
+ "ended": false
+ },
+ {
+ "day": "2026-08-31",
+ "kind": "declined",
+ "email": "[email protected]",
+ "customerId": "cus_d",
+ "objectId": "pi_3",
+ "amountMicros": "20000000",
+ "detail": "Your card has insufficient funds.",
+ "endsAt": null,
+ "ended": false
+ },
+ {
+ "day": "2026-08-30",
+ "kind": "paid",
+ "email": "[email protected]",
+ "customerId": "cus_p",
+ "objectId": "pi_1",
+ "amountMicros": "20000000",
+ "detail": "pro",
+ "endsAt": null,
+ "ended": false
+ }
+ ]
},
"days": [
"2026-08-30",
@@ -126,8 +126,20 @@
"2026-08-31"
],
"referrals": {
+ "sources": [
+ "discord",
+ "tiktok",
+ "instagram",
+ "youtube",
+ "reddit",
+ "search",
+ "friend",
+ "other"
+ ],
"days": [
{
+ "day": "2026-08-31",
+ "respondents": 1,
"counts": [
0,
0,
@@ -138,13 +150,13 @@
0,
1
],
- "day": "2026-08-31",
"others": [
"a podcast"
- ],
- "respondents": 1
+ ]
},
{
+ "day": "2026-09-01",
+ "respondents": 0,
"counts": [
0,
0,
@@ -155,20 +167,8 @@
0,
0
],
- "day": "2026-09-01",
- "others": [],
- "respondents": 0
+ "others": []
}
- ],
- "sources": [
- "discord",
- "tiktok",
- "instagram",
- "youtube",
- "reddit",
- "search",
- "friend",
- "other"
]
},
"registered": 3,
diff --git a/apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-users.json b/apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-users.json
index a17ce28c..ec8f8916 100644
--- a/apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-users.json
+++ b/apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-users.json
@@ -23,55 +23,55 @@
"total": 3,
"users": [
{
- "activity": [
- 65,
- 0,
- 64
- ],
- "balanceMicros": "3000000",
- "email": "[email protected]",
- "fundedMicros": "20000000",
"id": "u_pro",
+ "email": "[email protected]",
"name": "Pro Person",
"registeredAt": "2026-08-01T10:00:00.000Z",
+ "balanceMicros": "3000000",
+ "fundedMicros": "20000000",
"storageBytes": "1048576",
"storageQuotaBytes": 107374182400,
+ "activity": [
+ 65,
+ 0,
+ 64
+ ],
"activeDays": 2,
"lastActive": 2,
"score": 1.2557236642639067
},
{
- "activity": [
- 0,
- 0,
- 64
- ],
- "balanceMicros": "0",
- "email": "[email protected]",
- "fundedMicros": "5000000",
"id": "u_new",
+ "email": "[email protected]",
"name": "New Person",
"registeredAt": "2026-08-31T12:00:00.000Z",
+ "balanceMicros": "0",
+ "fundedMicros": "5000000",
"storageBytes": "0",
"storageQuotaBytes": 262144000,
+ "activity": [
+ 0,
+ 0,
+ 64
+ ],
"activeDays": 1,
"lastActive": 2,
"score": 0.35
},
{
- "activity": [
- 2,
- 0,
- 2
- ],
- "balanceMicros": "500000",
- "email": "[email protected]",
"id": "u_su",
+ "email": "[email protected]",
"name": "Super Person",
"registeredAt": "2026-07-01T00:00:00.000Z",
+ "balanceMicros": "500000",
"storageBytes": "0",
"storageQuotaBytes": null,
"superUser": true,
+ "activity": [
+ 2,
+ 0,
+ 2
+ ],
"activeDays": 2,
"lastActive": 2,
"score": 1.9057236642639066
diff --git a/docs/guides/ios-app.md b/docs/guides/ios-app.md
index dd685008..d2104761 100644
--- a/docs/guides/ios-app.md
+++ b/docs/guides/ios-app.md
@@ -213,11 +213,10 @@ platform has to be installed in Xcode (Settings → Components, or
The Analytics screen reads what the web dashboard reads: the summary the site
folds out of its nightly rollup — one point per day and the headline numbers,
no account rows — and the accounts a page at a time, following the cursor the
-API hands back. Two fixtures hold the sides together: the site builds them by
-running the real consolidation over a small fixed record and folding it the way
-the API does, so the JSON is what the server serves today; the checked-in copies live
-with the DonkeyKit tests, and the site's tests fail while one is behind the
-pipeline. The DonkeyKit tests decode them through the phone's model. Changing
+API hands back. Two fixtures hold the sides together: the site folds a copy of
+the back office's rollup fixture the way the API does; the checked-in copies
+live with the DonkeyKit tests, and the site's tests fail while one is behind.
+The DonkeyKit tests decode them through the phone's model. Changing
what either carries means regenerating them from `site/` with
`npm run analytics:summary-fixture` and running the DonkeyKit tests; the
Analytics Contract workflow runs both on every change to either side, and the
diff --git a/site/scripts/analytics-repair-credit-activity.ts b/site/scripts/analytics-repair-credit-activity.ts
deleted file mode 100644
index cec1f3da..00000000
--- a/site/scripts/analytics-repair-credit-activity.ts
+++ /dev/null
@@ -1,87 +0,0 @@
-// Run with --apply to repair the stored window after changing credit activity.
-// Each write checks the object's ETag so a concurrent refresh stops the repair.
-import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
-import { mkdtemp, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import { extractCreditActivity } from "../src/lib/analytics/pipeline";
-import {
- analyticsDbDayFileSchema,
- analyticsRollupSchema,
- dayDbKey,
- ROLLUP_KEY,
-} from "../src/lib/analytics/schema";
-import { prisma } from "../src/lib/prisma";
-
-const apply = process.argv.includes("--apply");
-const accountId = process.env.R2_ACCOUNT_ID;
-const accessKeyId = process.env.R2_ACCESS_KEY_ID;
-const secretAccessKey = process.env.R2_SECRET_ACCESS_KEY;
-if (!accountId || !accessKeyId || !secretAccessKey) throw new Error("R2 credentials are required.");
-const client = new S3Client({
- region: "auto",
- endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
- credentials: { accessKeyId, secretAccessKey },
-});
-const Bucket = "donkey-cut";
-
-async function read(Key: string) {
- const object = await client.send(new GetObjectCommand({ Bucket, Key }));
- if (!object.Body || !object.ETag) throw new Error(`Incomplete object: ${Key}`);
- const body = await object.Body.transformToString();
- return { key: Key, body, etag: object.ETag };
-}
-
-async function repair() {
- const original = await read(ROLLUP_KEY);
- const rollup = analyticsRollupSchema.parse(JSON.parse(original.body));
- const bitIndex = rollup.sources.indexOf("creditLedger");
- if (bitIndex < 0) throw new Error("Rollup has no credit activity source.");
- const bit = 1 << bitIndex;
- const writes: { original: Awaited<ReturnType<typeof read>>; body: string }[] = [];
- for (const [index, day] of rollup.days.entries()) {
- const stored = await read(dayDbKey(day));
- const db = analyticsDbDayFileSchema.parse(JSON.parse(stored.body));
- if (db.day !== day) throw new Error(`Day mismatch: ${day}`);
- const ids = await extractCreditActivity(day);
- const active = new Set(ids);
- const before = rollup.users.filter((user) => user.activity[index] !== 0).length;
- for (const user of rollup.users) {
- user.activity[index] = ((user.activity[index] ?? 0) & ~bit) | (active.has(user.id) ? bit : 0);
- }
- const after = rollup.users.filter((user) => user.activity[index] !== 0).length;
- db.active.creditLedger = ids;
- writes.push({ original: stored, body: JSON.stringify(db) });
- console.log(JSON.stringify({ day, before, after }));
- }
- writes.push({ original, body: JSON.stringify(rollup) });
- if (!apply) {
- console.log("Dry run complete. Use --apply to write the repaired window.");
- return;
- }
- const backup = await mkdtemp(join(tmpdir(), "donkey-analytics-backup-"));
- for (const [index, entry] of writes.entries()) {
- await writeFile(join(backup, `${index}.json`), entry.original.body, { mode: 0o600 });
- }
- console.log(`Original objects saved to ${backup}`);
- for (const entry of writes) {
- await client.send(new PutObjectCommand({
- Bucket,
- Key: entry.original.key,
- Body: entry.body,
- ContentType: "application/json",
- IfMatch: entry.original.etag,
- }));
- }
- const verified = await read(ROLLUP_KEY);
- if (verified.body !== writes[writes.length - 1].body) throw new Error("Rollup verification failed.");
- console.log(`Repaired and verified ${rollup.days.length} days.`);
-}
-
-try {
- await repair();
-} finally {
- await prisma.$disconnect();
- client.destroy();
-}
diff --git a/site/scripts/analytics-summary-fixture.ts b/site/scripts/analytics-summary-fixture.ts
index 5892c1a2..cdb8d6ab 100644
--- a/site/scripts/analytics-summary-fixture.ts
+++ b/site/scripts/analytics-summary-fixture.ts
@@ -1,5 +1,6 @@
// Rewrites the analytics fixtures the DonkeyKit tests decode — the summary and
-// the first page of accounts. Run it after changing what either carries, then
+// the first page of accounts, folded out of src/lib/analytics/fixtures/
+// analytics-rollup.json. Run it after that fixture or the fold changes, then
// run the DonkeyKit tests.
//
// npm run analytics:summary-fixture
diff --git a/site/src/lib/analytics/billing.test.ts b/site/src/lib/analytics/billing.test.ts
deleted file mode 100644
index b2795dde..00000000
--- a/site/src/lib/analytics/billing.test.ts
+++ /dev/null
@@ -1,110 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import type Stripe from "stripe";
-
-import { subscriptionCancelScheduled } from "../billing/pro-subscription";
-import { buildBilling } from "./pipeline";
-import type { AnalyticsStripeSnapshot } from "./schema";
-
-const days = ["2026-09-01", "2026-09-02", "2026-09-03"];
-
-const snapshot: AnalyticsStripeSnapshot = {
- abandonedCheckouts: [
- { customerId: "cus_a", day: "2026-09-02", email: "[email protected]", id: "cs_1", kind: "pro" },
- { customerId: "cus_b", day: "2026-08-01", email: "[email protected]", id: "cs_2", kind: "pro" },
- ],
- charges: [
- // Paid inside the window: one Pro, one top-up.
- { amountMicros: "20000000", customerId: "cus_p", day: "2026-09-01", email: "[email protected]", failure: null, id: "ch_1", kind: "pro", paymentIntentId: "pi_1", status: "paid", userId: "u_p" },
- { amountMicros: "5000000", customerId: "cus_t", day: "2026-09-03", email: "[email protected]", failure: null, id: "ch_2", kind: "topup", paymentIntentId: "pi_2", status: "paid", userId: "u_t" },
- // Paid before the window: funds all-time only.
- { amountMicros: "5000000", customerId: "cus_t", day: "2026-08-01", email: "[email protected]", failure: null, id: "ch_3", kind: "topup", paymentIntentId: "pi_3", status: "paid", userId: "u_t" },
- // Declined inside the window.
- { amountMicros: "20000000", customerId: "cus_d", day: "2026-09-02", email: "[email protected]", failure: "Your card has insufficient funds.", id: "ch_4", kind: "pro", paymentIntentId: "pi_4", status: "declined", userId: "u_d" },
- ],
- dashboardUrl: "https://dashboard.stripe.com",
- refunds: [{ amountMicros: "5000000", chargeId: "ch_2", day: "2026-09-03", id: "re_1" }],
- subscriptions: [
- // Live, and scheduled to end through the portal's cancel_at.
- { cancelAt: "2026-09-08T00:00:00.000Z", cancelScheduled: true, canceledAt: "2026-09-02T10:00:00.000Z", comment: "Export broke.", createdAt: "2026-08-09T00:00:00.000Z", customerId: "cus_p", email: "[email protected]", endedAt: null, feedback: "switched_service", id: "sub_1", status: "active", userId: "u_p" },
- // Live, staying.
- { cancelAt: null, cancelScheduled: false, canceledAt: null, comment: null, createdAt: "2026-09-01T00:00:00.000Z", customerId: "cus_q", email: "[email protected]", endedAt: null, feedback: null, id: "sub_2", status: "active", userId: "u_q" },
- // Ended before the window.
- { cancelAt: null, cancelScheduled: false, canceledAt: "2026-07-01T00:00:00.000Z", comment: null, createdAt: "2026-06-01T00:00:00.000Z", customerId: "cus_r", email: "[email protected]", endedAt: "2026-07-01T00:00:00.000Z", feedback: null, id: "sub_3", status: "canceled", userId: "u_r" },
- ],
-};
-
-describe("buildBilling", () => {
- const billing = buildBilling(snapshot, days);
-
- test("counts a portal-scheduled cancel as canceling", () => {
- expect(billing.days).toEqual(days);
- expect(billing.subscribers).toBe(2);
- expect(billing.canceling).toEqual([
- {
- comment: "Export broke.",
- customerId: "cus_p",
- email: "[email protected]",
- endsAt: "2026-09-08T00:00:00.000Z",
- feedback: "switched_service",
- requestedAt: "2026-09-02T10:00:00.000Z",
- subscriptionId: "sub_1",
- },
- ]);
- expect(billing.churned).toBe(1);
- });
-
- test("nets paid against refunds and keeps declines outside", () => {
- expect(billing.window).toEqual({
- abandonedCheckouts: 1,
- cancels: 1,
- declinedCustomers: 1,
- declinedMicros: "20000000",
- netMicros: "20000000",
- paidMicros: "25000000",
- refundedMicros: "5000000",
- });
- expect(billing.revenue).toEqual([
- { cancels: 0, declinedMicros: "0", otherMicros: "0", proMicros: "20000000", refundedMicros: "0", topupMicros: "0" },
- { cancels: 1, declinedMicros: "20000000", otherMicros: "0", proMicros: "0", refundedMicros: "0", topupMicros: "0" },
- { cancels: 0, declinedMicros: "0", otherMicros: "0", proMicros: "0", refundedMicros: "5000000", topupMicros: "5000000" },
- ]);
- });
-
- test("funds all time and lists the window's events newest first", () => {
- expect(billing.funded).toBe(2);
- expect(billing.fundedMicros).toBe("30000000");
- expect(billing.events.map((e) => [e.day, e.kind, e.objectId, e.detail])).toEqual([
- ["2026-09-03", "paid", "pi_2", "topup"],
- ["2026-09-02", "canceled", "sub_1", "switched_service · Export broke."],
- ["2026-09-02", "declined", "pi_4", "Your card has insufficient funds."],
- ["2026-09-01", "paid", "pi_1", "pro"],
- ]);
- });
-
- test("a cancel event says where the subscription stands", () => {
- const cancel = billing.events.find((e) => e.kind === "canceled");
- expect([cancel?.ended, cancel?.endsAt]).toEqual([false, "2026-09-08T00:00:00.000Z"]);
- const stopped = buildBilling(
- {
- ...snapshot,
- subscriptions: [
- { ...snapshot.subscriptions[2], canceledAt: "2026-09-01T00:00:00.000Z", endedAt: "2026-09-01T00:00:00.000Z" },
- ],
- },
- days,
- );
- const gone = stopped.events.find((e) => e.kind === "canceled");
- expect([gone?.ended, gone?.endsAt]).toEqual([true, "2026-09-01T00:00:00.000Z"]);
- });
-});
-
-describe("subscriptionCancelScheduled", () => {
- const sub = (fields: Partial<Stripe.Subscription>) =>
- ({ cancel_at: null, cancel_at_period_end: false, ...fields }) as Stripe.Subscription;
-
- test("reads either spelling of a scheduled end", () => {
- expect(subscriptionCancelScheduled(sub({}))).toBe(false);
- expect(subscriptionCancelScheduled(sub({ cancel_at_period_end: true }))).toBe(true);
- expect(subscriptionCancelScheduled(sub({ cancel_at: 1788966820 }))).toBe(true);
- });
-});
diff --git a/site/src/lib/analytics/fixtures/analytics-rollup.json b/site/src/lib/analytics/fixtures/analytics-rollup.json
new file mode 100644
index 00000000..e97b882f
--- /dev/null
+++ b/site/src/lib/analytics/fixtures/analytics-rollup.json
@@ -0,0 +1,233 @@
+{
+ "billing": {
+ "canceling": [
+ {
+ "comment": "Export broke.",
+ "customerId": "cus_p",
+ "email": "[email protected]",
+ "endsAt": "2026-09-08T00:00:00.000Z",
+ "feedback": "switched_service",
+ "requestedAt": "2026-08-31T10:00:00.000Z",
+ "subscriptionId": "sub_1"
+ }
+ ],
+ "churned": 1,
+ "dashboardUrl": "https://dashboard.stripe.com/test",
+ "days": [
+ "2026-08-30",
+ "2026-08-31",
+ "2026-09-01",
+ "2026-09-02"
+ ],
+ "events": [
+ {
+ "amountMicros": "5000000",
+ "customerId": "cus_n",
+ "day": "2026-09-01",
+ "detail": "topup",
+ "email": "[email protected]",
+ "ended": false,
+ "endsAt": null,
+ "kind": "paid",
+ "objectId": "pi_2"
+ },
+ {
+ "amountMicros": null,
+ "customerId": "cus_p",
+ "day": "2026-08-31",
+ "detail": "switched_service · Export broke.",
+ "email": "[email protected]",
+ "ended": false,
+ "endsAt": "2026-09-08T00:00:00.000Z",
+ "kind": "canceled",
+ "objectId": "sub_1"
+ },
+ {
+ "amountMicros": "20000000",
+ "customerId": "cus_d",
+ "day": "2026-08-31",
+ "detail": "Your card has insufficient funds.",
+ "email": "[email protected]",
+ "ended": false,
+ "endsAt": null,
+ "kind": "declined",
+ "objectId": "pi_3"
+ },
+ {
+ "amountMicros": "20000000",
+ "customerId": "cus_p",
+ "day": "2026-08-30",
+ "detail": "pro",
+ "email": "[email protected]",
+ "ended": false,
+ "endsAt": null,
+ "kind": "paid",
+ "objectId": "pi_1"
+ }
+ ],
+ "funded": 2,
+ "fundedMicros": "25000000",
+ "revenue": [
+ {
+ "cancels": 0,
+ "declinedMicros": "0",
+ "otherMicros": "0",
+ "proMicros": "20000000",
+ "refundedMicros": "0",
+ "topupMicros": "0"
+ },
+ {
+ "cancels": 1,
+ "declinedMicros": "20000000",
+ "otherMicros": "0",
+ "proMicros": "0",
+ "refundedMicros": "0",
+ "topupMicros": "0"
+ },
+ {
+ "cancels": 0,
+ "declinedMicros": "0",
+ "otherMicros": "0",
+ "proMicros": "0",
+ "refundedMicros": "0",
+ "topupMicros": "5000000"
+ },
+ {
+ "cancels": 0,
+ "declinedMicros": "0",
+ "otherMicros": "0",
+ "proMicros": "0",
+ "refundedMicros": "1000000",
+ "topupMicros": "0"
+ }
+ ],
+ "subscribers": 2,
+ "window": {
+ "abandonedCheckouts": 1,
+ "cancels": 1,
+ "declinedCustomers": 1,
+ "declinedMicros": "20000000",
+ "netMicros": "24000000",
+ "paidMicros": "25000000",
+ "refundedMicros": "1000000"
+ }
+ },
+ "days": [
+ "2026-08-30",
+ "2026-08-31",
+ "2026-09-01"
+ ],
+ "generatedAt": "2026-09-02T04:00:00.000Z",
+ "missing": [
+ {
+ "day": "2026-08-31",
+ "sources": [
+ "db",
+ "posthog"
+ ]
+ }
+ ],
+ "referrals": {
+ "days": [
+ {
+ "counts": [
+ 0,
+ 0,
+ 0,
+ 1,
+ 0,
+ 0,
+ 0,
+ 1
+ ],
+ "day": "2026-08-31",
+ "others": [
+ "a podcast"
+ ],
+ "respondents": 1
+ },
+ {
+ "counts": [
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0
+ ],
+ "day": "2026-09-01",
+ "others": [],
+ "respondents": 0
+ }
+ ],
+ "sources": [
+ "discord",
+ "tiktok",
+ "instagram",
+ "youtube",
+ "reddit",
+ "search",
+ "friend",
+ "other"
+ ]
+ },
+ "sources": [
+ "inference",
+ "renders",
+ "copies",
+ "libraryAssets",
+ "templates",
+ "creditLedger",
+ "posthog"
+ ],
+ "users": [
+ {
+ "activity": [
+ 0,
+ 0,
+ 64
+ ],
+ "balanceMicros": "0",
+ "email": "[email protected]",
+ "fundedMicros": "5000000",
+ "id": "u_new",
+ "name": "New Person",
+ "registeredAt": "2026-08-31T12:00:00.000Z",
+ "storageBytes": "0",
+ "storageQuotaBytes": 262144000
+ },
+ {
+ "activity": [
+ 65,
+ 0,
+ 64
+ ],
+ "balanceMicros": "3000000",
+ "email": "[email protected]",
+ "fundedMicros": "20000000",
+ "id": "u_pro",
+ "name": "Pro Person",
+ "registeredAt": "2026-08-01T10:00:00.000Z",
+ "storageBytes": "1048576",
+ "storageQuotaBytes": 107374182400
+ },
+ {
+ "activity": [
+ 2,
+ 0,
+ 2
+ ],
+ "balanceMicros": "500000",
+ "email": "[email protected]",
+ "id": "u_su",
+ "name": "Super Person",
+ "registeredAt": "2026-07-01T00:00:00.000Z",
+ "storageBytes": "0",
+ "storageQuotaBytes": null,
+ "superUser": true
+ }
+ ],
+ "version": 1
+}
diff --git a/site/src/lib/analytics/pipeline.ts b/site/src/lib/analytics/pipeline.ts
deleted file mode 100644
index 4e6f8e9b..00000000
--- a/site/src/lib/analytics/pipeline.ts
+++ /dev/null
@@ -1,684 +0,0 @@
-// Nightly analytics pipeline. Extraction writes one small JSON file per day
-// per source to R2, a snapshot of users and balances is rewritten every run,
-// and consolidation folds the window into the single rollup.json the
-// analytics routes read. Idempotency is per day: a day file is *final*
-// once it was written after its UTC day closed, and final files are skipped
-// unless forced — so a premature run of today, a per-day retrigger, and the
-// regular nightly run all compose without redoing finished work.
-import type { z } from "zod";
-
-import type { Prisma } from "@/generated/prisma/client";
-import { getObject, listObjectsWithDates, putObject } from "@/cut/server/cloud/r2";
-import { fetchActiveDistinctIds, isPosthogQueryConfigured } from "@/lib/analytics/posthog";
-import { pullStripeSnapshot } from "@/lib/analytics/stripe";
-import {
- addUtcDays,
- ANALYTICS_DB_SOURCES,
- ANALYTICS_ROLLUP_VERSION,
- ANALYTICS_SOURCES,
- analyticsDbDayFileSchema,
- analyticsPosthogDayFileSchema,
- analyticsRollupSchema,
- analyticsSnapshotFileSchema,
- DAYS_PREFIX,
- dayDbKey,
- dayPosthogKey,
- ROLLUP_KEY,
- SNAPSHOT_KEY,
- type AnalyticsDbDayFile,
- type AnalyticsPosthogDayFile,
- type AnalyticsRollup,
- type AnalyticsSnapshotFile,
- type AnalyticsStripeSnapshot,
- utcDayOf,
-} from "@/lib/analytics/schema";
-import { cutLimitsForTier } from "@/cut/server/cloud/limits";
-import { isActiveProStatus } from "@/lib/billing/pro-subscription";
-import { zeroCreditMicros } from "@/lib/credits/amounts";
-import { lapsedCreditByUser } from "@/lib/credits/inference";
-import { REFERRAL_SOURCES } from "@/lib/onboarding/sequence";
-import { prisma } from "@/lib/prisma";
-
-const WINDOW_DAYS = 60;
-// Backfill after an outage converges over nights instead of one heavy run.
-const MAX_EXTRACT_DAYS_PER_RUN = 5;
-const SNAPSHOT_PAGE_SIZE = 1000;
-
-// The rollup's day window: yesterday and the days before it, oldest first.
-// Activity is extracted once a UTC day has closed, so today is never in it.
-function windowDays(): string[] {
- return daysEnding(addUtcDays(utcDayOf(new Date()), -1));
-}
-
-// The billing window ends today: Stripe reports money and cancels as they
-// happen, and the webhook refresh exists to show them the same day.
-function billingDays(): string[] {
- return daysEnding(utcDayOf(new Date()));
-}
-
-function daysEnding(last: string): string[] {
- const days: string[] = [];
- for (let i = WINDOW_DAYS - 1; i >= 0; i--) days.push(addUtcDays(last, -i));
- return days;
-}
-
-export class InvalidDayError extends Error {}
-
-export async function runAnalyticsDaily(payload: {
- day?: string;
- force?: boolean;
-}): Promise<Prisma.InputJsonValue> {
- const today = utcDayOf(new Date());
- if (payload.day !== undefined) {
- if (addUtcDays(payload.day, 0) !== payload.day) {
- throw new InvalidDayError(`"${payload.day}" is not a calendar day.`);
- }
- if (payload.day > today) {
- throw new InvalidDayError(`"${payload.day}" is in the future.`);
- }
- }
-
- const posthogConfigured = isPosthogQueryConfigured();
- const lastWrite = new Map(
- (await listObjectsWithDates(DAYS_PREFIX)).map((o) => [o.key, o.lastModified]),
- );
- // Final = written after the UTC day closed. A premature mid-day extraction
- // of today stays non-final, so the nightly run redoes it once the day is
- // whole; a manual retrigger of a past day skips whatever already finished.
- const isFinal = (key: string, day: string) => {
- const at = lastWrite.get(key);
- return at !== undefined && at.getTime() >= Date.parse(`${addUtcDays(day, 1)}T00:00:00Z`);
- };
- const isComplete = (day: string) =>
- isFinal(dayDbKey(day), day) && (!posthogConfigured || isFinal(dayPosthogKey(day), day));
-
- const yesterday = addUtcDays(today, -1);
- let targets: string[];
- if (payload.day) {
- targets = [payload.day];
- } else {
- targets = [];
- for (let i = 0; i < WINDOW_DAYS; i++) {
- const day = addUtcDays(yesterday, -i); // newest first
- if (!isComplete(day)) targets.push(day);
- }
- }
- const deferred = Math.max(0, targets.length - MAX_EXTRACT_DAYS_PER_RUN);
- targets = targets.slice(0, MAX_EXTRACT_DAYS_PER_RUN);
-
- // A failed source is recorded and retried by the next night's backfill; it
- // never blocks the other source, other days, the snapshot, or the rollup.
- const errors: { day: string; source: string; message: string }[] = [];
- let extracted = 0;
- let skipped = 0;
- for (const day of targets) {
- if (!payload.force && isFinal(dayDbKey(day), day)) {
- skipped++;
- } else {
- try {
- await putJson(dayDbKey(day), await extractDbDay(day));
- extracted++;
- } catch (e) {
- errors.push({ day, message: messageOf(e), source: "db" });
- }
- }
- if (!posthogConfigured) continue;
- if (!payload.force && isFinal(dayPosthogKey(day), day)) {
- skipped++;
- } else {
- try {
- await putJson(dayPosthogKey(day), await extractPosthogDay(day));
- extracted++;
- } catch (e) {
- errors.push({ day, message: messageOf(e), source: "posthog" });
- }
- }
- }
-
- // Snapshot and rollup errors propagate: they mean the database or storage
- // itself is unhealthy, which is the queue's transient-retry case.
- const snapshot = await writeSnapshot();
- const rollup = await consolidate(snapshot, posthogConfigured);
-
- return {
- deferred,
- errors,
- extracted,
- rollupDays: rollup.days.length,
- rollupUsers: rollup.users.length,
- skipped,
- snapshotUsers: snapshot.users.length,
- targetedDays: targets,
- };
-}
-
-export async function extractCreditActivity(day: string): Promise<string[]> {
- // Usage charges record a user action. Grants, expirations and write-offs
- // are bookkeeping that can run while the account is idle.
- const rows = await prisma.userCreditLedgerEntry.groupBy({
- by: ["userId"],
- where: {
- createdAt: {
- gte: new Date(`${day}T00:00:00Z`),
- lt: new Date(`${addUtcDays(day, 1)}T00:00:00Z`),
- },
- type: "usage",
- },
- });
- return rows.map((row) => row.userId).sort();
-}
-
-async function extractDbDay(day: string): Promise<AnalyticsDbDayFile> {
- const range = {
- createdAt: {
- gte: new Date(`${day}T00:00:00Z`),
- lt: new Date(`${addUtcDays(day, 1)}T00:00:00Z`),
- },
- };
- // One bounded group-by per event table, run serially to keep the load light.
- // Every table indexes (userId, createdAt).
- const queries = {
- copies: () => prisma.cutCopyJob.groupBy({ by: ["userId"], where: range }),
- creditLedger: async () => (await extractCreditActivity(day)).map((userId) => ({ userId })),
- inference: () => prisma.inferenceUsageEvent.groupBy({ by: ["userId"], where: range }),
- libraryAssets: () => prisma.cutLibraryAsset.groupBy({ by: ["userId"], where: range }),
- renders: () => prisma.cutRenderJob.groupBy({ by: ["userId"], where: range }),
- templates: () => prisma.cutTemplate.groupBy({ by: ["userId"], where: range }),
- };
- const active: AnalyticsDbDayFile["active"] = {};
- for (const source of ANALYTICS_DB_SOURCES) {
- active[source] = (await queries[source]()).map((row) => row.userId).sort();
- }
- return { active, day, generatedAt: new Date().toISOString(), version: 1 };
-}
-
-async function extractPosthogDay(day: string): Promise<AnalyticsPosthogDayFile> {
- return {
- activeDistinctIds: await fetchActiveDistinctIds(day),
- day,
- generatedAt: new Date().toISOString(),
- version: 1,
- };
-}
-
-// Drain a cursor-paginated query. The page function spreads the cursor args
-// into its findMany, so every table pages the same way.
-async function fetchAllPages<T extends { id: string }>(
- fetchPage: (cursorArgs: { cursor: { id: string }; skip: 1 } | undefined) => Promise<T[]>,
-): Promise<T[]> {
- const rows: T[] = [];
- let cursor: string | undefined;
- for (;;) {
- const page = await fetchPage(
- cursor === undefined ? undefined : { cursor: { id: cursor }, skip: 1 },
- );
- rows.push(...page);
- if (page.length < SNAPSHOT_PAGE_SIZE) return rows;
- cursor = page[page.length - 1].id;
- }
-}
-
-async function writeSnapshot(): Promise<AnalyticsSnapshotFile> {
- const users = (
- await fetchAllPages((cursorArgs) =>
- prisma.user.findMany({
- orderBy: { id: "asc" },
- select: {
- createdAt: true,
- email: true,
- id: true,
- name: true,
- onboarding: {
- select: { referralAnsweredAt: true, referralOther: true, referralSources: true },
- },
- superUser: true,
- },
- take: SNAPSHOT_PAGE_SIZE,
- ...cursorArgs,
- }),
- )
- ).map((u) => ({
- createdAt: u.createdAt.toISOString(),
- email: u.email,
- id: u.id,
- name: u.name,
- ...(u.superUser ? { superUser: true } : {}),
- ...(u.onboarding?.referralAnsweredAt
- ? {
- referralAnsweredAt: u.onboarding.referralAnsweredAt.toISOString(),
- referralSources: u.onboarding.referralSources,
- ...(u.onboarding.referralOther?.trim()
- ? { referralOther: u.onboarding.referralOther.trim() }
- : {}),
- }
- : {}),
- }));
-
- // The stored balance still carries a lapsed grant until the account is
- // read, charged, or reached by the nightly sweep; the snapshot takes the
- // lapsed amount off so every balance is what the account can spend.
- const lapsed = await lapsedCreditByUser();
- const balances = (
- await fetchAllPages((cursorArgs) =>
- prisma.userCreditAccount.findMany({
- orderBy: { id: "asc" },
- select: { balanceMicros: true, id: true, userId: true },
- take: SNAPSHOT_PAGE_SIZE,
- ...cursorArgs,
- }),
- )
- ).map((account) => ({
- balanceMicros: (
- account.balanceMicros - (lapsed.get(account.userId) ?? zeroCreditMicros)
- ).toString(),
- userId: account.userId,
- }));
-
- // CutStorageUsage keys on userId, so it pages on that column.
- const storage: { bytes: string; userId: string }[] = [];
- for (let cursor: string | undefined; ; ) {
- const page = await prisma.cutStorageUsage.findMany({
- orderBy: { userId: "asc" },
- select: { bytes: true, userId: true },
- take: SNAPSHOT_PAGE_SIZE,
- ...(cursor === undefined ? {} : { cursor: { userId: cursor }, skip: 1 }),
- });
- storage.push(...page.map((row) => ({ bytes: row.bytes.toString(), userId: row.userId })));
- if (page.length < SNAPSHOT_PAGE_SIZE) break;
- cursor = page[page.length - 1].userId;
- }
-
- const stripe = await pullStripeSnapshot(billingDays()[0]);
-
- const snapshot: AnalyticsSnapshotFile = {
- balances,
- generatedAt: new Date().toISOString(),
- storage,
- stripe,
- users,
- version: 1,
- };
- await putJson(SNAPSHOT_KEY, snapshot);
- return snapshot;
-}
-
-// One day's stored inputs to consolidation, null where a file was absent or
-// unreadable.
-export type AnalyticsDayFiles = {
- db: AnalyticsDbDayFile | null;
- posthog: AnalyticsPosthogDayFile | null;
-};
-
-async function consolidate(
- snapshot: AnalyticsSnapshotFile,
- posthogConfigured: boolean,
-): Promise<AnalyticsRollup> {
- const rollup = await buildRollup({
- billingDays: billingDays(),
- days: windowDays(),
- generatedAt: new Date().toISOString(),
- posthogConfigured,
- readDayFiles: async (day) => ({
- db: readJson(await getObject(dayDbKey(day)), analyticsDbDayFileSchema),
- posthog: posthogConfigured
- ? readJson(await getObject(dayPosthogKey(day)), analyticsPosthogDayFileSchema)
- : null,
- }),
- snapshot,
- });
- await putJson(ROLLUP_KEY, rollup);
- return rollup;
-}
-
-// The whole rollup from its inputs. Day files come through the reader one
-// day at a time, so a 60-day window holds one day's ids at once; nothing here
-// touches storage or the clock. Every rollup in storage came out of this
-// function, so the fixture the phone's model is tested against comes out of
-// it too (summary-fixture.ts): a change to what the dashboard is served
-// reaches that test in the same change.
-export async function buildRollup(input: {
- snapshot: AnalyticsSnapshotFile;
- // The activity window, oldest first; every user's activity aligns with it.
- days: string[];
- // The billing window, oldest first, ending today.
- billingDays: string[];
- readDayFiles: (day: string) => Promise<AnalyticsDayFiles>;
- posthogConfigured: boolean;
- generatedAt: string;
-}): Promise<AnalyticsRollup> {
- const { days, snapshot } = input;
- const yesterday = days[days.length - 1];
-
- const masks = new Map<string, number[]>();
- for (const user of snapshot.users) masks.set(user.id, days.map(() => 0));
-
- const posthogBit = 1 << ANALYTICS_SOURCES.indexOf("posthog");
- const missing: AnalyticsRollup["missing"] = [];
- for (let i = 0; i < days.length; i++) {
- const day = days[i];
- const absent: string[] = [];
- const files = await input.readDayFiles(day);
-
- if (!files.db) {
- absent.push("db");
- } else {
- for (const source of ANALYTICS_DB_SOURCES) {
- const bit = 1 << ANALYTICS_SOURCES.indexOf(source);
- for (const id of files.db.active[source] ?? []) {
- const mask = masks.get(id);
- if (mask) mask[i] |= bit;
- }
- }
- }
-
- if (input.posthogConfigured) {
- if (!files.posthog) {
- absent.push("posthog");
- } else {
- // Anonymous pre-sign-in distinct_ids match no snapshot user and drop out.
- for (const id of files.posthog.activeDistinctIds) {
- const mask = masks.get(id);
- if (mask) mask[i] |= posthogBit;
- }
- }
- }
-
- if (absent.length > 0) missing.push({ day, sources: absent });
- }
-
- const balanceByUser = new Map(snapshot.balances.map((b) => [b.userId, b.balanceMicros]));
- const fundedByUser = fundedMicrosByUser(snapshot.stripe);
- const storageByUser = new Map((snapshot.storage ?? []).map((s) => [s.userId, s.bytes]));
- const proUsers = new Set(
- (snapshot.stripe?.subscriptions ?? [])
- .filter((sub) => isActiveProStatus(sub.status))
- .map((sub) => sub.userId),
- );
- const users = snapshot.users
- .map((u) => {
- const funded = fundedByUser.get(u.id);
- const limits = cutLimitsForTier({ superUser: u.superUser === true, pro: proUsers.has(u.id) });
- return {
- activity: masks.get(u.id) ?? days.map(() => 0),
- balanceMicros: balanceByUser.get(u.id) ?? "0",
- email: u.email,
- ...(funded !== undefined ? { fundedMicros: funded.toString() } : {}),
- id: u.id,
- name: u.name,
- registeredAt: u.createdAt,
- storageBytes: storageByUser.get(u.id) ?? "0",
- storageQuotaBytes: limits.storageBytes,
- ...(u.superUser ? { superUser: true } : {}),
- };
- })
- .sort((a, b) => (a.registeredAt < b.registeredAt ? 1 : -1));
-
- return {
- ...(snapshot.stripe ? { billing: buildBilling(snapshot.stripe, input.billingDays) } : {}),
- days,
- generatedAt: input.generatedAt,
- missing,
- referrals: buildReferrals(snapshot, yesterday),
- sources: [...ANALYTICS_SOURCES],
- users,
- version: ANALYTICS_ROLLUP_VERSION,
- };
-}
-
-// All-time paid charges per user, for the user list's funded column.
-function fundedMicrosByUser(stripe: AnalyticsStripeSnapshot | undefined): Map<string, bigint> {
- const funded = new Map<string, bigint>();
- for (const charge of stripe?.charges ?? []) {
- if (charge.status !== "paid" || !charge.userId) continue;
- funded.set(
- charge.userId,
- (funded.get(charge.userId) ?? BigInt(0)) + BigInt(charge.amountMicros),
- );
- }
- return funded;
-}
-
-// The Stripe record folded into what the dashboard shows: subscription state
-// now, per-day money, and the window's charges and cancel requests with the
-// ids that link into Stripe. Abandoned checkouts never subscribed, so they
-// count nowhere in churn; past_due and paused sit outside both buckets until
-// Stripe resolves them.
-export function buildBilling(
- stripe: AnalyticsStripeSnapshot,
- days: string[],
-): NonNullable<AnalyticsRollup["billing"]> {
- type Billing = NonNullable<AnalyticsRollup["billing"]>;
- const dayIndex = new Map(days.map((day, i) => [day, i]));
- const inWindow = (iso: string | null) => iso !== null && dayIndex.has(utcDayOf(new Date(iso)));
-
- let subscribers = 0;
- let churned = 0;
- const canceling: Billing["canceling"] = [];
- const events: Billing["events"] = [];
- const revenue = days.map(() => ({
- cancels: 0,
- declinedMicros: BigInt(0),
- otherMicros: BigInt(0),
- proMicros: BigInt(0),
- refundedMicros: BigInt(0),
- topupMicros: BigInt(0),
- }));
-
- for (const sub of stripe.subscriptions) {
- if (isActiveProStatus(sub.status)) {
- subscribers++;
- if (sub.cancelScheduled) {
- canceling.push({
- comment: sub.comment,
- customerId: sub.customerId,
- email: sub.email,
- endsAt: sub.cancelAt,
- feedback: sub.feedback,
- requestedAt: sub.canceledAt,
- subscriptionId: sub.id,
- });
- }
- } else if (sub.status === "canceled" || sub.status === "unpaid") {
- churned++;
- }
- // A cancel request lands on the day it was made, whether the
- // subscription is still running out its period or already ended.
- if (sub.canceledAt !== null && inWindow(sub.canceledAt)) {
- const day = utcDayOf(new Date(sub.canceledAt));
- revenue[dayIndex.get(day)!].cancels++;
- events.push({
- amountMicros: null,
- customerId: sub.customerId,
- day,
- detail: [sub.feedback, sub.comment].filter(Boolean).join(" · ") || null,
- email: sub.email,
- ended: !isActiveProStatus(sub.status),
- endsAt: sub.cancelAt ?? sub.endedAt,
- kind: "canceled",
- objectId: sub.id,
- });
- }
- }
- canceling.sort((a, b) => (a.endsAt ?? "").localeCompare(b.endsAt ?? ""));
-
- const fundedCustomers = new Set<string>();
- let fundedMicros = BigInt(0);
- let paidMicros = BigInt(0);
- let declinedMicros = BigInt(0);
- const declinedCustomers = new Set<string>();
- for (const charge of stripe.charges) {
- const amount = BigInt(charge.amountMicros);
- const i = dayIndex.get(charge.day);
- if (charge.status === "paid") {
- fundedMicros += amount;
- fundedCustomers.add(charge.customerId ?? charge.id);
- if (i === undefined) continue;
- paidMicros += amount;
- if (charge.kind === "pro") revenue[i].proMicros += amount;
- else if (charge.kind === "topup") revenue[i].topupMicros += amount;
- else revenue[i].otherMicros += amount;
- events.push({
- amountMicros: charge.amountMicros,
- customerId: charge.customerId,
- day: charge.day,
- detail: charge.kind,
- email: charge.email,
- ended: false,
- endsAt: null,
- kind: "paid",
- objectId: charge.paymentIntentId,
- });
- } else if (i !== undefined) {
- declinedMicros += amount;
- declinedCustomers.add(charge.customerId ?? charge.id);
- revenue[i].declinedMicros += amount;
- events.push({
- amountMicros: charge.amountMicros,
- customerId: charge.customerId,
- day: charge.day,
- detail: charge.failure,
- email: charge.email,
- ended: false,
- endsAt: null,
- kind: "declined",
- objectId: charge.paymentIntentId,
- });
- }
- }
-
- let refundedMicros = BigInt(0);
- for (const refund of stripe.refunds) {
- const i = dayIndex.get(refund.day);
- if (i === undefined) continue;
- refundedMicros += BigInt(refund.amountMicros);
- revenue[i].refundedMicros += BigInt(refund.amountMicros);
- }
-
- events.sort((a, b) => b.day.localeCompare(a.day));
-
- return {
- canceling,
- churned,
- dashboardUrl: stripe.dashboardUrl,
- days,
- events,
- funded: fundedCustomers.size,
- fundedMicros: fundedMicros.toString(),
- revenue: revenue.map((entry) => ({
- cancels: entry.cancels,
- declinedMicros: entry.declinedMicros.toString(),
- otherMicros: entry.otherMicros.toString(),
- proMicros: entry.proMicros.toString(),
- refundedMicros: entry.refundedMicros.toString(),
- topupMicros: entry.topupMicros.toString(),
- })),
- subscribers,
- window: {
- abandonedCheckouts: stripe.abandonedCheckouts.filter((c) => dayIndex.has(c.day)).length,
- cancels: revenue.reduce((sum, entry) => sum + entry.cancels, 0),
- declinedCustomers: declinedCustomers.size,
- declinedMicros: declinedMicros.toString(),
- netMicros: (paidMicros - refundedMicros).toString(),
- paidMicros: paidMicros.toString(),
- refundedMicros: refundedMicros.toString(),
- },
- };
-}
-
-// A Stripe billing webhook lands here: the Stripe record is re-read, the
-// snapshot keeps it for the next nightly consolidation, and the rollup's
-// billing section (plus each user's funded total) is rewritten in place so
-// the dashboard shows the event without waiting for the night. Without a
-// snapshot or rollup yet there is nothing to refresh; the nightly run builds
-// them.
-export async function refreshBilling(): Promise<Prisma.InputJsonValue> {
- const [snapshot, rollup] = await Promise.all([
- getObject(SNAPSHOT_KEY).then((obj) => readJson(obj, analyticsSnapshotFileSchema)),
- getObject(ROLLUP_KEY).then((obj) => readJson(obj, analyticsRollupSchema)),
- ]);
- if (!snapshot || !rollup) return { refreshed: false };
-
- const days = billingDays();
- const stripe = await pullStripeSnapshot(days[0]);
- await putJson(SNAPSHOT_KEY, { ...snapshot, stripe });
-
- const fundedByUser = fundedMicrosByUser(stripe);
- const users = rollup.users.map((user) => ({
- ...user,
- fundedMicros: fundedByUser.get(user.id)?.toString(),
- }));
- await putJson(ROLLUP_KEY, { ...rollup, billing: buildBilling(stripe, days), users });
- return {
- charges: stripe.charges.length,
- refreshed: true,
- subscriptions: stripe.subscriptions.length,
- };
-}
-
-// The snapshot covers every user, so this recomputes the whole referral
-// history each run — an answer moved by a replayed onboarding lands on its
-// new day with nothing left behind on the old one.
-function buildReferrals(
- snapshot: AnalyticsSnapshotFile,
- through: string,
-): AnalyticsRollup["referrals"] {
- const answered = snapshot.users
- .flatMap((u) =>
- u.referralAnsweredAt === undefined
- ? []
- : [
- {
- day: utcDayOf(new Date(u.referralAnsweredAt)),
- other: u.referralOther,
- sources: u.referralSources ?? [],
- },
- ],
- )
- .filter((a) => a.day <= through);
- if (answered.length === 0) return undefined;
-
- const sources: string[] = REFERRAL_SOURCES.map((s) => s.id);
- for (const a of answered) {
- for (const id of a.sources) if (!sources.includes(id)) sources.push(id);
- }
-
- const first = answered.reduce((min, a) => (a.day < min ? a.day : min), through);
- const byDay = new Map<
- string,
- { day: string; respondents: number; counts: number[]; others: string[] }
- >();
- const days: NonNullable<AnalyticsRollup["referrals"]>["days"] = [];
- for (let day = first; day <= through; day = addUtcDays(day, 1)) {
- const entry = { counts: sources.map(() => 0), day, others: [], respondents: 0 };
- byDay.set(day, entry);
- days.push(entry);
- }
- for (const a of answered) {
- const entry = byDay.get(a.day);
- if (!entry) continue;
- entry.respondents++;
- if (a.other) entry.others.push(a.other);
- for (const id of a.sources) entry.counts[sources.indexOf(id)]++;
- }
- return { days, sources };
-}
-
-async function putJson(key: string, value: unknown): Promise<void> {
- await putObject(key, Buffer.from(JSON.stringify(value)), "application/json");
-}
-
-/** Parse a stored JSON object against its schema; anything unreadable — gone,
- * corrupt, or from an incompatible version — reads as missing. */
-function readJson<T>(obj: { bytes: Buffer } | null, schema: z.ZodType<T>): T | null {
- if (!obj) return null;
- try {
- const parsed = schema.safeParse(JSON.parse(obj.bytes.toString("utf8")));
- return parsed.success ? parsed.data : null;
- } catch {
- return null;
- }
-}
-
-function messageOf(e: unknown): string {
- return e instanceof Error ? e.message : String(e);
-}
diff --git a/site/src/lib/analytics/posthog.test.ts b/site/src/lib/analytics/posthog.test.ts
deleted file mode 100644
index 209679c3..00000000
--- a/site/src/lib/analytics/posthog.test.ts
+++ /dev/null
@@ -1,58 +0,0 @@
-import { afterEach, describe, expect, test } from "bun:test";
-import { isPosthogQueryConfigured, PosthogConfigError } from "./posthog";
-
-const KEY = "POSTHOG_PERSONAL_API_KEY";
-const ID = "POSTHOG_PROJECT_ID";
-const saved = { key: process.env[KEY], id: process.env[ID] };
-
-const set = (name: string, value: string | undefined) => {
- if (value === undefined) delete process.env[name];
- else process.env[name] = value;
-};
-
-const raisesConfigError = (run: () => unknown): boolean => {
- try {
- run();
- } catch (e) {
- return e instanceof PosthogConfigError;
- }
- return false;
-};
-
-afterEach(() => {
- set(KEY, saved.key);
- set(ID, saved.id);
-});
-
-describe("PostHog query configuration", () => {
- test("both present reads as configured", () => {
- set(KEY, "phx_test");
- set(ID, "110583");
- expect(isPosthogQueryConfigured()).toBe(true);
- });
-
- test("either one missing skips PostHog", () => {
- set(KEY, "phx_test");
- set(ID, undefined);
- expect(isPosthogQueryConfigured()).toBe(false);
- set(KEY, undefined);
- set(ID, "110583");
- expect(isPosthogQueryConfigured()).toBe(false);
- });
-
- test("a project id that is not a number fails the run", () => {
- set(KEY, "phx_test");
- for (const bad of ["asdf", "0", "-3", "11.5", "110583abc"]) {
- set(ID, bad);
- expect(raisesConfigError(isPosthogQueryConfigured)).toBe(true);
- }
- });
-
- test("blank and whitespace count as unset", () => {
- set(KEY, "phx_test");
- for (const blank of ["", " "]) {
- set(ID, blank);
- expect(isPosthogQueryConfigured()).toBe(false);
- }
- });
-});
diff --git a/site/src/lib/analytics/posthog.ts b/site/src/lib/analytics/posthog.ts
deleted file mode 100644
index 166451a2..00000000
--- a/site/src/lib/analytics/posthog.ts
+++ /dev/null
@@ -1,123 +0,0 @@
-// PostHog read access for the analytics pipeline. Browser ingestion uses the
-// public project key through e.donkeycut.com; these reads go through the
-// query API on the main host, addressed by POSTHOG_PROJECT_ID and authorized
-// by POSTHOG_PERSONAL_API_KEY — both env, and both required before any
-// PostHog extraction runs.
-import { addUtcDays } from "@/lib/analytics/schema";
-
-const POSTHOG_HOST = "https://us.posthog.com";
-
-/** A POSTHOG_PROJECT_ID that is not a project id. Permanent: no retry fixes a
- * bad env value. */
-export class PosthogConfigError extends Error {}
-
-// The numeric project id from PostHog → Settings → Project. Unset leaves this
-// deployment without PostHog. A malformed value raises: a typo here once cost
-// the pipeline months of PostHog data while every run reported success.
-function posthogProjectId(): number | undefined {
- const raw = process.env.POSTHOG_PROJECT_ID?.trim();
- if (!raw) return undefined;
- const id = Number(raw);
- if (!Number.isInteger(id) || id <= 0) {
- throw new PosthogConfigError(
- `POSTHOG_PROJECT_ID must be the numeric project id from PostHog → Settings → Project; got "${raw}".`,
- );
- }
- return id;
-}
-
-export function isPosthogQueryConfigured(): boolean {
- return posthogProjectId() !== undefined && Boolean(process.env.POSTHOG_PERSONAL_API_KEY);
-}
-
-// PostHog's query cluster turns requests away when it is busy (503, 429). A
-// busy answer is momentary, so the run waits and asks again before giving the
-// day up to the next night's backfill.
-const BUSY_STATUSES = new Set([429, 502, 503, 504]);
-// Two waits per day keep a five-day backfill inside the worker's 300 s budget.
-const RETRY_DELAYS_MS = [10_000, 30_000];
-
-async function postQuery(projectId: number, apiKey: string, query: string): Promise<Response> {
- for (let attempt = 0; ; attempt++) {
- const res = await fetch(`${POSTHOG_HOST}/api/projects/${projectId}/query`, {
- body: JSON.stringify({ query: { kind: "HogQLQuery", query } }),
- headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
- method: "POST",
- });
- if (res.ok) return res;
- const detail = (await res.text().catch(() => "")).slice(0, 300);
- const delay = RETRY_DELAYS_MS[attempt];
- if (!BUSY_STATUSES.has(res.status) || delay === undefined) {
- throw new Error(`PostHog query failed (${res.status}): ${detail}`);
- }
- await new Promise((resolve) => setTimeout(resolve, delay));
- }
-}
-
-/** distinct_ids that fired app_loaded during the UTC day. app_loaded fires
- * only after posthog.identify(user.id), so these are app user ids plus
- * whatever anonymous ids a shared browser aliased in. */
-export async function fetchActiveDistinctIds(day: string): Promise<string[]> {
- const apiKey = process.env.POSTHOG_PERSONAL_API_KEY;
- const projectId = posthogProjectId();
- if (!apiKey || projectId === undefined) {
- throw new Error("PostHog query access is not configured.");
- }
- const query = [
- "SELECT DISTINCT distinct_id FROM events",
- "WHERE event = 'app_loaded'",
- `AND timestamp >= toDateTime('${day} 00:00:00', 'UTC')`,
- `AND timestamp < toDateTime('${addUtcDays(day, 1)} 00:00:00', 'UTC')`,
- "LIMIT 10000",
- ].join(" ");
- const res = await postQuery(projectId, apiKey, query);
- const data = (await res.json()) as { results?: unknown[][] };
- const ids = new Set<string>();
- for (const row of data.results ?? []) {
- const id = row?.[0];
- if (typeof id === "string" && id) ids.add(id);
- }
- return [...ids].sort();
-}
-
-const hogqlString = (value: string) => `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
-
-const hogqlTime = (at: Date) => `toDateTime('${at.toISOString().slice(0, 19).replace("T", " ")}', 'UTC')`;
-
-// Each account carries its own cutoff, so the clauses go in an OR list rather
-// than one shared threshold; a chunk this size keeps the query well inside
-// PostHog's limits.
-const ID_CHUNK = 200;
-
-/** Which of these accounts fired the event at or after their own `since`.
- * An account that fired it only before its cutoff is absent. */
-export async function fetchConvertedAfter(
- event: string,
- entries: { distinctId: string; since: Date }[],
-): Promise<Set<string>> {
- const apiKey = process.env.POSTHOG_PERSONAL_API_KEY;
- const projectId = posthogProjectId();
- if (!apiKey || projectId === undefined) {
- throw new Error("PostHog query access is not configured.");
- }
- const converted = new Set<string>();
- for (let i = 0; i < entries.length; i += ID_CHUNK) {
- const chunk = entries.slice(i, i + ID_CHUNK);
- const clauses = chunk
- .map((e) => `(distinct_id = ${hogqlString(e.distinctId)} AND timestamp >= ${hogqlTime(e.since)})`)
- .join(" OR ");
- const query = [
- "SELECT DISTINCT distinct_id FROM events",
- `WHERE event = ${hogqlString(event)}`,
- `AND (${clauses})`,
- `LIMIT ${ID_CHUNK}`,
- ].join(" ");
- const res = await postQuery(projectId, apiKey, query);
- const data = (await res.json()) as { results?: unknown[][] };
- for (const row of data.results ?? []) {
- const id = row?.[0];
- if (typeof id === "string" && id) converted.add(id);
- }
- }
- return converted;
-}
diff --git a/site/src/lib/analytics/schema.ts b/site/src/lib/analytics/schema.ts
index e47dd57b..75dd7535 100644
--- a/site/src/lib/analytics/schema.ts
+++ b/site/src/lib/analytics/schema.ts
@@ -1,168 +1,15 @@
-// The analytics data contract shared by the nightly pipeline, the superuser
-// API, and the dashboard UI. The UI reads rollup.json only — never the
-// database — so this file is the whole interface between them.
+// The analytics rollup as the API reads it. The back office's nightly
+// pipeline writes rollup.json to R2 and keeps its own copy of this contract;
+// the summary and users routes read rollup.json only — never the database —
+// so this file is the whole interface between them.
import { z } from "zod";
export const ANALYTICS_ROLLUP_VERSION = 1;
-// Where the pipeline's files live in R2. Here rather than beside the pipeline
-// so a reader of one file — the rollup route — names it without loading the
-// extraction code.
-const ANALYTICS_PREFIX = "analytics/";
-export const DAYS_PREFIX = `${ANALYTICS_PREFIX}days/`;
-export const dayDbKey = (day: string) => `${DAYS_PREFIX}${day}/db.json`;
-export const dayPosthogKey = (day: string) => `${DAYS_PREFIX}${day}/posthog.json`;
-export const SNAPSHOT_KEY = `${ANALYTICS_PREFIX}snapshot.json`;
-export const ROLLUP_KEY = `${ANALYTICS_PREFIX}rollup.json`;
+// Where the pipeline writes the rollup in R2.
+export const ROLLUP_KEY = "analytics/rollup.json";
-// Bit order for the per-day activity masks. Append-only: a new source takes
-// the next bit so already-written rollups keep decoding.
-export const ANALYTICS_DB_SOURCES = [
- "inference",
- "renders",
- "copies",
- "libraryAssets",
- "templates",
- "creditLedger",
-] as const;
-export const ANALYTICS_SOURCES = [...ANALYTICS_DB_SOURCES, "posthog"] as const;
-export type AnalyticsDbSource = (typeof ANALYTICS_DB_SOURCES)[number];
-export type AnalyticsSource = (typeof ANALYTICS_SOURCES)[number];
-
-export const analyticsDaySchema = z.string().regex(/^\d{4}-\d{2}-\d{2}$/);
-
-/** UTC calendar day of a moment, as YYYY-MM-DD. */
-export function utcDayOf(date: Date): string {
- return date.toISOString().slice(0, 10);
-}
-
-/** A day shifted by whole UTC days. Also normalizes: a string that is not a
- * real calendar day round-trips to a different string at delta 0. */
-export function addUtcDays(day: string, delta: number): string {
- const [y, m, d] = day.split("-").map(Number);
- return utcDayOf(new Date(Date.UTC(y, (m ?? 1) - 1, (d ?? 1) + delta)));
-}
-
-const userIdList = z.array(z.string());
-
-// Per-day extract of DB activity: which users each event table saw that day.
-export const analyticsDbDayFileSchema = z.object({
- version: z.literal(1),
- day: analyticsDaySchema,
- generatedAt: z.string(),
- active: z.object({
- inference: userIdList.optional(),
- renders: userIdList.optional(),
- copies: userIdList.optional(),
- libraryAssets: userIdList.optional(),
- templates: userIdList.optional(),
- creditLedger: userIdList.optional(),
- }),
-});
-export type AnalyticsDbDayFile = z.infer<typeof analyticsDbDayFileSchema>;
-
-export const analyticsPosthogDayFileSchema = z.object({
- version: z.literal(1),
- day: analyticsDaySchema,
- generatedAt: z.string(),
- // Raw distinct_ids. Anonymous pre-sign-in ids ride along and are dropped at
- // consolidation, so this list's length is not a user count.
- activeDistinctIds: userIdList,
-});
-export type AnalyticsPosthogDayFile = z.infer<typeof analyticsPosthogDayFileSchema>;
-
-// What the nightly run and the webhook-driven refresh read from Stripe. Money
-// is BigInt micros as decimal strings; times are ISO. Every row keeps its
-// Stripe id so the dashboard links straight to the object.
-export const analyticsStripeSnapshotSchema = z.object({
- // Root of the Stripe dashboard for this account's mode (live or test).
- dashboardUrl: z.string(),
- // Every Pro subscription ever created, whatever its status now.
- subscriptions: z.array(
- z.object({
- id: z.string(),
- customerId: z.string().nullable(),
- userId: z.string().nullable(),
- email: z.string().nullable(),
- status: z.string(),
- createdAt: z.string(),
- // True while the subscription is live but set to end on its own.
- cancelScheduled: z.boolean(),
- // When it ends (or ended), and when the person asked for that.
- cancelAt: z.string().nullable(),
- canceledAt: z.string().nullable(),
- // When it stopped, once it has.
- endedAt: z.string().nullable().catch(null),
- // What they told the portal on the way out.
- feedback: z.string().nullable(),
- comment: z.string().nullable(),
- }),
- ),
- // Every charge ever attempted: paid ones are the revenue, declined ones are
- // people who tried to pay.
- charges: z.array(
- z.object({
- id: z.string(),
- paymentIntentId: z.string().nullable(),
- customerId: z.string().nullable(),
- userId: z.string().nullable(),
- email: z.string().nullable(),
- day: analyticsDaySchema,
- kind: z.enum(["pro", "topup", "other"]),
- status: z.enum(["paid", "declined"]),
- amountMicros: z.string(),
- failure: z.string().nullable(),
- }),
- ),
- // Refunds issued inside the window.
- refunds: z.array(
- z.object({
- id: z.string(),
- chargeId: z.string().nullable(),
- day: analyticsDaySchema,
- amountMicros: z.string(),
- }),
- ),
- // Checkout pages opened inside the window that expired unpaid.
- abandonedCheckouts: z.array(
- z.object({
- id: z.string(),
- customerId: z.string().nullable(),
- email: z.string().nullable(),
- day: analyticsDaySchema,
- kind: z.enum(["pro", "topup"]),
- }),
- ),
-});
-export type AnalyticsStripeSnapshot = z.infer<typeof analyticsStripeSnapshotSchema>;
-
-export const analyticsSnapshotFileSchema = z.object({
- version: z.literal(1),
- generatedAt: z.string(),
- users: z.array(
- z.object({
- id: z.string(),
- email: z.string(),
- name: z.string(),
- createdAt: z.string(),
- // True for accounts with the super-user role.
- superUser: z.boolean().optional(),
- // The onboarding referral answer; present once the user answered.
- referralAnsweredAt: z.string().optional(),
- referralSources: z.array(z.string()).optional(),
- // The free text that comes with picking "other".
- referralOther: z.string().optional(),
- }),
- ),
- balances: z.array(z.object({ userId: z.string(), balanceMicros: z.string() })),
- // Cloud storage counters: bytes held in R2 per user, as decimal strings.
- // Absent from snapshots written before this shipped.
- storage: z.array(z.object({ userId: z.string(), bytes: z.string() })).optional(),
- // The billing record, read from Stripe. Absent from snapshots written
- // before this shipped.
- stripe: analyticsStripeSnapshotSchema.optional().catch(undefined),
-});
-export type AnalyticsSnapshotFile = z.infer<typeof analyticsSnapshotFileSchema>;
+const analyticsDaySchema = z.string().regex(/^\d{4}-\d{2}-\d{2}$/);
// The consolidated window the UI renders. Users come from the latest snapshot,
// so an account deleted since then carries no history here.
@@ -171,7 +18,7 @@ export const analyticsRollupSchema = z.object({
generatedAt: z.string(),
// Oldest → newest; every user's activity array aligns with this.
days: z.array(analyticsDaySchema),
- // Bit order of the activity masks (copy of ANALYTICS_SOURCES at write time).
+ // Bit order of the activity masks (the pipeline's source list at write time).
sources: z.array(z.string()),
// Day files that were absent or unreadable at consolidation; the UI can gray
// those columns out. Entries name the file kind: "db" or "posthog".
diff --git a/site/src/lib/analytics/stripe.ts b/site/src/lib/analytics/stripe.ts
deleted file mode 100644
index ca7a92b2..00000000
--- a/site/src/lib/analytics/stripe.ts
+++ /dev/null
@@ -1,156 +0,0 @@
-// The billing side of the analytics snapshot, read straight from Stripe.
-// Stripe is where the money is, so its charges, refunds, subscriptions and
-// abandoned checkouts are the record; every row keeps the Stripe id so the
-// dashboard can link into it.
-
-import { drainStripe, getStripe, stripeId, StripeNotConfiguredError } from "@/lib/billing/stripe";
-import { subscriptionCancelScheduled, subscriptionIsPro, syncProSubscription } from "@/lib/billing/pro-subscription";
-import { creditMicrosPerCent } from "@/lib/credits/amounts";
-import { creditAutoReloadKind, creditTopUpKind } from "@/lib/credits/top-up";
-import { type AnalyticsStripeSnapshot, utcDayOf } from "@/lib/analytics/schema";
-import { prisma } from "@/lib/prisma";
-
-const PAGE = { limit: 100 } as const;
-// The credit-purchase kinds stamped on a PaymentIntent at checkout.
-const TOPUP_KINDS = new Set<string>([creditTopUpKind, creditAutoReloadKind]);
-
-const microsOf = (cents: number) => (BigInt(cents) * creditMicrosPerCent).toString();
-const dayOfUnix = (seconds: number) => utcDayOf(new Date(seconds * 1000));
-const isoOfUnix = (seconds: number) => new Date(seconds * 1000).toISOString();
-
-// The dashboard for the account the key belongs to: a test-mode key names
-// test-mode objects, which live under /test.
-export function stripeDashboardUrl(): string {
- const key = process.env.STRIPE_SECRET_KEY;
- if (!key) throw new StripeNotConfiguredError();
- return key.includes("_test_") ? "https://dashboard.stripe.com/test" : "https://dashboard.stripe.com";
-}
-
-/** Everything the rollup's billing section is derived from. `since` bounds
- * the window-scoped lists (refunds, declines, abandoned checkouts); charges
- * and subscriptions come whole because the funded-all-time figures need them. */
-export async function pullStripeSnapshot(since: string): Promise<AnalyticsStripeSnapshot> {
- const stripe = getStripe();
- const sinceUnix = Math.floor(Date.parse(`${since}T00:00:00Z`) / 1000);
-
- // A customer names its user through the metadata stamped at creation, or
- // through the customer id the user row keeps, the same two routes the
- // webhook sync resolves by.
- const customers = await drainStripe(stripe.customers.list(PAGE));
- const customerById = new Map(customers.map((c) => [c.id, c]));
- const userByCustomer = new Map(
- (
- await prisma.user.findMany({
- select: { id: true, stripeCustomerId: true },
- where: { stripeCustomerId: { in: customers.map((c) => c.id) } },
- })
- ).map((user) => [user.stripeCustomerId as string, user.id]),
- );
- const customerOf = (ref: string | { id: string } | null | undefined) => {
- const id = stripeId(ref);
- const customer = id ? customerById.get(id) : undefined;
- return {
- customerId: id,
- email: customer?.email ?? null,
- userId: customer?.metadata?.userId ?? (id ? (userByCustomer.get(id) ?? null) : null),
- };
- };
-
- // Stripe lists newest first. A user has one row in our database, so only
- // their newest subscription is synced into it: that row follows Stripe
- // here too, and a cancel scheduled through the portal reaches the
- // account's settings card without waiting on a webhook.
- const subscriptions: AnalyticsStripeSnapshot["subscriptions"] = [];
- const synced = new Set<string>();
- for (const sub of await drainStripe(stripe.subscriptions.list({ ...PAGE, status: "all" }))) {
- if (!subscriptionIsPro(sub)) continue;
- const who = customerOf(sub.customer);
- const owner = sub.metadata?.userId ?? who.userId ?? who.customerId ?? sub.id;
- if (!synced.has(owner)) {
- synced.add(owner);
- await syncProSubscription(sub);
- }
- subscriptions.push({
- cancelAt: sub.cancel_at !== null ? isoOfUnix(sub.cancel_at) : null,
- canceledAt: sub.canceled_at !== null ? isoOfUnix(sub.canceled_at) : null,
- cancelScheduled: subscriptionCancelScheduled(sub),
- comment: sub.cancellation_details?.comment ?? null,
- createdAt: isoOfUnix(sub.created),
- customerId: who.customerId,
- email: who.email,
- endedAt: sub.ended_at !== null ? isoOfUnix(sub.ended_at) : null,
- feedback: sub.cancellation_details?.feedback ?? null,
- id: sub.id,
- status: sub.status,
- userId: sub.metadata?.userId ?? who.userId,
- });
- }
-
- // A charge is a top-up when its PaymentIntent carries a credit-purchase
- // kind, Pro when its customer holds a Pro subscription, and otherwise
- // "other": money the dashboard shows as its own bucket.
- const proCustomers = new Set(subscriptions.map((sub) => sub.customerId));
- const charges: AnalyticsStripeSnapshot["charges"] = [];
- for (const charge of await drainStripe(
- stripe.charges.list({ ...PAGE, expand: ["data.payment_intent"] }),
- )) {
- if (charge.status === "pending") continue;
- const intent =
- typeof charge.payment_intent === "object" ? charge.payment_intent : null;
- const who = customerOf(charge.customer);
- charges.push({
- amountMicros: microsOf(charge.amount),
- customerId: who.customerId,
- day: dayOfUnix(charge.created),
- email: charge.billing_details?.email ?? who.email,
- failure: charge.failure_message ?? charge.failure_code,
- id: charge.id,
- kind: TOPUP_KINDS.has(intent?.metadata?.kind ?? "")
- ? "topup"
- : proCustomers.has(who.customerId)
- ? "pro"
- : "other",
- paymentIntentId: stripeId(charge.payment_intent),
- status: charge.status === "succeeded" && charge.paid ? "paid" : "declined",
- userId: intent?.metadata?.userId ?? who.userId,
- });
- }
-
- const refunds: AnalyticsStripeSnapshot["refunds"] = [];
- for (const refund of await drainStripe(
- stripe.refunds.list({ ...PAGE, created: { gte: sinceUnix } }),
- )) {
- if (refund.status === "failed" || refund.status === "canceled") continue;
- refunds.push({
- amountMicros: microsOf(refund.amount),
- chargeId: stripeId(refund.charge),
- day: dayOfUnix(refund.created),
- id: refund.id,
- });
- }
-
- // A checkout that expired unpaid is someone who opened the payment page and
- // walked away; an open session may still complete, so it waits.
- const abandonedCheckouts: AnalyticsStripeSnapshot["abandonedCheckouts"] = [];
- for (const session of await drainStripe(
- stripe.checkout.sessions.list({ ...PAGE, created: { gte: sinceUnix } }),
- )) {
- if (session.status !== "expired") continue;
- const who = customerOf(session.customer);
- abandonedCheckouts.push({
- customerId: who.customerId,
- day: dayOfUnix(session.created),
- email: session.customer_details?.email ?? who.email,
- id: session.id,
- kind: session.mode === "subscription" ? "pro" : "topup",
- });
- }
-
- return {
- abandonedCheckouts,
- charges,
- dashboardUrl: stripeDashboardUrl(),
- refunds,
- subscriptions,
- };
-}
diff --git a/site/src/lib/analytics/summary-fixture.test.ts b/site/src/lib/analytics/summary-fixture.test.ts
index 2394d083..75988afe 100644
--- a/site/src/lib/analytics/summary-fixture.test.ts
+++ b/site/src/lib/analytics/summary-fixture.test.ts
@@ -6,17 +6,19 @@ import { fileURLToPath } from "node:url";
import {
SUMMARY_FIXTURE_PATH,
USERS_FIXTURE_PATH,
- buildRollupFixture,
buildSummaryFixture,
buildUsersFixture,
summaryFixtureJson,
+ readRollupFixture,
usersFixtureJson,
} from "./summary-fixture";
import { analyticsRollupSchema } from "./schema";
+// The rollup fixture is the back office's: its own test builds it with the
+// buildRollup production runs, and this copy must match it byte for byte.
describe("the phone's summary fixture", () => {
test("is a rollup the dashboards read", async () => {
- const parsed = analyticsRollupSchema.parse(await buildRollupFixture());
+ const parsed = analyticsRollupSchema.parse(readRollupFixture());
expect(parsed.days).toHaveLength(3);
expect(parsed.billing?.days).toHaveLength(4);
expect(parsed.missing).toEqual([{ day: "2026-08-31", sources: ["db", "posthog"] }]);
diff --git a/site/src/lib/analytics/summary-fixture.ts b/site/src/lib/analytics/summary-fixture.ts
index ea5af9ea..3a9eb257 100644
--- a/site/src/lib/analytics/summary-fixture.ts
+++ b/site/src/lib/analytics/summary-fixture.ts
@@ -1,107 +1,35 @@
-// The summary the phone's model is tested against. It is the real
-// consolidation (buildRollup) run over a small fixed record, folded by the
-// same summarizeRollup the API answers with, so the JSON says exactly what
-// /api/analytics/summary serves today.
+// The fixtures the phone's model is tested against: the summary and the
+// first page of accounts, folded out of the contract's rollup fixture by the
+// same summarizeRollup and rankUsers the API answers with, so the JSON says
+// exactly what /api/analytics/summary and /api/analytics/users serve.
//
-// The checked-in copy lives with the DonkeyKit tests; `npm run
-// analytics:summary-fixture` rewrites it. summary-fixture.test.ts fails while
-// the copy is behind this code, and the DonkeyKit tests decode the copy, so a
-// change to the summary's shape reaches the phone in the same change.
-import { buildRollup } from "./pipeline";
+// The rollup fixture (fixtures/analytics-rollup.json) is a byte-identical
+// copy of the one the back office's rollup-fixture.test.ts builds with the
+// buildRollup that writes production rollups. The folded copies live with the
+// DonkeyKit tests; `npm run analytics:summary-fixture` rewrites them.
+// summary-fixture.test.ts fails while they are behind this code, and the
+// DonkeyKit tests decode them, so a change to the summary's shape reaches the
+// phone in the same change.
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
import { rankUsers, USER_SORTS, USERS_PAGE_SIZE, type AnalyticsUsersPage } from "./rank";
import { summarizeRollup, type AnalyticsSummary } from "./summarize";
-import type { AnalyticsRollup, AnalyticsSnapshotFile } from "./schema";
-import type { AnalyticsDayFiles } from "./pipeline";
+import { analyticsRollupSchema, type AnalyticsRollup } from "./schema";
export const SUMMARY_FIXTURE_PATH =
"apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-summary.json";
export const USERS_FIXTURE_PATH =
"apps/ios/DonkeyKit/Tests/DonkeyKitModelsTests/Fixtures/analytics-users.json";
-// Three closed days of activity, and a billing window that runs one day
-// further, to today: the two windows never line up by index.
-const DAYS = ["2026-08-30", "2026-08-31", "2026-09-01"];
-const BILLING_DAYS = [...DAYS, "2026-09-02"];
-
-const snapshot: AnalyticsSnapshotFile = {
- balances: [
- { balanceMicros: "3000000", userId: "u_pro" },
- { balanceMicros: "500000", userId: "u_su" },
- ],
- generatedAt: "2026-09-02T04:00:00.000Z",
- storage: [{ bytes: "1048576", userId: "u_pro" }],
- stripe: {
- abandonedCheckouts: [
- { customerId: "cus_a", day: "2026-08-31", email: "[email protected]", id: "cs_1", kind: "pro" },
- ],
- charges: [
- { amountMicros: "20000000", customerId: "cus_p", day: "2026-08-30", email: "[email protected]", failure: null, id: "ch_1", kind: "pro", paymentIntentId: "pi_1", status: "paid", userId: "u_pro" },
- { amountMicros: "5000000", customerId: "cus_n", day: "2026-09-01", email: "[email protected]", failure: null, id: "ch_2", kind: "topup", paymentIntentId: "pi_2", status: "paid", userId: "u_new" },
- { amountMicros: "20000000", customerId: "cus_d", day: "2026-08-31", email: "[email protected]", failure: "Your card has insufficient funds.", id: "ch_3", kind: "pro", paymentIntentId: "pi_3", status: "declined", userId: null },
- ],
- dashboardUrl: "https://dashboard.stripe.com/test",
- refunds: [{ amountMicros: "1000000", chargeId: "ch_2", day: "2026-09-02", id: "re_1" }],
- subscriptions: [
- { cancelAt: "2026-09-08T00:00:00.000Z", cancelScheduled: true, canceledAt: "2026-08-31T10:00:00.000Z", comment: "Export broke.", createdAt: "2026-08-09T00:00:00.000Z", customerId: "cus_p", email: "[email protected]", endedAt: null, feedback: "switched_service", id: "sub_1", status: "active", userId: "u_pro" },
- { cancelAt: null, cancelScheduled: false, canceledAt: null, comment: null, createdAt: "2026-07-01T00:00:00.000Z", customerId: "cus_s", email: "[email protected]", endedAt: null, feedback: null, id: "sub_2", status: "active", userId: "u_su" },
- { cancelAt: null, cancelScheduled: false, canceledAt: "2026-07-15T00:00:00.000Z", comment: null, createdAt: "2026-06-01T00:00:00.000Z", customerId: "cus_r", email: "[email protected]", endedAt: "2026-07-01T00:00:00.000Z", feedback: null, id: "sub_3", status: "canceled", userId: "u_gone" },
- ],
- },
- users: [
- { createdAt: "2026-08-01T10:00:00.000Z", email: "[email protected]", id: "u_pro", name: "Pro Person" },
- {
- createdAt: "2026-08-31T12:00:00.000Z",
- email: "[email protected]",
- id: "u_new",
- name: "New Person",
- referralAnsweredAt: "2026-08-31T12:05:00.000Z",
- referralOther: "a podcast",
- referralSources: ["youtube", "other"],
- },
- {
- createdAt: "2026-07-01T00:00:00.000Z",
- email: "[email protected]",
- id: "u_su",
- name: "Super Person",
- superUser: true,
- },
- ],
- version: 1,
-};
-
-const dayFiles = new Map<string, AnalyticsDayFiles>([
- [
- "2026-08-30",
- {
- db: { active: { inference: ["u_pro"], renders: ["u_su"] }, day: "2026-08-30", generatedAt: "2026-08-31T04:00:00.000Z", version: 1 },
- posthog: { activeDistinctIds: ["u_pro"], day: "2026-08-30", generatedAt: "2026-08-31T04:00:00.000Z", version: 1 },
- },
- ],
- // A night the extraction missed: both files absent.
- ["2026-08-31", { db: null, posthog: null }],
- [
- "2026-09-01",
- {
- db: { active: { renders: ["u_su"] }, day: "2026-09-01", generatedAt: "2026-09-02T04:00:00.000Z", version: 1 },
- // An anonymous pre-sign-in id rides along and drops out.
- posthog: { activeDistinctIds: ["u_pro", "u_new", "anon-7f3a"], day: "2026-09-01", generatedAt: "2026-09-02T04:00:00.000Z", version: 1 },
- },
- ],
-]);
+const ROLLUP_FIXTURE = new URL("./fixtures/analytics-rollup.json", import.meta.url);
-export function buildRollupFixture(): Promise<AnalyticsRollup> {
- return buildRollup({
- billingDays: BILLING_DAYS,
- days: DAYS,
- generatedAt: "2026-09-02T04:00:00.000Z",
- posthogConfigured: true,
- readDayFiles: async (day) => dayFiles.get(day) ?? { db: null, posthog: null },
- snapshot,
- });
+export function readRollupFixture(): AnalyticsRollup {
+ return analyticsRollupSchema.parse(JSON.parse(readFileSync(fileURLToPath(ROLLUP_FIXTURE), "utf8")));
}
export async function buildSummaryFixture(): Promise<AnalyticsSummary> {
- return summarizeRollup(await buildRollupFixture());
+ return summarizeRollup(readRollupFixture());
}
export async function summaryFixtureJson(): Promise<string> {
@@ -110,7 +38,7 @@ export async function summaryFixtureJson(): Promise<string> {
/** The first page of accounts, exactly as the users route answers it. */
export async function buildUsersFixture(): Promise<AnalyticsUsersPage> {
- const rollup = await buildRollupFixture();
+ const rollup = readRollupFixture();
const ranked = rankUsers(rollup, "active");
const end = USERS_PAGE_SIZE;
return {
diff --git a/site/src/lib/credits/inference.ts b/site/src/lib/credits/inference.ts
index 3115c123..875c2370 100644
--- a/site/src/lib/credits/inference.ts
+++ b/site/src/lib/credits/inference.ts
@@ -248,59 +248,6 @@ export async function expireCredits(userId: string, now = new Date()) {
));
}
-// Expiry is otherwise applied when an account is read or charged, so an
-// account nobody touched after its grant lapsed still carries the lapsed
-// amount in balanceMicros. The nightly sweep settles those accounts here.
-// Each account is its own serializable transaction, the same path a live read
-// takes; an account whose transaction keeps losing to live charges is logged
-// and skipped, and the sweep stops at its time budget so the job never runs
-// past its worker. Whatever is left waits for the next run, or for the
-// account's next read.
-export async function expireAllCredits(now = new Date(), budgetMs = 240_000) {
- const startedAt = Date.now();
- const overdue = await prisma.userCreditGrant.findMany({
- distinct: ["userId"],
- orderBy: { expiresAt: "asc" },
- select: { userId: true },
- where: {
- expiresAt: { lte: now },
- remainingAmountMicros: { gt: zeroCreditMicros },
- status: "active",
- },
- });
- let settled = 0;
- let failed = 0;
- let index = 0;
- for (; index < overdue.length; index++) {
- if (Date.now() - startedAt > budgetMs) break;
- const { userId } = overdue[index];
- try {
- await expireCredits(userId, now);
- settled++;
- } catch (error) {
- failed++;
- console.error("[credit-expiry] failed to settle an account", { error, userId });
- }
- }
- return { failed, remaining: overdue.length - index, settled };
-}
-
-// What each account could spend right now, with grants past their expiry
-// taken off even where the sweep has not reached the account yet. A read,
-// so a snapshot of every balance costs one aggregate on top of the page scan.
-export async function lapsedCreditByUser(now = new Date()) {
- const rows = await prisma.userCreditGrant.groupBy({
- _sum: { remainingAmountMicros: true },
- by: ["userId"],
- where: {
- expiresAt: { lte: now },
- remainingAmountMicros: { gt: zeroCreditMicros },
- status: "active",
- },
- });
- return new Map(rows.map((row) => [row.userId, row._sum.remainingAmountMicros ?? zeroCreditMicros]));
-}
-
export async function assertCanUseInference(input: CreditPreflightInput) {
await ensureCreditAccount(input.userId);
await expireCredits(input.userId);