Skip to content

Open source CapCut alternative.

TypeScript

feat(site): add nightly analytics rollup pipeline

David Le38a4ff324771
diff --git a/site/.env.example b/site/.env.example
index 66960916..05b60f1b 100644
--- a/site/.env.example
+++ b/site/.env.example
@@ -61,6 +61,13 @@ RESEND_FROM_EMAIL="Sender Name <[email protected]>"
# Leave unset to disable tracking (e.g. local dev).
NEXT_PUBLIC_POSTHOG_KEY="replace-with-posthog-project-key"
+# PostHog query access for the nightly analytics job's HogQL reads: the
+# personal API key (query-read scope) and the numeric project id from
+# PostHog → Settings → Project. Optional: unset either and the job skips
+# PostHog extraction, reporting DB metrics only.
+POSTHOG_PERSONAL_API_KEY="replace-with-posthog-personal-api-key"
+POSTHOG_PROJECT_ID="replace-with-posthog-numeric-project-id"
+
# Cloudflare R2 (Cut cloud-mode media storage)
R2_ACCOUNT_ID="replace-with-r2-account-id"
R2_ACCESS_KEY_ID="replace-with-r2-access-key-id"
diff --git a/site/src/app/api/analytics/rollup/route.ts b/site/src/app/api/analytics/rollup/route.ts
new file mode 100644
index 00000000..1d943fb1
--- /dev/null
+++ b/site/src/app/api/analytics/rollup/route.ts
@@ -0,0 +1,26 @@
+import { NextResponse } from "next/server";
+
+import { getObject, R2NotConfiguredError } from "@/cut/server/cloud/r2";
+import { ROLLUP_KEY } from "@/lib/analytics/pipeline";
+import { withSuperUser } from "@/lib/donkey-api-auth";
+
+export const dynamic = "force-dynamic";
+
+// The consolidated analytics JSON, passed through exactly as the nightly job
+// wrote it. It carries emails and balances, so it stays behind the super-user
+// gate; the dashboard reads this and never the database.
+export const GET = withSuperUser(async () => {
+ let rollup;
+ try {
+ rollup = await getObject(ROLLUP_KEY);
+ } catch (e) {
+ if (e instanceof R2NotConfiguredError) {
+ return NextResponse.json({ error: "Storage is not configured." }, { status: 503 });
+ }
+ throw e;
+ }
+ if (!rollup) return NextResponse.json({ error: "No rollup yet." }, { status: 404 });
+ return new NextResponse(new Uint8Array(rollup.bytes), {
+ headers: { "Cache-Control": "no-store", "Content-Type": "application/json" },
+ });
+});
diff --git a/site/src/app/api/analytics/run/route.ts b/site/src/app/api/analytics/run/route.ts
new file mode 100644
index 00000000..a1188fa1
--- /dev/null
+++ b/site/src/app/api/analytics/run/route.ts
@@ -0,0 +1,23 @@
+import { NextRequest, NextResponse } from "next/server";
+
+import { withSuperUser } from "@/lib/donkey-api-auth";
+import { enqueueJob } from "@/lib/jobs/queue";
+
+export const dynamic = "force-dynamic";
+// Without a queue configured (local dev) the job runs inline before the
+// response; give it room.
+export const maxDuration = 300;
+
+async function enqueueNightly(createdBy: string) {
+ return NextResponse.json(await enqueueJob("analytics-daily", {}, createdBy));
+}
+
+const manual = withSuperUser((request) => enqueueNightly(request.donkey.userId));
+
+// The nightly analytics run. Vercel's cron carries x-vercel-cron, which the
+// platform strips from outside traffic; anyone else must be a signed-in super
+// user. Per-day retriggers go through POST /api/jobs with {day, force}.
+export const GET = (request: NextRequest) =>
+ request.headers.get("x-vercel-cron") === "1"
+ ? enqueueNightly("vercel-cron")
+ : manual(request);
diff --git a/site/src/cut/server/cloud/gc.ts b/site/src/cut/server/cloud/gc.ts
index 69582626..ae38e4c1 100644
--- a/site/src/cut/server/cloud/gc.ts
+++ b/site/src/cut/server/cloud/gc.ts
@@ -315,7 +315,9 @@ export async function runGc(): Promise<Response> {
: { count: 0 };
// Scratch inference media, swept by prefix: it never counted toward usage and
- // no row points at it, so age is the only thing that makes it garbage.
+ // no row points at it, so age is the only thing that makes it garbage. The
+ // nightly analytics pipeline also keeps rowless JSON under analytics/ — any
+ // future sweep of unknown prefixes must leave that tree alone.
const scratch = await listOlderThan(
INFERENCE_PREFIX,
new Date(Date.now() - SCRATCH_MAX_AGE_MS)
diff --git a/site/src/cut/server/cloud/r2.ts b/site/src/cut/server/cloud/r2.ts
index 2c73d771..320d67ff 100644
--- a/site/src/cut/server/cloud/r2.ts
+++ b/site/src/cut/server/cloud/r2.ts
@@ -165,6 +165,32 @@ export async function listUnder(prefix: string, limit = 10_000): Promise<string[
return out;
}
+/** Keys and last-write times under `prefix` — freshness checks without
+ * per-object GETs. */
+export async function listObjectsWithDates(
+ prefix: string,
+ limit = 10_000
+): Promise<{ key: string; lastModified: Date }[]> {
+ const out: { key: string; lastModified: Date }[] = [];
+ let token: string | undefined;
+ do {
+ const res = await r2().send(
+ new ListObjectsV2Command({
+ Bucket: R2_BUCKET,
+ Prefix: prefix,
+ ContinuationToken: token,
+ MaxKeys: 1000,
+ })
+ );
+ for (const o of res.Contents ?? []) {
+ if (o.Key && o.LastModified) out.push({ key: o.Key, lastModified: o.LastModified });
+ if (out.length >= limit) return out;
+ }
+ token = res.IsTruncated ? res.NextContinuationToken : undefined;
+ } while (token);
+ return out;
+}
+
/** Keys under `prefix` last written before `before`, paged to `limit` objects.
* For prefix sweeps of objects no database row tracks. */
export async function listOlderThan(
diff --git a/site/src/lib/analytics/pipeline.ts b/site/src/lib/analytics/pipeline.ts
new file mode 100644
index 00000000..70f00944
--- /dev/null
+++ b/site/src/lib/analytics/pipeline.ts
@@ -0,0 +1,297 @@
+// Nightly analytics pipeline. Extraction writes one small JSON file per day
+// per source to R2, a snapshot of users and balances is rewritten every run,
+// and consolidation folds the window into the single rollup.json the
+// superuser dashboard reads. Idempotency is per day: a day file is *final*
+// once it was written after its UTC day closed, and final files are skipped
+// unless forced — so a premature run of today, a per-day retrigger, and the
+// regular nightly run all compose without redoing finished work.
+import type { z } from "zod";
+
+import type { Prisma } from "@/generated/prisma/client";
+import { getObject, listObjectsWithDates, putObject } from "@/cut/server/cloud/r2";
+import { fetchActiveDistinctIds, isPosthogQueryConfigured } from "@/lib/analytics/posthog";
+import {
+ addUtcDays,
+ ANALYTICS_DB_SOURCES,
+ ANALYTICS_ROLLUP_VERSION,
+ ANALYTICS_SOURCES,
+ analyticsDbDayFileSchema,
+ analyticsPosthogDayFileSchema,
+ type AnalyticsDbDayFile,
+ type AnalyticsPosthogDayFile,
+ type AnalyticsRollup,
+ type AnalyticsSnapshotFile,
+ utcDayOf,
+} from "@/lib/analytics/schema";
+import { prisma } from "@/lib/prisma";
+
+const ANALYTICS_PREFIX = "analytics/";
+const DAYS_PREFIX = `${ANALYTICS_PREFIX}days/`;
+export const dayDbKey = (day: string) => `${DAYS_PREFIX}${day}/db.json`;
+export const dayPosthogKey = (day: string) => `${DAYS_PREFIX}${day}/posthog.json`;
+export const SNAPSHOT_KEY = `${ANALYTICS_PREFIX}snapshot.json`;
+export const ROLLUP_KEY = `${ANALYTICS_PREFIX}rollup.json`;
+
+const WINDOW_DAYS = 60;
+// Backfill after an outage converges over nights instead of one heavy run.
+const MAX_EXTRACT_DAYS_PER_RUN = 5;
+const SNAPSHOT_PAGE_SIZE = 1000;
+
+export class InvalidDayError extends Error {}
+
+export async function runAnalyticsDaily(payload: {
+ day?: string;
+ force?: boolean;
+}): Promise<Prisma.InputJsonValue> {
+ const today = utcDayOf(new Date());
+ if (payload.day !== undefined) {
+ if (addUtcDays(payload.day, 0) !== payload.day) {
+ throw new InvalidDayError(`"${payload.day}" is not a calendar day.`);
+ }
+ if (payload.day > today) {
+ throw new InvalidDayError(`"${payload.day}" is in the future.`);
+ }
+ }
+
+ const posthogConfigured = isPosthogQueryConfigured();
+ const lastWrite = new Map(
+ (await listObjectsWithDates(DAYS_PREFIX)).map((o) => [o.key, o.lastModified]),
+ );
+ // Final = written after the UTC day closed. A premature mid-day extraction
+ // of today stays non-final, so the nightly run redoes it once the day is
+ // whole; a manual retrigger of a past day skips whatever already finished.
+ const isFinal = (key: string, day: string) => {
+ const at = lastWrite.get(key);
+ return at !== undefined && at.getTime() >= Date.parse(`${addUtcDays(day, 1)}T00:00:00Z`);
+ };
+ const isComplete = (day: string) =>
+ isFinal(dayDbKey(day), day) && (!posthogConfigured || isFinal(dayPosthogKey(day), day));
+
+ const yesterday = addUtcDays(today, -1);
+ let targets: string[];
+ if (payload.day) {
+ targets = [payload.day];
+ } else {
+ targets = [];
+ for (let i = 0; i < WINDOW_DAYS; i++) {
+ const day = addUtcDays(yesterday, -i); // newest first
+ if (!isComplete(day)) targets.push(day);
+ }
+ }
+ const deferred = Math.max(0, targets.length - MAX_EXTRACT_DAYS_PER_RUN);
+ targets = targets.slice(0, MAX_EXTRACT_DAYS_PER_RUN);
+
+ // A failed source is recorded and retried by the next night's backfill; it
+ // never blocks the other source, other days, the snapshot, or the rollup.
+ const errors: { day: string; source: string; message: string }[] = [];
+ let extracted = 0;
+ let skipped = 0;
+ for (const day of targets) {
+ if (!payload.force && isFinal(dayDbKey(day), day)) {
+ skipped++;
+ } else {
+ try {
+ await putJson(dayDbKey(day), await extractDbDay(day));
+ extracted++;
+ } catch (e) {
+ errors.push({ day, message: messageOf(e), source: "db" });
+ }
+ }
+ if (!posthogConfigured) continue;
+ if (!payload.force && isFinal(dayPosthogKey(day), day)) {
+ skipped++;
+ } else {
+ try {
+ await putJson(dayPosthogKey(day), await extractPosthogDay(day));
+ extracted++;
+ } catch (e) {
+ errors.push({ day, message: messageOf(e), source: "posthog" });
+ }
+ }
+ }
+
+ // Snapshot and rollup errors propagate: they mean the database or storage
+ // itself is unhealthy, which is the queue's transient-retry case.
+ const snapshot = await writeSnapshot();
+ const rollup = await consolidate(snapshot, posthogConfigured);
+
+ return {
+ deferred,
+ errors,
+ extracted,
+ rollupDays: rollup.days.length,
+ rollupUsers: rollup.users.length,
+ skipped,
+ snapshotUsers: snapshot.users.length,
+ targetedDays: targets,
+ };
+}
+
+async function extractDbDay(day: string): Promise<AnalyticsDbDayFile> {
+ const range = {
+ createdAt: {
+ gte: new Date(`${day}T00:00:00Z`),
+ lt: new Date(`${addUtcDays(day, 1)}T00:00:00Z`),
+ },
+ };
+ // One bounded group-by per event table, run serially to keep the load light.
+ // Every table indexes (userId, createdAt).
+ const queries = {
+ copies: () => prisma.cutCopyJob.groupBy({ by: ["userId"], where: range }),
+ creditLedger: () => prisma.userCreditLedgerEntry.groupBy({ by: ["userId"], where: range }),
+ inference: () => prisma.inferenceUsageEvent.groupBy({ by: ["userId"], where: range }),
+ libraryAssets: () => prisma.cutLibraryAsset.groupBy({ by: ["userId"], where: range }),
+ renders: () => prisma.cutRenderJob.groupBy({ by: ["userId"], where: range }),
+ templates: () => prisma.cutTemplate.groupBy({ by: ["userId"], where: range }),
+ };
+ const active: AnalyticsDbDayFile["active"] = {};
+ for (const source of ANALYTICS_DB_SOURCES) {
+ active[source] = (await queries[source]()).map((row) => row.userId).sort();
+ }
+ return { active, day, generatedAt: new Date().toISOString(), version: 1 };
+}
+
+async function extractPosthogDay(day: string): Promise<AnalyticsPosthogDayFile> {
+ return {
+ activeDistinctIds: await fetchActiveDistinctIds(day),
+ day,
+ generatedAt: new Date().toISOString(),
+ version: 1,
+ };
+}
+
+async function writeSnapshot(): Promise<AnalyticsSnapshotFile> {
+ const users: AnalyticsSnapshotFile["users"] = [];
+ let userCursor: string | undefined;
+ for (;;) {
+ const page = await prisma.user.findMany({
+ orderBy: { id: "asc" },
+ select: { createdAt: true, email: true, id: true, name: true },
+ take: SNAPSHOT_PAGE_SIZE,
+ ...(userCursor === undefined ? {} : { cursor: { id: userCursor }, skip: 1 }),
+ });
+ for (const u of page) {
+ users.push({ createdAt: u.createdAt.toISOString(), email: u.email, id: u.id, name: u.name });
+ }
+ if (page.length < SNAPSHOT_PAGE_SIZE) break;
+ userCursor = page[page.length - 1].id;
+ }
+
+ const balances: AnalyticsSnapshotFile["balances"] = [];
+ let accountCursor: string | undefined;
+ for (;;) {
+ const page = await prisma.userCreditAccount.findMany({
+ orderBy: { id: "asc" },
+ select: { balanceMicros: true, id: true, userId: true },
+ take: SNAPSHOT_PAGE_SIZE,
+ ...(accountCursor === undefined ? {} : { cursor: { id: accountCursor }, skip: 1 }),
+ });
+ for (const account of page) {
+ balances.push({ balanceMicros: account.balanceMicros.toString(), userId: account.userId });
+ }
+ if (page.length < SNAPSHOT_PAGE_SIZE) break;
+ accountCursor = page[page.length - 1].id;
+ }
+
+ const snapshot: AnalyticsSnapshotFile = {
+ balances,
+ generatedAt: new Date().toISOString(),
+ users,
+ version: 1,
+ };
+ await putJson(SNAPSHOT_KEY, snapshot);
+ return snapshot;
+}
+
+async function consolidate(
+ snapshot: AnalyticsSnapshotFile,
+ posthogConfigured: boolean,
+): Promise<AnalyticsRollup> {
+ const yesterday = addUtcDays(utcDayOf(new Date()), -1);
+ const days: string[] = [];
+ for (let i = WINDOW_DAYS - 1; i >= 0; i--) days.push(addUtcDays(yesterday, -i));
+
+ const masks = new Map<string, number[]>();
+ for (const user of snapshot.users) masks.set(user.id, days.map(() => 0));
+
+ const posthogBit = 1 << ANALYTICS_SOURCES.indexOf("posthog");
+ const missing: AnalyticsRollup["missing"] = [];
+ for (let i = 0; i < days.length; i++) {
+ const day = days[i];
+ const absent: string[] = [];
+
+ const dbFile = readJson(await getObject(dayDbKey(day)), analyticsDbDayFileSchema);
+ if (!dbFile) {
+ absent.push("db");
+ } else {
+ for (const source of ANALYTICS_DB_SOURCES) {
+ const bit = 1 << ANALYTICS_SOURCES.indexOf(source);
+ for (const id of dbFile.active[source] ?? []) {
+ const mask = masks.get(id);
+ if (mask) mask[i] |= bit;
+ }
+ }
+ }
+
+ if (posthogConfigured) {
+ const posthogFile = readJson(
+ await getObject(dayPosthogKey(day)),
+ analyticsPosthogDayFileSchema,
+ );
+ if (!posthogFile) {
+ absent.push("posthog");
+ } else {
+ // Anonymous pre-sign-in distinct_ids match no snapshot user and drop out.
+ for (const id of posthogFile.activeDistinctIds) {
+ const mask = masks.get(id);
+ if (mask) mask[i] |= posthogBit;
+ }
+ }
+ }
+
+ if (absent.length > 0) missing.push({ day, sources: absent });
+ }
+
+ const balanceByUser = new Map(snapshot.balances.map((b) => [b.userId, b.balanceMicros]));
+ const users = snapshot.users
+ .map((u) => ({
+ activity: masks.get(u.id) ?? days.map(() => 0),
+ balanceMicros: balanceByUser.get(u.id) ?? "0",
+ email: u.email,
+ id: u.id,
+ name: u.name,
+ registeredAt: u.createdAt,
+ }))
+ .sort((a, b) => (a.registeredAt < b.registeredAt ? 1 : -1));
+
+ const rollup: AnalyticsRollup = {
+ days,
+ generatedAt: new Date().toISOString(),
+ missing,
+ sources: [...ANALYTICS_SOURCES],
+ users,
+ version: ANALYTICS_ROLLUP_VERSION,
+ };
+ await putJson(ROLLUP_KEY, rollup);
+ return rollup;
+}
+
+async function putJson(key: string, value: unknown): Promise<void> {
+ await putObject(key, Buffer.from(JSON.stringify(value)), "application/json");
+}
+
+/** Parse a stored JSON object against its schema; anything unreadable — gone,
+ * corrupt, or from an incompatible version — reads as missing. */
+function readJson<T>(obj: { bytes: Buffer } | null, schema: z.ZodType<T>): T | null {
+ if (!obj) return null;
+ try {
+ const parsed = schema.safeParse(JSON.parse(obj.bytes.toString("utf8")));
+ return parsed.success ? parsed.data : null;
+ } catch {
+ return null;
+ }
+}
+
+function messageOf(e: unknown): string {
+ return e instanceof Error ? e.message : String(e);
+}
diff --git a/site/src/lib/analytics/posthog.ts b/site/src/lib/analytics/posthog.ts
new file mode 100644
index 00000000..f7d96980
--- /dev/null
+++ b/site/src/lib/analytics/posthog.ts
@@ -0,0 +1,52 @@
+// PostHog read access for the analytics pipeline. Browser ingestion uses the
+// public project key against us.i.posthog.com; these reads go through the
+// query API on the main host, addressed by POSTHOG_PROJECT_ID and authorized
+// by POSTHOG_PERSONAL_API_KEY — both env, and both required before any
+// PostHog extraction runs.
+import { addUtcDays } from "@/lib/analytics/schema";
+
+const POSTHOG_HOST = "https://us.posthog.com";
+
+// The numeric project id from PostHog → Settings → Project.
+function posthogProjectId(): number {
+ const id = Number(process.env.POSTHOG_PROJECT_ID);
+ return Number.isInteger(id) && id > 0 ? id : 0;
+}
+
+export function isPosthogQueryConfigured(): boolean {
+ return posthogProjectId() > 0 && Boolean(process.env.POSTHOG_PERSONAL_API_KEY);
+}
+
+/** distinct_ids that fired app_loaded during the UTC day. app_loaded fires
+ * only after posthog.identify(user.id), so these are app user ids plus
+ * whatever anonymous ids a shared browser aliased in. */
+export async function fetchActiveDistinctIds(day: string): Promise<string[]> {
+ const apiKey = process.env.POSTHOG_PERSONAL_API_KEY;
+ const projectId = posthogProjectId();
+ if (!apiKey || projectId <= 0) {
+ throw new Error("PostHog query access is not configured.");
+ }
+ const query = [
+ "SELECT DISTINCT distinct_id FROM events",
+ "WHERE event = 'app_loaded'",
+ `AND timestamp >= toDateTime('${day} 00:00:00', 'UTC')`,
+ `AND timestamp < toDateTime('${addUtcDays(day, 1)} 00:00:00', 'UTC')`,
+ "LIMIT 10000",
+ ].join(" ");
+ const res = await fetch(`${POSTHOG_HOST}/api/projects/${projectId}/query`, {
+ body: JSON.stringify({ query: { kind: "HogQLQuery", query } }),
+ headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+ method: "POST",
+ });
+ if (!res.ok) {
+ const detail = (await res.text().catch(() => "")).slice(0, 300);
+ throw new Error(`PostHog query failed (${res.status}): ${detail}`);
+ }
+ const data = (await res.json()) as { results?: unknown[][] };
+ const ids = new Set<string>();
+ for (const row of data.results ?? []) {
+ const id = row?.[0];
+ if (typeof id === "string" && id) ids.add(id);
+ }
+ return [...ids].sort();
+}
diff --git a/site/src/lib/analytics/schema.ts b/site/src/lib/analytics/schema.ts
new file mode 100644
index 00000000..cf587f6b
--- /dev/null
+++ b/site/src/lib/analytics/schema.ts
@@ -0,0 +1,105 @@
+// The analytics data contract shared by the nightly pipeline, the superuser
+// API, and the dashboard UI. The UI reads rollup.json only — never the
+// database — so this file is the whole interface between them.
+import { z } from "zod";
+
+export const ANALYTICS_ROLLUP_VERSION = 1;
+
+// Bit order for the per-day activity masks. Append-only: a new source takes
+// the next bit so already-written rollups keep decoding.
+export const ANALYTICS_DB_SOURCES = [
+ "inference",
+ "renders",
+ "copies",
+ "libraryAssets",
+ "templates",
+ "creditLedger",
+] as const;
+export const ANALYTICS_SOURCES = [...ANALYTICS_DB_SOURCES, "posthog"] as const;
+export type AnalyticsDbSource = (typeof ANALYTICS_DB_SOURCES)[number];
+export type AnalyticsSource = (typeof ANALYTICS_SOURCES)[number];
+
+export const analyticsDaySchema = z.string().regex(/^\d{4}-\d{2}-\d{2}$/);
+
+/** UTC calendar day of a moment, as YYYY-MM-DD. */
+export function utcDayOf(date: Date): string {
+ return date.toISOString().slice(0, 10);
+}
+
+/** A day shifted by whole UTC days. Also normalizes: a string that is not a
+ * real calendar day round-trips to a different string at delta 0. */
+export function addUtcDays(day: string, delta: number): string {
+ const [y, m, d] = day.split("-").map(Number);
+ return utcDayOf(new Date(Date.UTC(y, (m ?? 1) - 1, (d ?? 1) + delta)));
+}
+
+const userIdList = z.array(z.string());
+
+// Per-day extract of DB activity: which users each event table saw that day.
+export const analyticsDbDayFileSchema = z.object({
+ version: z.literal(1),
+ day: analyticsDaySchema,
+ generatedAt: z.string(),
+ active: z.object({
+ inference: userIdList.optional(),
+ renders: userIdList.optional(),
+ copies: userIdList.optional(),
+ libraryAssets: userIdList.optional(),
+ templates: userIdList.optional(),
+ creditLedger: userIdList.optional(),
+ }),
+});
+export type AnalyticsDbDayFile = z.infer<typeof analyticsDbDayFileSchema>;
+
+export const analyticsPosthogDayFileSchema = z.object({
+ version: z.literal(1),
+ day: analyticsDaySchema,
+ generatedAt: z.string(),
+ // Raw distinct_ids. Anonymous pre-sign-in ids ride along and are dropped at
+ // consolidation, so this list's length is not a user count.
+ activeDistinctIds: userIdList,
+});
+export type AnalyticsPosthogDayFile = z.infer<typeof analyticsPosthogDayFileSchema>;
+
+export const analyticsSnapshotFileSchema = z.object({
+ version: z.literal(1),
+ generatedAt: z.string(),
+ users: z.array(
+ z.object({
+ id: z.string(),
+ email: z.string(),
+ name: z.string(),
+ createdAt: z.string(),
+ }),
+ ),
+ balances: z.array(z.object({ userId: z.string(), balanceMicros: z.string() })),
+});
+export type AnalyticsSnapshotFile = z.infer<typeof analyticsSnapshotFileSchema>;
+
+// The consolidated window the UI renders. Users come from the latest snapshot,
+// so an account deleted since then carries no history here.
+export const analyticsRollupSchema = z.object({
+ version: z.literal(ANALYTICS_ROLLUP_VERSION),
+ generatedAt: z.string(),
+ // Oldest → newest; every user's activity array aligns with this.
+ days: z.array(analyticsDaySchema),
+ // Bit order of the activity masks (copy of ANALYTICS_SOURCES at write time).
+ sources: z.array(z.string()),
+ // Day files that were absent or unreadable at consolidation; the UI can gray
+ // those columns out. Entries name the file kind: "db" or "posthog".
+ missing: z.array(z.object({ day: analyticsDaySchema, sources: z.array(z.string()) })),
+ users: z.array(
+ z.object({
+ id: z.string(),
+ email: z.string(),
+ name: z.string(),
+ registeredAt: z.string(),
+ // BigInt micros as a decimal string; "0" for users without an account row.
+ balanceMicros: z.string(),
+ // One mask per entry of days; a dot is mask !== 0.
+ activity: z.array(z.number()),
+ }),
+ ),
+});
+export type AnalyticsRollup = z.infer<typeof analyticsRollupSchema>;
+export type AnalyticsRollupUser = AnalyticsRollup["users"][number];
diff --git a/site/src/lib/jobs/analytics-daily.ts b/site/src/lib/jobs/analytics-daily.ts
new file mode 100644
index 00000000..036e96d9
--- /dev/null
+++ b/site/src/lib/jobs/analytics-daily.ts
@@ -0,0 +1,32 @@
+import { z } from "zod";
+
+import { R2NotConfiguredError } from "@/cut/server/cloud/r2";
+import { InvalidDayError, runAnalyticsDaily } from "@/lib/analytics/pipeline";
+import { defineJob, JobFailure } from "@/lib/jobs/registry";
+
+// The nightly analytics run, also retriggerable by hand. An empty payload is
+// the regular run: yesterday plus any unfinished days in the window. A
+// specific day re-runs just that day; force re-extracts even final files.
+export const analyticsDailyJob = defineJob(
+ z
+ .object({
+ day: z
+ .string()
+ .regex(/^\d{4}-\d{2}-\d{2}$/)
+ .optional(),
+ force: z.boolean().optional(),
+ })
+ .strict(),
+ async (payload) => {
+ try {
+ return await runAnalyticsDaily(payload);
+ } catch (e) {
+ // Both are permanent: retrying an unconfigured store or an impossible
+ // day can never succeed.
+ if (e instanceof R2NotConfiguredError || e instanceof InvalidDayError) {
+ throw new JobFailure(e.message);
+ }
+ throw e;
+ }
+ },
+);
diff --git a/site/src/lib/jobs/registry.ts b/site/src/lib/jobs/registry.ts
index 27676f65..6200fc71 100644
--- a/site/src/lib/jobs/registry.ts
+++ b/site/src/lib/jobs/registry.ts
@@ -1,6 +1,7 @@
import type { z } from "zod";
import type { Prisma } from "@/generated/prisma/client";
+import { analyticsDailyJob } from "@/lib/jobs/analytics-daily";
import { deleteUserJob } from "@/lib/jobs/delete-user";
// Thrown by an executor when the job can never succeed — the message lands on
@@ -23,5 +24,6 @@ export function defineJob<S extends z.ZodType<unknown>>(
}
export const jobKinds: Record<string, JobKind> = {
+ "analytics-daily": analyticsDailyJob,
"delete-user": deleteUserJob,
};
diff --git a/site/src/queries/analytics.ts b/site/src/queries/analytics.ts
new file mode 100644
index 00000000..11ac3ccf
--- /dev/null
+++ b/site/src/queries/analytics.ts
@@ -0,0 +1,17 @@
+"use client";
+
+import { useQuery } from "@tanstack/react-query";
+
+import type { AnalyticsRollup } from "@/lib/analytics/schema";
+import { apiFetch } from "@/queries/apiClient";
+
+export const analyticsRollupQueryKey = ["analytics", "rollup"] as const;
+
+// Super-user only: the nightly job's consolidated rollup. Stale until the
+// next run by design.
+export function useAnalyticsRollup() {
+ return useQuery({
+ queryFn: () => apiFetch<AnalyticsRollup>("/api/analytics/rollup"),
+ queryKey: analyticsRollupQueryKey,
+ });
+}
diff --git a/site/vercel.json b/site/vercel.json
index f4d85865..a6cd7b2f 100644
--- a/site/vercel.json
+++ b/site/vercel.json
@@ -1,3 +1,6 @@
{
- "crons": [{ "path": "/api/cut-cloud/gc", "schedule": "0 4 * * *" }]
+ "crons": [
+ { "path": "/api/cut-cloud/gc", "schedule": "0 4 * * *" },
+ { "path": "/api/analytics/run", "schedule": "0 9 * * *" }
+ ]
}