The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Back fill every program when no program is named
Rafi507285c96dcd
diff --git a/apps/web/scripts/partners/backfill-partner-search-args.ts b/apps/web/scripts/partners/backfill-partner-search-args.tsnew file mode 100644index 0000000000..c57ff8f091--- /dev/null+++ b/apps/web/scripts/partners/backfill-partner-search-args.ts@@ -0,0 +1,84 @@+import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";++export const DEFAULT_BATCH_SIZE = 500;+export const MAX_BATCH_SIZE = 1_000;++export interface BackfillArguments {+ programId?: string;+ all: boolean;+ batchSize: number;+ after?: string;+ afterProgram?: string;+}++/**+ * Kept out of the script itself so the guardrails can be tested without the+ * module executing a backfill on import.+ */+export function parseBackfillArguments(args: string[]): BackfillArguments {+ let programId: string | undefined;+ let all = false;+ let batchSize = DEFAULT_BATCH_SIZE;+ let after: string | undefined;+ let afterProgram: string | undefined;++ for (const arg of args) {+ if (arg.startsWith("--programId=")) {+ programId = arg.slice("--programId=".length);+ } else if (arg === "--all") {+ all = true;+ } else if (arg.startsWith("--batchSize=")) {+ batchSize = parsePositiveInteger(+ arg.slice("--batchSize=".length),+ "--batchSize",+ );+ } else if (arg.startsWith("--after=")) {+ after = arg.slice("--after=".length);+ } else if (arg.startsWith("--afterProgram=")) {+ afterProgram = arg.slice("--afterProgram=".length);+ } else {+ throw new Error(`Unknown argument: ${arg}`);+ }+ }++ // Indexing every program is ~1.6M writes against whichever namespace the+ // configured key points at, so it has to be asked for rather than defaulted+ // into by running the script bare.+ if (!programId && !all) {+ throw new Error(+ "Pass --programId=<id> for one program, or --all for every program.",+ );+ }++ if (programId && all) {+ throw new Error("--programId and --all are mutually exclusive.");+ }++ if (programId === "") {+ throw new Error("--programId cannot be empty.");+ }++ if (batchSize > MAX_BATCH_SIZE) {+ throw new Error(`--batchSize cannot exceed ${MAX_BATCH_SIZE}.`);+ }++ if (after === "") {+ throw new Error("--after cannot be empty.");+ }++ if (afterProgram === "") {+ throw new Error("--afterProgram cannot be empty.");+ }++ if (afterProgram && !all) {+ throw new Error("--afterProgram only applies to --all runs.");+ }++ // Without the program it belongs to, an enrollment cursor says nothing about+ // where an --all run stopped.+ if (all && after && !afterProgram) {+ throw new Error("--after requires --afterProgram on an --all run.");+ }++ return { programId, all, batchSize, after, afterProgram };+}diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.tsindex 42bf99ea25..e880b1840a 100644--- a/apps/web/scripts/partners/backfill-partner-search.ts+++ b/apps/web/scripts/partners/backfill-partner-search.ts@@ -1,18 +1,26 @@/**- * Indexes a program's partner enrollments into the configured search provider.+ * Indexes partner enrollments into the configured search provider.*- * Pages by enrollment ID rather than offset, so per-batch cost stays flat across- * a 100K-partner program. A failed run prints the command to resume from the- * last indexed document, so a partial backfill is never redone from zero.+ * Runs one program or every program. Pages by enrollment ID rather than offset,+ * so per-batch cost stays flat across a 100K-partner program, and iterates+ * programs in ID order so a resumed run picks up exactly where it stopped.+ *+ * Programs are the outer loop rather than one cursor over the whole table. A+ * single global cursor would be simpler, but it leaves every program partially+ * indexed until the entire run finishes, so no program is ever in a state worth+ * trusting. Per-program means each finished program is genuinely done.** The backfill only upserts. Documents whose enrollment was deleted are removed- * by rebuilding the index wholesale, not incrementally — delete it and backfill+ * by rebuilding the index wholesale, not incrementally: delete it and backfill* again.** cd apps/web* pnpm run script partners/backfill-partner-search --programId=prog_123* [--batchSize=500] [--after=pge_123]*+ * pnpm run script partners/backfill-partner-search --all+ * [--batchSize=500] [--afterProgram=prog_123 --after=pge_123]+ ** Requires TURBOPUFFER_API_KEY to be configured.*/@@ -22,129 +30,235 @@ import {type PartnerSearchBackfillProgress,} from "@/lib/api/partners/search";import { prisma } from "@/lib/prisma";-import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";+import { parseBackfillArguments } from "@/scripts/partners/backfill-partner-search-args";import "dotenv-flow/config";-const DEFAULT_BATCH_SIZE = 500;-const MAX_BATCH_SIZE = 1_000;+const PROGRAM_PAGE_SIZE = 1_000;-interface BackfillArguments {- programId: string;- batchSize: number;- after?: string;-}+/**+ * Every program ID in ID order, paged so a large account does not load them all+ * in one query. Starts at `afterProgram` inclusive, because that program is the+ * one a resumed run left partway through.+ */+async function* iterateProgramIds(afterProgram?: string) {+ let cursor = afterProgram;+ let inclusive = Boolean(afterProgram);-function parseArguments(args: string[]): BackfillArguments {- let programId: string | undefined;- let batchSize = DEFAULT_BATCH_SIZE;- let after: string | undefined;-- for (const arg of args) {- if (arg.startsWith("--programId=")) {- programId = arg.slice("--programId=".length);- } else if (arg.startsWith("--batchSize=")) {- batchSize = parsePositiveInteger(- arg.slice("--batchSize=".length),- "--batchSize",- );- } else if (arg.startsWith("--after=")) {- after = arg.slice("--after=".length);- } else {- throw new Error(`Unknown argument: ${arg}`);+ while (true) {+ const programs = await prisma.program.findMany({+ where: cursor ? { id: inclusive ? { gte: cursor } : { gt: cursor } } : {},+ select: { id: true },+ orderBy: { id: "asc" },+ take: PROGRAM_PAGE_SIZE,+ });++ if (programs.length === 0) {+ return;}- }- if (!programId) {- throw new Error("--programId is required.");- }+ for (const { id } of programs) {+ yield id;+ }- if (batchSize > MAX_BATCH_SIZE) {- throw new Error(`--batchSize cannot exceed ${MAX_BATCH_SIZE}.`);- }+ cursor = programs[programs.length - 1].id;+ inclusive = false;- if (after === "") {- throw new Error("--after cannot be empty.");+ if (programs.length < PROGRAM_PAGE_SIZE) {+ return;+ }}-- return { programId, batchSize, after };}-let resumeAfter: string | undefined;+// Tracks where to resume from, so a failure can print a command that skips+// everything already indexed rather than starting the run over.+const resumeState: { programId?: string; after?: string } = {};-// `totalDocuments` is a snapshot from before the run, so enrollments written-// while the backfill is paging can push the final chunk past it; the percentage-// is clamped rather than reporting >100%.-function createProgressReporter(totalDocuments: number, batchSize: number) {- const totalChunks = Math.max(1, Math.ceil(totalDocuments / batchSize));+function createProgressReporter({+ programId,+ totalDocuments,+ batchSize,+ label,+}: {+ programId: string;+ totalDocuments: number | null;+ batchSize: number;+ label: string;+}) {+ const totalChunks =+ totalDocuments === null+ ? null+ : Math.max(1, Math.ceil(totalDocuments / batchSize));const startTime = Date.now();let chunk = 0;- return ({- batchSize: indexedInChunk,- processed,- lastDocumentId,- }: PartnerSearchBackfillProgress) => {- resumeAfter = lastDocumentId;+ return ({ processed, lastDocumentId }: PartnerSearchBackfillProgress) => {+ resumeState.programId = programId;+ resumeState.after = lastDocumentId;chunk += 1;- const progressPct = Math.min(- 100,- totalDocuments > 0 ? (processed / totalDocuments) * 100 : 100,- ).toFixed(0);const elapsedSec = ((Date.now() - startTime) / 1000).toFixed(1);++ // `totalDocuments` is a snapshot from before the run, so enrollments+ // written while it pages can push the final chunk past it; the percentage+ // is clamped rather than reporting >100%.+ const progress =+ totalDocuments === null+ ? `chunk ${chunk}`+ : `chunk ${chunk}/${totalChunks} (${Math.min(+ 100,+ totalDocuments > 0 ? (processed / totalDocuments) * 100 : 100,+ ).toFixed(0)}%)`;+console.log(- `[Chunk ${chunk}/${totalChunks}] (${progressPct}%) Processed ${processed.toLocaleString()}/${totalDocuments.toLocaleString()} partners (${indexedInChunk.toLocaleString()} indexed, through ${lastDocumentId})... (${elapsedSec}s elapsed)`,+ `${label} ${progress}: ${processed.toLocaleString()} indexed, through ${lastDocumentId} (${elapsedSec}s)`,);};}+async function backfillProgram({+ programId,+ batchSize,+ after,+ label,+ countDocuments,+}: {+ programId: string;+ batchSize: number;+ after?: string;+ label: string;+ countDocuments: boolean;+}) {+ // Skipped on --all runs: a count per program doubles the query count across+ // thousands of programs to buy a percentage nobody is watching that closely.+ const totalDocuments = countDocuments+ ? await prisma.programEnrollment.count({+ where: {+ programId,+ ...(after && { id: { gt: after } }),+ },+ })+ : null;++ if (totalDocuments !== null) {+ console.log(`${totalDocuments.toLocaleString()} enrollments to index\n`);+ }++ const { processed } = await backfillPartnerSearch({+ programId,+ batchSize,+ after,+ onProgress: createProgressReporter({+ programId,+ totalDocuments,+ batchSize,+ label,+ }),+ });++ return processed;+}+async function main() {- const { programId, batchSize, after } = parseArguments(process.argv.slice(2));- resumeAfter = after;+ const { programId, all, batchSize, after, afterProgram } =+ parseBackfillArguments(process.argv.slice(2));+if (!getPartnerSearchProvider()) {throw new Error("TURBOPUFFER_API_KEY is not configured.");}- // Same where-clause the backfill pages with, so chunk totals line up.- const totalDocuments = await prisma.programEnrollment.count({- where: {+ const startTime = Date.now();++ if (programId) {+ resumeState.programId = programId;+ resumeState.after = after;++ console.log(`Starting partner search backfill for program ${programId}`);+ console.log(+ `Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,+ );++ const processed = await backfillProgram({programId,- ...(after && { id: { gt: after } }),- },- });+ batchSize,+ after,+ label: ` [${programId}]`,+ countDocuments: true,+ });++ console.log(+ `\nPartner search backfill complete: ${processed.toLocaleString()} documents indexed.`,+ );++ return;+ }- console.log(`Starting partner search backfill for program ${programId}`);+ console.log("Starting partner search backfill for every program");console.log(- `Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,+ `Batch size: ${batchSize.toLocaleString()}${+ afterProgram+ ? `, resuming at program ${afterProgram}${after ? ` after ${after}` : ""}`+ : ""+ }\n`,);- console.log(`${totalDocuments.toLocaleString()} enrollments to index\n`);- const result = await backfillPartnerSearch({- programId,- batchSize,- after,- onProgress: createProgressReporter(totalDocuments, batchSize),- });+ let programCount = 0;+ let totalProcessed = 0;++ for await (const currentProgramId of iterateProgramIds(afterProgram)) {+ programCount += 1;++ // The enrollment cursor belongs to the program the run stopped inside, so+ // it applies to that one only. Every program after it starts from scratch.+ const programAfter = currentProgramId === afterProgram ? after : undefined;++ resumeState.programId = currentProgramId;+ resumeState.after = programAfter;++ const processed = await backfillProgram({+ programId: currentProgramId,+ batchSize,+ after: programAfter,+ label: ` [${programCount}] ${currentProgramId}`,+ countDocuments: false,+ });++ totalProcessed += processed;+ console.log(+ `[${programCount}] ${currentProgramId}: ${processed.toLocaleString()} indexed (${totalProcessed.toLocaleString()} total)`,+ );+ }++ const elapsedMin = ((Date.now() - startTime) / 60_000).toFixed(1);console.log(- `Partner search backfill complete: ${result.processed.toLocaleString()} documents indexed.`,+ `\nPartner search backfill complete: ${totalProcessed.toLocaleString()} documents across ${programCount.toLocaleString()} programs in ${elapsedMin} minutes.`,);- if (result.lastDocumentId) {- console.log(`Last document: ${result.lastDocumentId}`);- }}main().catch((error) => {console.error("Partner search backfill failed:", error);- if (resumeAfter) {+ if (resumeState.programId) {const args = process.argv.slice(2)- .filter((arg) => !arg.startsWith("--after="))- .join(" ");+ .filter(+ (arg) =>+ !arg.startsWith("--after=") && !arg.startsWith("--afterProgram="),+ );++ // An --all run needs both halves of the cursor. A single-program run+ // already names its program, so it only needs the enrollment.+ const resumeArgs = args.includes("--all")+ ? [+ ...args,+ `--afterProgram=${resumeState.programId}`,+ ...(resumeState.after ? [`--after=${resumeState.after}`] : []),+ ]+ : [...args, ...(resumeState.after ? [`--after=${resumeState.after}`] : [])];+console.error(- `Documents up to ${resumeAfter} are indexed. Resume with:\n pnpm run script partners/backfill-partner-search ${args} --after=${resumeAfter}`,+ `Resume with:\n pnpm run script partners/backfill-partner-search ${resumeArgs.join(" ")}`,);}diff --git a/apps/web/tests/partners/backfill-partner-search-args.test.ts b/apps/web/tests/partners/backfill-partner-search-args.test.tsnew file mode 100644index 0000000000..a42ffc3a96--- /dev/null+++ b/apps/web/tests/partners/backfill-partner-search-args.test.ts@@ -0,0 +1,103 @@+import {+ DEFAULT_BATCH_SIZE,+ MAX_BATCH_SIZE,+ parseBackfillArguments,+} from "@/scripts/partners/backfill-partner-search-args";+import { describe, expect, it } from "vitest";++describe("parseBackfillArguments", () => {+ it("refuses to run bare, so a full backfill cannot start by accident", () => {+ expect(() => parseBackfillArguments([])).toThrow(+ "Pass --programId=<id> for one program, or --all for every program.",+ );+ });++ it("takes a single program", () => {+ expect(parseBackfillArguments(["--programId=prog_1"])).toEqual({+ programId: "prog_1",+ all: false,+ batchSize: DEFAULT_BATCH_SIZE,+ after: undefined,+ afterProgram: undefined,+ });+ });++ it("takes every program behind an explicit flag", () => {+ expect(parseBackfillArguments(["--all"])).toEqual({+ programId: undefined,+ all: true,+ batchSize: DEFAULT_BATCH_SIZE,+ after: undefined,+ afterProgram: undefined,+ });+ });++ it("rejects one program and every program together", () => {+ expect(() =>+ parseBackfillArguments(["--programId=prog_1", "--all"]),+ ).toThrow("--programId and --all are mutually exclusive.");+ });++ it("resumes a single-program run from an enrollment", () => {+ expect(+ parseBackfillArguments(["--programId=prog_1", "--after=pge_9"]),+ ).toMatchObject({+ programId: "prog_1",+ after: "pge_9",+ });+ });++ it("resumes an --all run from both halves of the cursor", () => {+ expect(+ parseBackfillArguments([+ "--all",+ "--afterProgram=prog_5",+ "--after=pge_9",+ ]),+ ).toMatchObject({+ all: true,+ afterProgram: "prog_5",+ after: "pge_9",+ });+ });++ it("rejects an enrollment cursor with no program on an --all run", () => {+ expect(() => parseBackfillArguments(["--all", "--after=pge_9"])).toThrow(+ "--after requires --afterProgram on an --all run.",+ );+ });++ it("rejects a program cursor on a single-program run", () => {+ expect(() =>+ parseBackfillArguments(["--programId=prog_1", "--afterProgram=prog_5"]),+ ).toThrow("--afterProgram only applies to --all runs.");+ });++ it("rejects empty cursors, which would silently mean no cursor", () => {+ expect(() => parseBackfillArguments(["--all", "--afterProgram="])).toThrow(+ "--afterProgram cannot be empty.",+ );+ expect(() =>+ parseBackfillArguments(["--programId=prog_1", "--after="]),+ ).toThrow("--after cannot be empty.");+ expect(() => parseBackfillArguments(["--programId="])).toThrow(+ "Pass --programId=<id> for one program, or --all for every program.",+ );+ });++ it("caps the batch size", () => {+ expect(+ parseBackfillArguments(["--all", `--batchSize=${MAX_BATCH_SIZE}`]),+ ).toMatchObject({ batchSize: MAX_BATCH_SIZE });++ expect(() =>+ parseBackfillArguments(["--all", `--batchSize=${MAX_BATCH_SIZE + 1}`]),+ ).toThrow(`--batchSize cannot exceed ${MAX_BATCH_SIZE}.`);+ });++ it("rejects an unknown argument rather than ignoring it", () => {+ expect(() => parseBackfillArguments(["--all", "--dryRun"])).toThrow(+ "Unknown argument: --dryRun",+ );+ });+});