Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Back fill every program when no program is named

Rafi507285c96dcd
diff --git a/apps/web/scripts/partners/backfill-partner-search-args.ts b/apps/web/scripts/partners/backfill-partner-search-args.ts
new file mode 100644
index 0000000000..c57ff8f091
--- /dev/null
+++ b/apps/web/scripts/partners/backfill-partner-search-args.ts
@@ -0,0 +1,84 @@
+import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";
+
+export const DEFAULT_BATCH_SIZE = 500;
+export const MAX_BATCH_SIZE = 1_000;
+
+export interface BackfillArguments {
+ programId?: string;
+ all: boolean;
+ batchSize: number;
+ after?: string;
+ afterProgram?: string;
+}
+
+/**
+ * Kept out of the script itself so the guardrails can be tested without the
+ * module executing a backfill on import.
+ */
+export function parseBackfillArguments(args: string[]): BackfillArguments {
+ let programId: string | undefined;
+ let all = false;
+ let batchSize = DEFAULT_BATCH_SIZE;
+ let after: string | undefined;
+ let afterProgram: string | undefined;
+
+ for (const arg of args) {
+ if (arg.startsWith("--programId=")) {
+ programId = arg.slice("--programId=".length);
+ } else if (arg === "--all") {
+ all = true;
+ } else if (arg.startsWith("--batchSize=")) {
+ batchSize = parsePositiveInteger(
+ arg.slice("--batchSize=".length),
+ "--batchSize",
+ );
+ } else if (arg.startsWith("--after=")) {
+ after = arg.slice("--after=".length);
+ } else if (arg.startsWith("--afterProgram=")) {
+ afterProgram = arg.slice("--afterProgram=".length);
+ } else {
+ throw new Error(`Unknown argument: ${arg}`);
+ }
+ }
+
+ // Indexing every program is ~1.6M writes against whichever namespace the
+ // configured key points at, so it has to be asked for rather than defaulted
+ // into by running the script bare.
+ if (!programId && !all) {
+ throw new Error(
+ "Pass --programId=<id> for one program, or --all for every program.",
+ );
+ }
+
+ if (programId && all) {
+ throw new Error("--programId and --all are mutually exclusive.");
+ }
+
+ if (programId === "") {
+ throw new Error("--programId cannot be empty.");
+ }
+
+ if (batchSize > MAX_BATCH_SIZE) {
+ throw new Error(`--batchSize cannot exceed ${MAX_BATCH_SIZE}.`);
+ }
+
+ if (after === "") {
+ throw new Error("--after cannot be empty.");
+ }
+
+ if (afterProgram === "") {
+ throw new Error("--afterProgram cannot be empty.");
+ }
+
+ if (afterProgram && !all) {
+ throw new Error("--afterProgram only applies to --all runs.");
+ }
+
+ // Without the program it belongs to, an enrollment cursor says nothing about
+ // where an --all run stopped.
+ if (all && after && !afterProgram) {
+ throw new Error("--after requires --afterProgram on an --all run.");
+ }
+
+ return { programId, all, batchSize, after, afterProgram };
+}
diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.ts
index 42bf99ea25..e880b1840a 100644
--- a/apps/web/scripts/partners/backfill-partner-search.ts
+++ b/apps/web/scripts/partners/backfill-partner-search.ts
@@ -1,18 +1,26 @@
/**
- * Indexes a program's partner enrollments into the configured search provider.
+ * Indexes partner enrollments into the configured search provider.
*
- * Pages by enrollment ID rather than offset, so per-batch cost stays flat across
- * a 100K-partner program. A failed run prints the command to resume from the
- * last indexed document, so a partial backfill is never redone from zero.
+ * Runs one program or every program. Pages by enrollment ID rather than offset,
+ * so per-batch cost stays flat across a 100K-partner program, and iterates
+ * programs in ID order so a resumed run picks up exactly where it stopped.
+ *
+ * Programs are the outer loop rather than one cursor over the whole table. A
+ * single global cursor would be simpler, but it leaves every program partially
+ * indexed until the entire run finishes, so no program is ever in a state worth
+ * trusting. Per-program means each finished program is genuinely done.
*
* The backfill only upserts. Documents whose enrollment was deleted are removed
- * by rebuilding the index wholesale, not incrementally — delete it and backfill
+ * by rebuilding the index wholesale, not incrementally: delete it and backfill
* again.
*
* cd apps/web
* pnpm run script partners/backfill-partner-search --programId=prog_123
* [--batchSize=500] [--after=pge_123]
*
+ * pnpm run script partners/backfill-partner-search --all
+ * [--batchSize=500] [--afterProgram=prog_123 --after=pge_123]
+ *
* Requires TURBOPUFFER_API_KEY to be configured.
*/
@@ -22,129 +30,235 @@ import {
type PartnerSearchBackfillProgress,
} from "@/lib/api/partners/search";
import { prisma } from "@/lib/prisma";
-import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";
+import { parseBackfillArguments } from "@/scripts/partners/backfill-partner-search-args";
import "dotenv-flow/config";
-const DEFAULT_BATCH_SIZE = 500;
-const MAX_BATCH_SIZE = 1_000;
+const PROGRAM_PAGE_SIZE = 1_000;
-interface BackfillArguments {
- programId: string;
- batchSize: number;
- after?: string;
-}
+/**
+ * Every program ID in ID order, paged so a large account does not load them all
+ * in one query. Starts at `afterProgram` inclusive, because that program is the
+ * one a resumed run left partway through.
+ */
+async function* iterateProgramIds(afterProgram?: string) {
+ let cursor = afterProgram;
+ let inclusive = Boolean(afterProgram);
-function parseArguments(args: string[]): BackfillArguments {
- let programId: string | undefined;
- let batchSize = DEFAULT_BATCH_SIZE;
- let after: string | undefined;
-
- for (const arg of args) {
- if (arg.startsWith("--programId=")) {
- programId = arg.slice("--programId=".length);
- } else if (arg.startsWith("--batchSize=")) {
- batchSize = parsePositiveInteger(
- arg.slice("--batchSize=".length),
- "--batchSize",
- );
- } else if (arg.startsWith("--after=")) {
- after = arg.slice("--after=".length);
- } else {
- throw new Error(`Unknown argument: ${arg}`);
+ while (true) {
+ const programs = await prisma.program.findMany({
+ where: cursor ? { id: inclusive ? { gte: cursor } : { gt: cursor } } : {},
+ select: { id: true },
+ orderBy: { id: "asc" },
+ take: PROGRAM_PAGE_SIZE,
+ });
+
+ if (programs.length === 0) {
+ return;
}
- }
- if (!programId) {
- throw new Error("--programId is required.");
- }
+ for (const { id } of programs) {
+ yield id;
+ }
- if (batchSize > MAX_BATCH_SIZE) {
- throw new Error(`--batchSize cannot exceed ${MAX_BATCH_SIZE}.`);
- }
+ cursor = programs[programs.length - 1].id;
+ inclusive = false;
- if (after === "") {
- throw new Error("--after cannot be empty.");
+ if (programs.length < PROGRAM_PAGE_SIZE) {
+ return;
+ }
}
-
- return { programId, batchSize, after };
}
-let resumeAfter: string | undefined;
+// Tracks where to resume from, so a failure can print a command that skips
+// everything already indexed rather than starting the run over.
+const resumeState: { programId?: string; after?: string } = {};
-// `totalDocuments` is a snapshot from before the run, so enrollments written
-// while the backfill is paging can push the final chunk past it; the percentage
-// is clamped rather than reporting >100%.
-function createProgressReporter(totalDocuments: number, batchSize: number) {
- const totalChunks = Math.max(1, Math.ceil(totalDocuments / batchSize));
+function createProgressReporter({
+ programId,
+ totalDocuments,
+ batchSize,
+ label,
+}: {
+ programId: string;
+ totalDocuments: number | null;
+ batchSize: number;
+ label: string;
+}) {
+ const totalChunks =
+ totalDocuments === null
+ ? null
+ : Math.max(1, Math.ceil(totalDocuments / batchSize));
const startTime = Date.now();
let chunk = 0;
- return ({
- batchSize: indexedInChunk,
- processed,
- lastDocumentId,
- }: PartnerSearchBackfillProgress) => {
- resumeAfter = lastDocumentId;
+ return ({ processed, lastDocumentId }: PartnerSearchBackfillProgress) => {
+ resumeState.programId = programId;
+ resumeState.after = lastDocumentId;
chunk += 1;
- const progressPct = Math.min(
- 100,
- totalDocuments > 0 ? (processed / totalDocuments) * 100 : 100,
- ).toFixed(0);
const elapsedSec = ((Date.now() - startTime) / 1000).toFixed(1);
+
+ // `totalDocuments` is a snapshot from before the run, so enrollments
+ // written while it pages can push the final chunk past it; the percentage
+ // is clamped rather than reporting >100%.
+ const progress =
+ totalDocuments === null
+ ? `chunk ${chunk}`
+ : `chunk ${chunk}/${totalChunks} (${Math.min(
+ 100,
+ totalDocuments > 0 ? (processed / totalDocuments) * 100 : 100,
+ ).toFixed(0)}%)`;
+
console.log(
- `[Chunk ${chunk}/${totalChunks}] (${progressPct}%) Processed ${processed.toLocaleString()}/${totalDocuments.toLocaleString()} partners (${indexedInChunk.toLocaleString()} indexed, through ${lastDocumentId})... (${elapsedSec}s elapsed)`,
+ `${label} ${progress}: ${processed.toLocaleString()} indexed, through ${lastDocumentId} (${elapsedSec}s)`,
);
};
}
+async function backfillProgram({
+ programId,
+ batchSize,
+ after,
+ label,
+ countDocuments,
+}: {
+ programId: string;
+ batchSize: number;
+ after?: string;
+ label: string;
+ countDocuments: boolean;
+}) {
+ // Skipped on --all runs: a count per program doubles the query count across
+ // thousands of programs to buy a percentage nobody is watching that closely.
+ const totalDocuments = countDocuments
+ ? await prisma.programEnrollment.count({
+ where: {
+ programId,
+ ...(after && { id: { gt: after } }),
+ },
+ })
+ : null;
+
+ if (totalDocuments !== null) {
+ console.log(`${totalDocuments.toLocaleString()} enrollments to index\n`);
+ }
+
+ const { processed } = await backfillPartnerSearch({
+ programId,
+ batchSize,
+ after,
+ onProgress: createProgressReporter({
+ programId,
+ totalDocuments,
+ batchSize,
+ label,
+ }),
+ });
+
+ return processed;
+}
+
async function main() {
- const { programId, batchSize, after } = parseArguments(process.argv.slice(2));
- resumeAfter = after;
+ const { programId, all, batchSize, after, afterProgram } =
+ parseBackfillArguments(process.argv.slice(2));
+
if (!getPartnerSearchProvider()) {
throw new Error("TURBOPUFFER_API_KEY is not configured.");
}
- // Same where-clause the backfill pages with, so chunk totals line up.
- const totalDocuments = await prisma.programEnrollment.count({
- where: {
+ const startTime = Date.now();
+
+ if (programId) {
+ resumeState.programId = programId;
+ resumeState.after = after;
+
+ console.log(`Starting partner search backfill for program ${programId}`);
+ console.log(
+ `Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,
+ );
+
+ const processed = await backfillProgram({
programId,
- ...(after && { id: { gt: after } }),
- },
- });
+ batchSize,
+ after,
+ label: ` [${programId}]`,
+ countDocuments: true,
+ });
+
+ console.log(
+ `\nPartner search backfill complete: ${processed.toLocaleString()} documents indexed.`,
+ );
+
+ return;
+ }
- console.log(`Starting partner search backfill for program ${programId}`);
+ console.log("Starting partner search backfill for every program");
console.log(
- `Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,
+ `Batch size: ${batchSize.toLocaleString()}${
+ afterProgram
+ ? `, resuming at program ${afterProgram}${after ? ` after ${after}` : ""}`
+ : ""
+ }\n`,
);
- console.log(`${totalDocuments.toLocaleString()} enrollments to index\n`);
- const result = await backfillPartnerSearch({
- programId,
- batchSize,
- after,
- onProgress: createProgressReporter(totalDocuments, batchSize),
- });
+ let programCount = 0;
+ let totalProcessed = 0;
+
+ for await (const currentProgramId of iterateProgramIds(afterProgram)) {
+ programCount += 1;
+
+ // The enrollment cursor belongs to the program the run stopped inside, so
+ // it applies to that one only. Every program after it starts from scratch.
+ const programAfter = currentProgramId === afterProgram ? after : undefined;
+
+ resumeState.programId = currentProgramId;
+ resumeState.after = programAfter;
+
+ const processed = await backfillProgram({
+ programId: currentProgramId,
+ batchSize,
+ after: programAfter,
+ label: ` [${programCount}] ${currentProgramId}`,
+ countDocuments: false,
+ });
+
+ totalProcessed += processed;
+ console.log(
+ `[${programCount}] ${currentProgramId}: ${processed.toLocaleString()} indexed (${totalProcessed.toLocaleString()} total)`,
+ );
+ }
+
+ const elapsedMin = ((Date.now() - startTime) / 60_000).toFixed(1);
console.log(
- `Partner search backfill complete: ${result.processed.toLocaleString()} documents indexed.`,
+ `\nPartner search backfill complete: ${totalProcessed.toLocaleString()} documents across ${programCount.toLocaleString()} programs in ${elapsedMin} minutes.`,
);
- if (result.lastDocumentId) {
- console.log(`Last document: ${result.lastDocumentId}`);
- }
}
main()
.catch((error) => {
console.error("Partner search backfill failed:", error);
- if (resumeAfter) {
+ if (resumeState.programId) {
const args = process.argv
.slice(2)
- .filter((arg) => !arg.startsWith("--after="))
- .join(" ");
+ .filter(
+ (arg) =>
+ !arg.startsWith("--after=") && !arg.startsWith("--afterProgram="),
+ );
+
+ // An --all run needs both halves of the cursor. A single-program run
+ // already names its program, so it only needs the enrollment.
+ const resumeArgs = args.includes("--all")
+ ? [
+ ...args,
+ `--afterProgram=${resumeState.programId}`,
+ ...(resumeState.after ? [`--after=${resumeState.after}`] : []),
+ ]
+ : [...args, ...(resumeState.after ? [`--after=${resumeState.after}`] : [])];
+
console.error(
- `Documents up to ${resumeAfter} are indexed. Resume with:\n pnpm run script partners/backfill-partner-search ${args} --after=${resumeAfter}`,
+ `Resume with:\n pnpm run script partners/backfill-partner-search ${resumeArgs.join(" ")}`,
);
}
diff --git a/apps/web/tests/partners/backfill-partner-search-args.test.ts b/apps/web/tests/partners/backfill-partner-search-args.test.ts
new file mode 100644
index 0000000000..a42ffc3a96
--- /dev/null
+++ b/apps/web/tests/partners/backfill-partner-search-args.test.ts
@@ -0,0 +1,103 @@
+import {
+ DEFAULT_BATCH_SIZE,
+ MAX_BATCH_SIZE,
+ parseBackfillArguments,
+} from "@/scripts/partners/backfill-partner-search-args";
+import { describe, expect, it } from "vitest";
+
+describe("parseBackfillArguments", () => {
+ it("refuses to run bare, so a full backfill cannot start by accident", () => {
+ expect(() => parseBackfillArguments([])).toThrow(
+ "Pass --programId=<id> for one program, or --all for every program.",
+ );
+ });
+
+ it("takes a single program", () => {
+ expect(parseBackfillArguments(["--programId=prog_1"])).toEqual({
+ programId: "prog_1",
+ all: false,
+ batchSize: DEFAULT_BATCH_SIZE,
+ after: undefined,
+ afterProgram: undefined,
+ });
+ });
+
+ it("takes every program behind an explicit flag", () => {
+ expect(parseBackfillArguments(["--all"])).toEqual({
+ programId: undefined,
+ all: true,
+ batchSize: DEFAULT_BATCH_SIZE,
+ after: undefined,
+ afterProgram: undefined,
+ });
+ });
+
+ it("rejects one program and every program together", () => {
+ expect(() =>
+ parseBackfillArguments(["--programId=prog_1", "--all"]),
+ ).toThrow("--programId and --all are mutually exclusive.");
+ });
+
+ it("resumes a single-program run from an enrollment", () => {
+ expect(
+ parseBackfillArguments(["--programId=prog_1", "--after=pge_9"]),
+ ).toMatchObject({
+ programId: "prog_1",
+ after: "pge_9",
+ });
+ });
+
+ it("resumes an --all run from both halves of the cursor", () => {
+ expect(
+ parseBackfillArguments([
+ "--all",
+ "--afterProgram=prog_5",
+ "--after=pge_9",
+ ]),
+ ).toMatchObject({
+ all: true,
+ afterProgram: "prog_5",
+ after: "pge_9",
+ });
+ });
+
+ it("rejects an enrollment cursor with no program on an --all run", () => {
+ expect(() => parseBackfillArguments(["--all", "--after=pge_9"])).toThrow(
+ "--after requires --afterProgram on an --all run.",
+ );
+ });
+
+ it("rejects a program cursor on a single-program run", () => {
+ expect(() =>
+ parseBackfillArguments(["--programId=prog_1", "--afterProgram=prog_5"]),
+ ).toThrow("--afterProgram only applies to --all runs.");
+ });
+
+ it("rejects empty cursors, which would silently mean no cursor", () => {
+ expect(() => parseBackfillArguments(["--all", "--afterProgram="])).toThrow(
+ "--afterProgram cannot be empty.",
+ );
+ expect(() =>
+ parseBackfillArguments(["--programId=prog_1", "--after="]),
+ ).toThrow("--after cannot be empty.");
+ expect(() => parseBackfillArguments(["--programId="])).toThrow(
+ "Pass --programId=<id> for one program, or --all for every program.",
+ );
+ });
+
+ it("caps the batch size", () => {
+ expect(
+ parseBackfillArguments(["--all", `--batchSize=${MAX_BATCH_SIZE}`]),
+ ).toMatchObject({ batchSize: MAX_BATCH_SIZE });
+
+ expect(() =>
+ parseBackfillArguments(["--all", `--batchSize=${MAX_BATCH_SIZE + 1}`]),
+ ).toThrow(`--batchSize cannot exceed ${MAX_BATCH_SIZE}.`);
+ });
+
+ it("rejects an unknown argument rather than ignoring it", () => {
+ expect(() => parseBackfillArguments(["--all", "--dryRun"])).toThrow(
+ "Unknown argument: --dryRun",
+ );
+ });
+});