The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Bound a sweep hop by time rather than by batch count
Rafif5c3bd852c2f
diff --git a/apps/web/lib/api/partners/search/index-enrollments.ts b/apps/web/lib/api/partners/search/index-enrollments.tsindex 8f8c73bae7..ef41154383 100644--- a/apps/web/lib/api/partners/search/index-enrollments.ts+++ b/apps/web/lib/api/partners/search/index-enrollments.ts@@ -22,10 +22,19 @@ interface IndexPartnerSearchEnrollmentsOptions {after?: string;batchSize: number;/**- * Batches to run before returning, so a caller on a request timeout can stop- * and resume from the returned cursor. Runs to exhaustion when omitted.+ * Wall-clock budget, checked between batches, so a caller running under a+ * function timeout stops on its own and resumes from the returned cursor.+ * Runs to exhaustion when omitted.+ *+ * A budget rather than a batch count because the constraint being respected+ * is a duration. A batch count only stands in for one if you already know how+ * long a batch takes, which depends on document size, provider latency, and+ * how loaded the database is, and is therefore exactly the thing not worth+ * pinning to a constant.*/- maxBatches?: number;+ timeBudgetMs?: number;+ /** Injectable clock, so the budget is testable without real time passing. */+ now?: () => number;onProgress?: (progress: PartnerSearchIndexProgress) => void;}@@ -46,15 +55,16 @@ export async function indexPartnerSearchEnrollments({where,after,batchSize,- maxBatches,+ timeBudgetMs,+ now = Date.now,onProgress,}: IndexPartnerSearchEnrollmentsOptions) {+ const startedAt = now();let lastDocumentId = after;let processed = 0;- let batches = 0;let done = false;- while (maxBatches === undefined || batches < maxBatches) {+ while (true) {const enrollments = await prisma.programEnrollment.findMany({where: {...where,@@ -80,7 +90,6 @@ export async function indexPartnerSearchEnrollments({lastDocumentId = enrollments[enrollments.length - 1].id;processed += enrollments.length;- batches++;onProgress?.({batchSize: enrollments.length,@@ -93,6 +102,13 @@ export async function indexPartnerSearchEnrollments({done = true;break;}++ // Checked after a batch rather than before, so a run always makes progress+ // however tight the budget is. One batch can therefore overrun it, which is+ // why callers leave headroom rather than budgeting to their whole limit.+ if (timeBudgetMs !== undefined && now() - startedAt >= timeBudgetMs) {+ break;+ }}return {diff --git a/apps/web/lib/api/partners/search/sweep.ts b/apps/web/lib/api/partners/search/sweep.tsindex 6dafab3dbd..4a3f1b0bfb 100644--- a/apps/web/lib/api/partners/search/sweep.ts+++ b/apps/web/lib/api/partners/search/sweep.ts@@ -6,11 +6,18 @@ import type { PartnerSearchProvider } from "./types";export const PARTNER_SEARCH_SWEEP_BATCH_SIZE = 500;/**- * Batches per invocation. At 500 documents each this is 20K per hop, which- * leaves room under the job route's duration limit while keeping the number of- * hops for a full pass in the dozens rather than the thousands.+ * How long one hop indexes before handing off to the next.+ *+ * The job route allows 600s, so this leaves more than half the budget spare.+ * The margin covers the batch in flight when the budget runs out, since the+ * check happens between batches and cannot interrupt one.+ *+ * Deliberately generous rather than tuned. Overrunning the route's limit kills+ * the hop, and QStash then retries the same cursor until it gives up, which+ * stalls the pass with nothing to say so. Finishing a hop early costs one extra+ * message.*/-export const PARTNER_SEARCH_SWEEP_MAX_BATCHES = 40;+export const PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS = 240_000;interface SweepPartnerSearchOptions {after?: string;@@ -28,7 +35,8 @@ interface SweepPartnerSearchOptions {*/since?: Date;batchSize?: number;- maxBatches?: number;+ timeBudgetMs?: number;+ now?: () => number;searchProvider?: PartnerSearchProvider | null;}@@ -67,7 +75,8 @@ export async function sweepPartnerSearch({after,since,batchSize = PARTNER_SEARCH_SWEEP_BATCH_SIZE,- maxBatches = PARTNER_SEARCH_SWEEP_MAX_BATCHES,+ timeBudgetMs = PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,+ now,searchProvider = getPartnerSearchProvider(),}: SweepPartnerSearchOptions = {}) {if (!searchProvider) {@@ -78,8 +87,8 @@ export async function sweepPartnerSearch({throw new Error("Batch size must be a positive integer.");}- if (!Number.isSafeInteger(maxBatches) || maxBatches <= 0) {- throw new Error("Max batches must be a positive integer.");+ if (!Number.isSafeInteger(timeBudgetMs) || timeBudgetMs <= 0) {+ throw new Error("Time budget must be a positive integer.");}return await indexPartnerSearchEnrollments({@@ -87,6 +96,7 @@ export async function sweepPartnerSearch({where: buildSweepWhere(since),after,batchSize,- maxBatches,+ timeBudgetMs,+ ...(now && { now }),});}diff --git a/apps/web/tests/partners/partner-search-sweep.test.ts b/apps/web/tests/partners/partner-search-sweep.test.tsindex 52b606f482..d3304bfcef 100644--- a/apps/web/tests/partners/partner-search-sweep.test.ts+++ b/apps/web/tests/partners/partner-search-sweep.test.ts@@ -1,4 +1,5 @@import {+ PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,partnerSearchDocumentSelect,sweepPartnerSearch,type PartnerSearchDocumentSource,@@ -79,15 +80,20 @@ describe("sweepPartnerSearch", () => {});});- it("stops at the batch ceiling and returns a resumable cursor", async () => {+ it("stops at the time budget and returns a resumable cursor", async () => {const searchProvider = createProvider();mocks.findMany.mockResolvedValueOnce([createSource("pge_1"), createSource("pge_2")]).mockResolvedValueOnce([createSource("pge_3"), createSource("pge_4")]);+ // Each batch costs 40s against a 60s budget, so the second one crosses it.+ let clock = 0;+ const now = () => (clock += 40_000);+const result = await sweepPartnerSearch({batchSize: 2,- maxBatches: 2,+ timeBudgetMs: 60_000,+ now,searchProvider,});@@ -99,6 +105,45 @@ describe("sweepPartnerSearch", () => {expect(mocks.findMany).toHaveBeenCalledTimes(2);});+ // Otherwise a hop that starts with no time left makes no progress, and the+ // pass retries the same cursor forever without advancing.+ it("always indexes at least one batch, however tight the budget", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([+ createSource("pge_1"),+ createSource("pge_2"),+ ]);++ // Budget already blown by the time the first batch finishes.+ let reads = 0;+ const now = () => (reads++ === 0 ? 0 : 10_000);++ const result = await sweepPartnerSearch({+ batchSize: 2,+ timeBudgetMs: 1,+ now,+ searchProvider,+ });++ expect(result.processed).toBe(2);+ expect(result.lastDocumentId).toBe("pge_2");+ expect(mocks.findMany).toHaveBeenCalledTimes(1);+ });++ it("reports done rather than exhausted when the range runs out first", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);++ const result = await sweepPartnerSearch({+ batchSize: 2,+ timeBudgetMs: 60_000,+ now: () => 0,+ searchProvider,+ });++ expect(result.done).toBe(true);+ });+it("resumes from the cursor it was given", async () => {const searchProvider = createProvider();mocks.findMany.mockResolvedValueOnce([]);@@ -168,11 +213,21 @@ describe("sweepPartnerSearch", () => {expect(mocks.findMany).not.toHaveBeenCalled();});- it("rejects a non-positive batch ceiling", async () => {+ it("rejects a non-positive time budget", async () => {const searchProvider = createProvider();await expect(- sweepPartnerSearch({ maxBatches: 0, searchProvider }),- ).rejects.toThrow("Max batches must be a positive integer.");+ sweepPartnerSearch({ timeBudgetMs: 0, searchProvider }),+ ).rejects.toThrow("Time budget must be a positive integer.");+ });++ // The hop budget is only meaningful if it finishes well inside the function+ // limit, since the batch in flight when it expires cannot be interrupted.+ it("leaves the job route room to finish the batch in flight", () => {+ const JOB_ROUTE_MAX_DURATION_MS = 600_000;++ expect(PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS).toBeLessThanOrEqual(+ JOB_ROUTE_MAX_DURATION_MS / 2,+ );});});