Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Bound a sweep hop by time rather than by batch count

Rafif5c3bd852c2f
diff --git a/apps/web/lib/api/partners/search/index-enrollments.ts b/apps/web/lib/api/partners/search/index-enrollments.ts
index 8f8c73bae7..ef41154383 100644
--- a/apps/web/lib/api/partners/search/index-enrollments.ts
+++ b/apps/web/lib/api/partners/search/index-enrollments.ts
@@ -22,10 +22,19 @@ interface IndexPartnerSearchEnrollmentsOptions {
after?: string;
batchSize: number;
/**
- * Batches to run before returning, so a caller on a request timeout can stop
- * and resume from the returned cursor. Runs to exhaustion when omitted.
+ * Wall-clock budget, checked between batches, so a caller running under a
+ * function timeout stops on its own and resumes from the returned cursor.
+ * Runs to exhaustion when omitted.
+ *
+ * A budget rather than a batch count because the constraint being respected
+ * is a duration. A batch count only stands in for one if you already know how
+ * long a batch takes, which depends on document size, provider latency, and
+ * how loaded the database is, and is therefore exactly the thing not worth
+ * pinning to a constant.
*/
- maxBatches?: number;
+ timeBudgetMs?: number;
+ /** Injectable clock, so the budget is testable without real time passing. */
+ now?: () => number;
onProgress?: (progress: PartnerSearchIndexProgress) => void;
}
@@ -46,15 +55,16 @@ export async function indexPartnerSearchEnrollments({
where,
after,
batchSize,
- maxBatches,
+ timeBudgetMs,
+ now = Date.now,
onProgress,
}: IndexPartnerSearchEnrollmentsOptions) {
+ const startedAt = now();
let lastDocumentId = after;
let processed = 0;
- let batches = 0;
let done = false;
- while (maxBatches === undefined || batches < maxBatches) {
+ while (true) {
const enrollments = await prisma.programEnrollment.findMany({
where: {
...where,
@@ -80,7 +90,6 @@ export async function indexPartnerSearchEnrollments({
lastDocumentId = enrollments[enrollments.length - 1].id;
processed += enrollments.length;
- batches++;
onProgress?.({
batchSize: enrollments.length,
@@ -93,6 +102,13 @@ export async function indexPartnerSearchEnrollments({
done = true;
break;
}
+
+ // Checked after a batch rather than before, so a run always makes progress
+ // however tight the budget is. One batch can therefore overrun it, which is
+ // why callers leave headroom rather than budgeting to their whole limit.
+ if (timeBudgetMs !== undefined && now() - startedAt >= timeBudgetMs) {
+ break;
+ }
}
return {
diff --git a/apps/web/lib/api/partners/search/sweep.ts b/apps/web/lib/api/partners/search/sweep.ts
index 6dafab3dbd..4a3f1b0bfb 100644
--- a/apps/web/lib/api/partners/search/sweep.ts
+++ b/apps/web/lib/api/partners/search/sweep.ts
@@ -6,11 +6,18 @@ import type { PartnerSearchProvider } from "./types";
export const PARTNER_SEARCH_SWEEP_BATCH_SIZE = 500;
/**
- * Batches per invocation. At 500 documents each this is 20K per hop, which
- * leaves room under the job route's duration limit while keeping the number of
- * hops for a full pass in the dozens rather than the thousands.
+ * How long one hop indexes before handing off to the next.
+ *
+ * The job route allows 600s, so this leaves more than half the budget spare.
+ * The margin covers the batch in flight when the budget runs out, since the
+ * check happens between batches and cannot interrupt one.
+ *
+ * Deliberately generous rather than tuned. Overrunning the route's limit kills
+ * the hop, and QStash then retries the same cursor until it gives up, which
+ * stalls the pass with nothing to say so. Finishing a hop early costs one extra
+ * message.
*/
-export const PARTNER_SEARCH_SWEEP_MAX_BATCHES = 40;
+export const PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS = 240_000;
interface SweepPartnerSearchOptions {
after?: string;
@@ -28,7 +35,8 @@ interface SweepPartnerSearchOptions {
*/
since?: Date;
batchSize?: number;
- maxBatches?: number;
+ timeBudgetMs?: number;
+ now?: () => number;
searchProvider?: PartnerSearchProvider | null;
}
@@ -67,7 +75,8 @@ export async function sweepPartnerSearch({
after,
since,
batchSize = PARTNER_SEARCH_SWEEP_BATCH_SIZE,
- maxBatches = PARTNER_SEARCH_SWEEP_MAX_BATCHES,
+ timeBudgetMs = PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,
+ now,
searchProvider = getPartnerSearchProvider(),
}: SweepPartnerSearchOptions = {}) {
if (!searchProvider) {
@@ -78,8 +87,8 @@ export async function sweepPartnerSearch({
throw new Error("Batch size must be a positive integer.");
}
- if (!Number.isSafeInteger(maxBatches) || maxBatches <= 0) {
- throw new Error("Max batches must be a positive integer.");
+ if (!Number.isSafeInteger(timeBudgetMs) || timeBudgetMs <= 0) {
+ throw new Error("Time budget must be a positive integer.");
}
return await indexPartnerSearchEnrollments({
@@ -87,6 +96,7 @@ export async function sweepPartnerSearch({
where: buildSweepWhere(since),
after,
batchSize,
- maxBatches,
+ timeBudgetMs,
+ ...(now && { now }),
});
}
diff --git a/apps/web/tests/partners/partner-search-sweep.test.ts b/apps/web/tests/partners/partner-search-sweep.test.ts
index 52b606f482..d3304bfcef 100644
--- a/apps/web/tests/partners/partner-search-sweep.test.ts
+++ b/apps/web/tests/partners/partner-search-sweep.test.ts
@@ -1,4 +1,5 @@
import {
+ PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,
partnerSearchDocumentSelect,
sweepPartnerSearch,
type PartnerSearchDocumentSource,
@@ -79,15 +80,20 @@ describe("sweepPartnerSearch", () => {
});
});
- it("stops at the batch ceiling and returns a resumable cursor", async () => {
+ it("stops at the time budget and returns a resumable cursor", async () => {
const searchProvider = createProvider();
mocks.findMany
.mockResolvedValueOnce([createSource("pge_1"), createSource("pge_2")])
.mockResolvedValueOnce([createSource("pge_3"), createSource("pge_4")]);
+ // Each batch costs 40s against a 60s budget, so the second one crosses it.
+ let clock = 0;
+ const now = () => (clock += 40_000);
+
const result = await sweepPartnerSearch({
batchSize: 2,
- maxBatches: 2,
+ timeBudgetMs: 60_000,
+ now,
searchProvider,
});
@@ -99,6 +105,45 @@ describe("sweepPartnerSearch", () => {
expect(mocks.findMany).toHaveBeenCalledTimes(2);
});
+ // Otherwise a hop that starts with no time left makes no progress, and the
+ // pass retries the same cursor forever without advancing.
+ it("always indexes at least one batch, however tight the budget", async () => {
+ const searchProvider = createProvider();
+ mocks.findMany.mockResolvedValueOnce([
+ createSource("pge_1"),
+ createSource("pge_2"),
+ ]);
+
+ // Budget already blown by the time the first batch finishes.
+ let reads = 0;
+ const now = () => (reads++ === 0 ? 0 : 10_000);
+
+ const result = await sweepPartnerSearch({
+ batchSize: 2,
+ timeBudgetMs: 1,
+ now,
+ searchProvider,
+ });
+
+ expect(result.processed).toBe(2);
+ expect(result.lastDocumentId).toBe("pge_2");
+ expect(mocks.findMany).toHaveBeenCalledTimes(1);
+ });
+
+ it("reports done rather than exhausted when the range runs out first", async () => {
+ const searchProvider = createProvider();
+ mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);
+
+ const result = await sweepPartnerSearch({
+ batchSize: 2,
+ timeBudgetMs: 60_000,
+ now: () => 0,
+ searchProvider,
+ });
+
+ expect(result.done).toBe(true);
+ });
+
it("resumes from the cursor it was given", async () => {
const searchProvider = createProvider();
mocks.findMany.mockResolvedValueOnce([]);
@@ -168,11 +213,21 @@ describe("sweepPartnerSearch", () => {
expect(mocks.findMany).not.toHaveBeenCalled();
});
- it("rejects a non-positive batch ceiling", async () => {
+ it("rejects a non-positive time budget", async () => {
const searchProvider = createProvider();
await expect(
- sweepPartnerSearch({ maxBatches: 0, searchProvider }),
- ).rejects.toThrow("Max batches must be a positive integer.");
+ sweepPartnerSearch({ timeBudgetMs: 0, searchProvider }),
+ ).rejects.toThrow("Time budget must be a positive integer.");
+ });
+
+ // The hop budget is only meaningful if it finishes well inside the function
+ // limit, since the batch in flight when it expires cannot be interrupted.
+ it("leaves the job route room to finish the batch in flight", () => {
+ const JOB_ROUTE_MAX_DURATION_MS = 600_000;
+
+ expect(PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS).toBeLessThanOrEqual(
+ JOB_ROUTE_MAX_DURATION_MS / 2,
+ );
});
});