Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Move reconciliation sweep to a follow-up PR

Rafid5aa7301dcea
diff --git a/apps/web/app/(ee)/api/cron/partners/search-sweep/route.ts b/apps/web/app/(ee)/api/cron/partners/search-sweep/route.ts
deleted file mode 100644
index faf9104bd1..0000000000
--- a/apps/web/app/(ee)/api/cron/partners/search-sweep/route.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import { getPartnerSearchProvider } from "@/lib/api/partners/search";
-import { withCron } from "@/lib/cron/with-cron";
-import { partnerSearchSweepJob } from "@/lib/jobs/handlers/partner-search-sweep-job";
-import { logAndRespond } from "../../utils";
-
-export const dynamic = "force-dynamic";
-
-// Kicks off a full re-index, which the job carries to completion by
-// re-dispatching itself with a cursor. The backstop for call-site drift, so the
-// pass interval is the worst-case staleness of any document.
-//
-// Runs weekly at 03:00 UTC on Sunday (0 3 * * 0)
-// GET /api/cron/partners/search-sweep
-export const GET = withCron(async () => {
- if (!getPartnerSearchProvider()) {
- return logAndRespond(
- "Partner search provider is not configured, skipping sweep.",
- );
- }
-
- // No cursor: every run starts a fresh pass rather than resuming an
- // interrupted one, so a stalled pass cannot wedge the schedule.
- await partnerSearchSweepJob.dispatch({});
-
- return logAndRespond("Partner search sweep started.");
-});
diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.ts
index 15120f4d8b..0fd472fb0f 100644
--- a/apps/web/lib/api/partners/search/index.ts
+++ b/apps/web/lib/api/partners/search/index.ts
@@ -6,6 +6,5 @@ export * from "./order-search-results";
export * from "./provider";
export * from "./searchable-values";
export * from "./serialize-document";
-export * from "./sweep";
export * from "./sync";
export * from "./types";
diff --git a/apps/web/lib/api/partners/search/sweep.ts b/apps/web/lib/api/partners/search/sweep.ts
deleted file mode 100644
index 8f50a5f5c2..0000000000
--- a/apps/web/lib/api/partners/search/sweep.ts
+++ /dev/null
@@ -1,60 +0,0 @@
-import { indexPartnerSearchEnrollments } from "./index-enrollments";
-import { getPartnerSearchProvider } from "./provider";
-import type { PartnerSearchProvider } from "./types";
-
-export const PARTNER_SEARCH_SWEEP_BATCH_SIZE = 500;
-
-/**
- * How long one hop indexes before handing off. The job route allows 600s, and
- * the margin covers the batch in flight when the budget expires, since the
- * check happens between batches. Errs generous: overrunning kills the hop and
- * QStash retries the same cursor, stalling the pass silently.
- */
-export const PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS = 240_000;
-
-interface SweepPartnerSearchOptions {
- after?: string;
- batchSize?: number;
- timeBudgetMs?: number;
- now?: () => number;
- searchProvider?: PartnerSearchProvider | null;
-}
-
-/**
- * Re-indexes a slice of the corpus, resuming from `after`. The backstop for
- * call-site drift: a mutation path that never queued a sync is corrected by the
- * next pass.
- *
- * Re-indexes everything rather than filtering by timestamp, because a watermark
- * misses a tag change (ProgramPartnerTag has no timestamps) and a link edit
- * (which moves neither the enrollment's timestamp nor the partner's).
- *
- * Upserts only: a pass re-indexes the rows it reads.
- */
-export async function sweepPartnerSearch({
- after,
- batchSize = PARTNER_SEARCH_SWEEP_BATCH_SIZE,
- timeBudgetMs = PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,
- now,
- searchProvider = getPartnerSearchProvider(),
-}: SweepPartnerSearchOptions = {}) {
- if (!searchProvider) {
- throw new Error("Partner search provider is not configured.");
- }
-
- if (!Number.isSafeInteger(batchSize) || batchSize <= 0) {
- throw new Error("Batch size must be a positive integer.");
- }
-
- if (!Number.isSafeInteger(timeBudgetMs) || timeBudgetMs <= 0) {
- throw new Error("Time budget must be a positive integer.");
- }
-
- return await indexPartnerSearchEnrollments({
- searchProvider,
- after,
- batchSize,
- timeBudgetMs,
- ...(now && { now }),
- });
-}
diff --git a/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts b/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts
deleted file mode 100644
index a3b5fc04ec..0000000000
--- a/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts
+++ /dev/null
@@ -1,69 +0,0 @@
-import {
- getPartnerSearchProvider,
- sweepPartnerSearch,
-} from "@/lib/api/partners/search";
-import * as z from "zod/v4";
-import { defineJob } from "../index";
-
-const inputSchema = z.object({
- /** Keyset cursor, carried across hops. Absent starts a fresh pass. */
- after: z.string().optional(),
- /**
- * Documents indexed so far in this pass, for logging continuity. Defaulted in
- * the handler rather than by the schema, because defineJob types the dispatch
- * payload from the schema's output and a zod default would make this required
- * at every call site.
- */
- processed: z.number().int().nonnegative().optional(),
-});
-
-export const partnerSearchSweepJob = defineJob({
- name: "partner-search-sweep-job",
- schema: inputSchema,
- defaults: {
- retries: 3,
- flowControl: {
- // A pass is inherently sequential, and holding it to one in flight keeps
- // a long sweep from competing with interactive syncs for write capacity.
- key: "partner-search-sweep",
- parallelism: 1,
- },
- },
- async handle({ after, processed: alreadyProcessed = 0 }) {
- if (!getPartnerSearchProvider()) {
- console.log(
- "[partnerSearchSweepJob] No search provider configured. Skipping...",
- );
- return;
- }
-
- const { processed, lastDocumentId, done } = await sweepPartnerSearch({
- after,
- });
-
- const totalProcessed = alreadyProcessed + processed;
-
- if (done) {
- console.log(
- `[partnerSearchSweepJob] Pass complete. Re-indexed ${totalProcessed} documents.`,
- );
- return;
- }
-
- console.log(
- `[partnerSearchSweepJob] Re-indexed ${totalProcessed} documents so far, continuing after ${lastDocumentId}.`,
- );
-
- // Cursor lives in the payload rather than in storage, so a pass needs no
- // state of its own and an interrupted one simply restarts on the next cron.
- await partnerSearchSweepJob.dispatch(
- {
- ...(lastDocumentId && { after: lastDocumentId }),
- processed: totalProcessed,
- },
- {
- delay: 1,
- },
- );
- },
-});
diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.ts
index 89e5f8c809..078257b5c9 100644
--- a/apps/web/lib/jobs/registry.ts
+++ b/apps/web/lib/jobs/registry.ts
@@ -34,11 +34,6 @@ const jobLoaders = {
import("./handlers/partner-search-sync-job").then(
(m) => m.partnerSearchSyncJob,
),
-
- "partner-search-sweep-job": () =>
- import("./handlers/partner-search-sweep-job").then(
- (m) => m.partnerSearchSweepJob,
- ),
} as const satisfies Record<string, () => Promise<JobDefinition>>;
const jobCache = new Map<string, JobDefinition>();
diff --git a/apps/web/tests/partners/partner-search-sweep-job.test.ts b/apps/web/tests/partners/partner-search-sweep-job.test.ts
deleted file mode 100644
index 014e88679d..0000000000
--- a/apps/web/tests/partners/partner-search-sweep-job.test.ts
+++ /dev/null
@@ -1,70 +0,0 @@
-import { partnerSearchSweepJob } from "@/lib/jobs/handlers/partner-search-sweep-job";
-import { beforeEach, describe, expect, it, vi } from "vitest";
-
-const mocks = vi.hoisted(() => ({
- getPartnerSearchProvider: vi.fn(),
- sweepPartnerSearch: vi.fn(),
-}));
-
-vi.mock("@/lib/api/partners/search", () => ({
- getPartnerSearchProvider: mocks.getPartnerSearchProvider,
- sweepPartnerSearch: mocks.sweepPartnerSearch,
-}));
-
-describe("partnerSearchSweepJob", () => {
- beforeEach(() => {
- vi.restoreAllMocks();
- mocks.getPartnerSearchProvider
- .mockReset()
- .mockReturnValue({ name: "turbopuffer" });
- mocks.sweepPartnerSearch.mockReset().mockResolvedValue({
- processed: 0,
- lastDocumentId: null,
- done: true,
- });
- });
-
- it("skips entirely when no provider is configured", async () => {
- mocks.getPartnerSearchProvider.mockReturnValue(null);
-
- await partnerSearchSweepJob.execute({});
-
- expect(mocks.sweepPartnerSearch).not.toHaveBeenCalled();
- });
-
- it("stops when the pass is done", async () => {
- const dispatch = vi
- .spyOn(partnerSearchSweepJob, "dispatch")
- .mockResolvedValue({ status: "published", messageId: "msg_1" });
-
- mocks.sweepPartnerSearch.mockResolvedValue({
- processed: 12,
- lastDocumentId: "pge_12",
- done: true,
- });
-
- await partnerSearchSweepJob.execute({});
-
- expect(dispatch).not.toHaveBeenCalled();
- });
-
- it("carries the cursor and running total to the next hop", async () => {
- const dispatch = vi
- .spyOn(partnerSearchSweepJob, "dispatch")
- .mockResolvedValue({ status: "published", messageId: "msg_1" });
-
- mocks.sweepPartnerSearch.mockResolvedValue({
- processed: 20,
- lastDocumentId: "pge_20",
- done: false,
- });
-
- await partnerSearchSweepJob.execute({ after: "pge_10", processed: 30 });
-
- expect(mocks.sweepPartnerSearch).toHaveBeenCalledWith({ after: "pge_10" });
- expect(dispatch).toHaveBeenCalledWith(
- { after: "pge_20", processed: 50 },
- { delay: 1 },
- );
- });
-});
diff --git a/apps/web/tests/partners/partner-search-sweep.test.ts b/apps/web/tests/partners/partner-search-sweep.test.ts
deleted file mode 100644
index 5607c3c658..0000000000
--- a/apps/web/tests/partners/partner-search-sweep.test.ts
+++ /dev/null
@@ -1,197 +0,0 @@
-import {
- PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,
- partnerSearchDocumentSelect,
- sweepPartnerSearch,
- type PartnerSearchDocumentSource,
- type PartnerSearchProvider,
-} from "@/lib/api/partners/search";
-import { beforeEach, describe, expect, it, vi } from "vitest";
-
-const mocks = vi.hoisted(() => ({
- findMany: vi.fn(),
-}));
-
-vi.mock("@/lib/prisma", () => ({
- prisma: {
- programEnrollment: {
- findMany: mocks.findMany,
- },
- },
-}));
-
-function createSource(id: string): PartnerSearchDocumentSource {
- return {
- id,
- programId: "prog_test",
- partnerId: `pn_${id}`,
- status: "approved" as const,
- groupId: null,
- partner: {
- name: "Rafi Hasan",
- email: "[email protected]",
- companyName: null,
- description: null,
- country: null,
- programPartnerTags: [],
- platforms: [],
- },
- links: [],
- };
-}
-
-function createProvider(): PartnerSearchProvider {
- return {
- searchCandidates: vi.fn(),
- countCandidates: vi.fn(),
- upsert: vi.fn(),
- delete: vi.fn(),
- };
-}
-
-describe("sweepPartnerSearch", () => {
- beforeEach(() => {
- mocks.findMany.mockReset();
- });
-
- it("re-indexes the whole corpus rather than a filtered slice", async () => {
- const searchProvider = createProvider();
- mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);
-
- await sweepPartnerSearch({ batchSize: 2, searchProvider });
-
- expect(mocks.findMany).toHaveBeenCalledWith({
- where: {},
- select: partnerSearchDocumentSelect,
- orderBy: { id: "asc" },
- take: 2,
- });
- });
-
- it("reports the pass as done when a page comes back short", async () => {
- const searchProvider = createProvider();
- mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);
-
- const result = await sweepPartnerSearch({ batchSize: 2, searchProvider });
-
- expect(result).toEqual({
- processed: 1,
- lastDocumentId: "pge_1",
- done: true,
- });
- });
-
- it("stops at the time budget and returns a resumable cursor", async () => {
- const searchProvider = createProvider();
- mocks.findMany
- .mockResolvedValueOnce([createSource("pge_1"), createSource("pge_2")])
- .mockResolvedValueOnce([createSource("pge_3"), createSource("pge_4")]);
-
- // Each batch costs 40s against a 60s budget, so the second one crosses it.
- let clock = 0;
- const now = () => (clock += 40_000);
-
- const result = await sweepPartnerSearch({
- batchSize: 2,
- timeBudgetMs: 60_000,
- now,
- searchProvider,
- });
-
- expect(result).toEqual({
- processed: 4,
- lastDocumentId: "pge_4",
- done: false,
- });
- expect(mocks.findMany).toHaveBeenCalledTimes(2);
- });
-
- // Otherwise a hop that starts with no time left makes no progress, and the
- // pass retries the same cursor forever without advancing.
- it("always indexes at least one batch, however tight the budget", async () => {
- const searchProvider = createProvider();
- mocks.findMany.mockResolvedValueOnce([
- createSource("pge_1"),
- createSource("pge_2"),
- ]);
-
- // Budget already blown by the time the first batch finishes.
- let reads = 0;
- const now = () => (reads++ === 0 ? 0 : 10_000);
-
- const result = await sweepPartnerSearch({
- batchSize: 2,
- timeBudgetMs: 1,
- now,
- searchProvider,
- });
-
- expect(result.processed).toBe(2);
- expect(result.lastDocumentId).toBe("pge_2");
- expect(mocks.findMany).toHaveBeenCalledTimes(1);
- });
-
- it("reports done rather than exhausted when the range runs out first", async () => {
- const searchProvider = createProvider();
- mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);
-
- const result = await sweepPartnerSearch({
- batchSize: 2,
- timeBudgetMs: 60_000,
- now: () => 0,
- searchProvider,
- });
-
- expect(result.done).toBe(true);
- });
-
- it("resumes from the cursor it was given", async () => {
- const searchProvider = createProvider();
- mocks.findMany.mockResolvedValueOnce([]);
-
- await sweepPartnerSearch({
- after: "pge_100",
- batchSize: 2,
- searchProvider,
- });
-
- expect(mocks.findMany.mock.calls[0][0].where).toEqual({
- id: { gt: "pge_100" },
- });
- });
-
- it("never deletes, because an out-of-range id looks the same as a missing one", async () => {
- const searchProvider = createProvider();
- mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);
-
- await sweepPartnerSearch({ batchSize: 2, searchProvider });
-
- expect(searchProvider.delete).not.toHaveBeenCalled();
- expect(searchProvider.upsert).toHaveBeenCalledTimes(1);
- });
-
- it("requires a configured provider", async () => {
- await expect(sweepPartnerSearch({ searchProvider: null })).rejects.toThrow(
- "Partner search provider is not configured.",
- );
-
- expect(mocks.findMany).not.toHaveBeenCalled();
- });
-
- it("rejects a non-positive time budget", async () => {
- const searchProvider = createProvider();
-
- await expect(
- sweepPartnerSearch({ timeBudgetMs: 0, searchProvider }),
- ).rejects.toThrow("Time budget must be a positive integer.");
- });
-
- // The hop budget is only meaningful if it finishes well inside the function
- // limit, since the batch in flight when it expires cannot be interrupted.
- it("leaves the job route room to finish the batch in flight", () => {
- const JOB_ROUTE_MAX_DURATION_MS = 600_000;
-
- expect(PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS).toBeLessThanOrEqual(
- JOB_ROUTE_MAX_DURATION_MS / 2,
- );
- });
-});
diff --git a/apps/web/vercel.json b/apps/web/vercel.json
index e6200d1dbc..e59dfb5eda 100644
--- a/apps/web/vercel.json
+++ b/apps/web/vercel.json
@@ -95,10 +95,6 @@
{
"path": "/api/cron/campaigns/queue-scheduled",
"schedule": "* * * * *"
- },
- {
- "path": "/api/cron/partners/search-sweep",
- "schedule": "0 3 * * 0"
}
],
"functions": {