The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Move reconciliation sweep to a follow-up PR
Rafid5aa7301dcea
diff --git a/apps/web/app/(ee)/api/cron/partners/search-sweep/route.ts b/apps/web/app/(ee)/api/cron/partners/search-sweep/route.tsdeleted file mode 100644index faf9104bd1..0000000000--- a/apps/web/app/(ee)/api/cron/partners/search-sweep/route.ts+++ /dev/null@@ -1,26 +0,0 @@-import { getPartnerSearchProvider } from "@/lib/api/partners/search";-import { withCron } from "@/lib/cron/with-cron";-import { partnerSearchSweepJob } from "@/lib/jobs/handlers/partner-search-sweep-job";-import { logAndRespond } from "../../utils";--export const dynamic = "force-dynamic";--// Kicks off a full re-index, which the job carries to completion by-// re-dispatching itself with a cursor. The backstop for call-site drift, so the-// pass interval is the worst-case staleness of any document.-//-// Runs weekly at 03:00 UTC on Sunday (0 3 * * 0)-// GET /api/cron/partners/search-sweep-export const GET = withCron(async () => {- if (!getPartnerSearchProvider()) {- return logAndRespond(- "Partner search provider is not configured, skipping sweep.",- );- }-- // No cursor: every run starts a fresh pass rather than resuming an- // interrupted one, so a stalled pass cannot wedge the schedule.- await partnerSearchSweepJob.dispatch({});-- return logAndRespond("Partner search sweep started.");-});diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.tsindex 15120f4d8b..0fd472fb0f 100644--- a/apps/web/lib/api/partners/search/index.ts+++ b/apps/web/lib/api/partners/search/index.ts@@ -6,6 +6,5 @@ export * from "./order-search-results";export * from "./provider";export * from "./searchable-values";export * from "./serialize-document";-export * from "./sweep";export * from "./sync";export * from "./types";diff --git a/apps/web/lib/api/partners/search/sweep.ts b/apps/web/lib/api/partners/search/sweep.tsdeleted file mode 100644index 8f50a5f5c2..0000000000--- a/apps/web/lib/api/partners/search/sweep.ts+++ /dev/null@@ -1,60 +0,0 @@-import { indexPartnerSearchEnrollments } from "./index-enrollments";-import { getPartnerSearchProvider } from "./provider";-import type { PartnerSearchProvider } from "./types";--export const PARTNER_SEARCH_SWEEP_BATCH_SIZE = 500;--/**- * How long one hop indexes before handing off. The job route allows 600s, and- * the margin covers the batch in flight when the budget expires, since the- * check happens between batches. Errs generous: overrunning kills the hop and- * QStash retries the same cursor, stalling the pass silently.- */-export const PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS = 240_000;--interface SweepPartnerSearchOptions {- after?: string;- batchSize?: number;- timeBudgetMs?: number;- now?: () => number;- searchProvider?: PartnerSearchProvider | null;-}--/**- * Re-indexes a slice of the corpus, resuming from `after`. The backstop for- * call-site drift: a mutation path that never queued a sync is corrected by the- * next pass.- *- * Re-indexes everything rather than filtering by timestamp, because a watermark- * misses a tag change (ProgramPartnerTag has no timestamps) and a link edit- * (which moves neither the enrollment's timestamp nor the partner's).- *- * Upserts only: a pass re-indexes the rows it reads.- */-export async function sweepPartnerSearch({- after,- batchSize = PARTNER_SEARCH_SWEEP_BATCH_SIZE,- timeBudgetMs = PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,- now,- searchProvider = getPartnerSearchProvider(),-}: SweepPartnerSearchOptions = {}) {- if (!searchProvider) {- throw new Error("Partner search provider is not configured.");- }-- if (!Number.isSafeInteger(batchSize) || batchSize <= 0) {- throw new Error("Batch size must be a positive integer.");- }-- if (!Number.isSafeInteger(timeBudgetMs) || timeBudgetMs <= 0) {- throw new Error("Time budget must be a positive integer.");- }-- return await indexPartnerSearchEnrollments({- searchProvider,- after,- batchSize,- timeBudgetMs,- ...(now && { now }),- });-}diff --git a/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts b/apps/web/lib/jobs/handlers/partner-search-sweep-job.tsdeleted file mode 100644index a3b5fc04ec..0000000000--- a/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts+++ /dev/null@@ -1,69 +0,0 @@-import {- getPartnerSearchProvider,- sweepPartnerSearch,-} from "@/lib/api/partners/search";-import * as z from "zod/v4";-import { defineJob } from "../index";--const inputSchema = z.object({- /** Keyset cursor, carried across hops. Absent starts a fresh pass. */- after: z.string().optional(),- /**- * Documents indexed so far in this pass, for logging continuity. Defaulted in- * the handler rather than by the schema, because defineJob types the dispatch- * payload from the schema's output and a zod default would make this required- * at every call site.- */- processed: z.number().int().nonnegative().optional(),-});--export const partnerSearchSweepJob = defineJob({- name: "partner-search-sweep-job",- schema: inputSchema,- defaults: {- retries: 3,- flowControl: {- // A pass is inherently sequential, and holding it to one in flight keeps- // a long sweep from competing with interactive syncs for write capacity.- key: "partner-search-sweep",- parallelism: 1,- },- },- async handle({ after, processed: alreadyProcessed = 0 }) {- if (!getPartnerSearchProvider()) {- console.log(- "[partnerSearchSweepJob] No search provider configured. Skipping...",- );- return;- }-- const { processed, lastDocumentId, done } = await sweepPartnerSearch({- after,- });-- const totalProcessed = alreadyProcessed + processed;-- if (done) {- console.log(- `[partnerSearchSweepJob] Pass complete. Re-indexed ${totalProcessed} documents.`,- );- return;- }-- console.log(- `[partnerSearchSweepJob] Re-indexed ${totalProcessed} documents so far, continuing after ${lastDocumentId}.`,- );-- // Cursor lives in the payload rather than in storage, so a pass needs no- // state of its own and an interrupted one simply restarts on the next cron.- await partnerSearchSweepJob.dispatch(- {- ...(lastDocumentId && { after: lastDocumentId }),- processed: totalProcessed,- },- {- delay: 1,- },- );- },-});diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.tsindex 89e5f8c809..078257b5c9 100644--- a/apps/web/lib/jobs/registry.ts+++ b/apps/web/lib/jobs/registry.ts@@ -34,11 +34,6 @@ const jobLoaders = {import("./handlers/partner-search-sync-job").then((m) => m.partnerSearchSyncJob,),-- "partner-search-sweep-job": () =>- import("./handlers/partner-search-sweep-job").then(- (m) => m.partnerSearchSweepJob,- ),} as const satisfies Record<string, () => Promise<JobDefinition>>;const jobCache = new Map<string, JobDefinition>();diff --git a/apps/web/tests/partners/partner-search-sweep-job.test.ts b/apps/web/tests/partners/partner-search-sweep-job.test.tsdeleted file mode 100644index 014e88679d..0000000000--- a/apps/web/tests/partners/partner-search-sweep-job.test.ts+++ /dev/null@@ -1,70 +0,0 @@-import { partnerSearchSweepJob } from "@/lib/jobs/handlers/partner-search-sweep-job";-import { beforeEach, describe, expect, it, vi } from "vitest";--const mocks = vi.hoisted(() => ({- getPartnerSearchProvider: vi.fn(),- sweepPartnerSearch: vi.fn(),-}));--vi.mock("@/lib/api/partners/search", () => ({- getPartnerSearchProvider: mocks.getPartnerSearchProvider,- sweepPartnerSearch: mocks.sweepPartnerSearch,-}));--describe("partnerSearchSweepJob", () => {- beforeEach(() => {- vi.restoreAllMocks();- mocks.getPartnerSearchProvider- .mockReset()- .mockReturnValue({ name: "turbopuffer" });- mocks.sweepPartnerSearch.mockReset().mockResolvedValue({- processed: 0,- lastDocumentId: null,- done: true,- });- });-- it("skips entirely when no provider is configured", async () => {- mocks.getPartnerSearchProvider.mockReturnValue(null);-- await partnerSearchSweepJob.execute({});-- expect(mocks.sweepPartnerSearch).not.toHaveBeenCalled();- });-- it("stops when the pass is done", async () => {- const dispatch = vi- .spyOn(partnerSearchSweepJob, "dispatch")- .mockResolvedValue({ status: "published", messageId: "msg_1" });-- mocks.sweepPartnerSearch.mockResolvedValue({- processed: 12,- lastDocumentId: "pge_12",- done: true,- });-- await partnerSearchSweepJob.execute({});-- expect(dispatch).not.toHaveBeenCalled();- });-- it("carries the cursor and running total to the next hop", async () => {- const dispatch = vi- .spyOn(partnerSearchSweepJob, "dispatch")- .mockResolvedValue({ status: "published", messageId: "msg_1" });-- mocks.sweepPartnerSearch.mockResolvedValue({- processed: 20,- lastDocumentId: "pge_20",- done: false,- });-- await partnerSearchSweepJob.execute({ after: "pge_10", processed: 30 });-- expect(mocks.sweepPartnerSearch).toHaveBeenCalledWith({ after: "pge_10" });- expect(dispatch).toHaveBeenCalledWith(- { after: "pge_20", processed: 50 },- { delay: 1 },- );- });-});diff --git a/apps/web/tests/partners/partner-search-sweep.test.ts b/apps/web/tests/partners/partner-search-sweep.test.tsdeleted file mode 100644index 5607c3c658..0000000000--- a/apps/web/tests/partners/partner-search-sweep.test.ts+++ /dev/null@@ -1,197 +0,0 @@-import {- PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,- partnerSearchDocumentSelect,- sweepPartnerSearch,- type PartnerSearchDocumentSource,- type PartnerSearchProvider,-} from "@/lib/api/partners/search";-import { beforeEach, describe, expect, it, vi } from "vitest";--const mocks = vi.hoisted(() => ({- findMany: vi.fn(),-}));--vi.mock("@/lib/prisma", () => ({- prisma: {- programEnrollment: {- findMany: mocks.findMany,- },- },-}));--function createSource(id: string): PartnerSearchDocumentSource {- return {- id,- programId: "prog_test",- partnerId: `pn_${id}`,- status: "approved" as const,- groupId: null,- partner: {- name: "Rafi Hasan",- email: "[email protected]",- companyName: null,- description: null,- country: null,- programPartnerTags: [],- platforms: [],- },- links: [],- };-}--function createProvider(): PartnerSearchProvider {- return {- searchCandidates: vi.fn(),- countCandidates: vi.fn(),- upsert: vi.fn(),- delete: vi.fn(),- };-}--describe("sweepPartnerSearch", () => {- beforeEach(() => {- mocks.findMany.mockReset();- });-- it("re-indexes the whole corpus rather than a filtered slice", async () => {- const searchProvider = createProvider();- mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);-- await sweepPartnerSearch({ batchSize: 2, searchProvider });-- expect(mocks.findMany).toHaveBeenCalledWith({- where: {},- select: partnerSearchDocumentSelect,- orderBy: { id: "asc" },- take: 2,- });- });-- it("reports the pass as done when a page comes back short", async () => {- const searchProvider = createProvider();- mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);-- const result = await sweepPartnerSearch({ batchSize: 2, searchProvider });-- expect(result).toEqual({- processed: 1,- lastDocumentId: "pge_1",- done: true,- });- });-- it("stops at the time budget and returns a resumable cursor", async () => {- const searchProvider = createProvider();- mocks.findMany- .mockResolvedValueOnce([createSource("pge_1"), createSource("pge_2")])- .mockResolvedValueOnce([createSource("pge_3"), createSource("pge_4")]);-- // Each batch costs 40s against a 60s budget, so the second one crosses it.- let clock = 0;- const now = () => (clock += 40_000);-- const result = await sweepPartnerSearch({- batchSize: 2,- timeBudgetMs: 60_000,- now,- searchProvider,- });-- expect(result).toEqual({- processed: 4,- lastDocumentId: "pge_4",- done: false,- });- expect(mocks.findMany).toHaveBeenCalledTimes(2);- });-- // Otherwise a hop that starts with no time left makes no progress, and the- // pass retries the same cursor forever without advancing.- it("always indexes at least one batch, however tight the budget", async () => {- const searchProvider = createProvider();- mocks.findMany.mockResolvedValueOnce([- createSource("pge_1"),- createSource("pge_2"),- ]);-- // Budget already blown by the time the first batch finishes.- let reads = 0;- const now = () => (reads++ === 0 ? 0 : 10_000);-- const result = await sweepPartnerSearch({- batchSize: 2,- timeBudgetMs: 1,- now,- searchProvider,- });-- expect(result.processed).toBe(2);- expect(result.lastDocumentId).toBe("pge_2");- expect(mocks.findMany).toHaveBeenCalledTimes(1);- });-- it("reports done rather than exhausted when the range runs out first", async () => {- const searchProvider = createProvider();- mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);-- const result = await sweepPartnerSearch({- batchSize: 2,- timeBudgetMs: 60_000,- now: () => 0,- searchProvider,- });-- expect(result.done).toBe(true);- });-- it("resumes from the cursor it was given", async () => {- const searchProvider = createProvider();- mocks.findMany.mockResolvedValueOnce([]);-- await sweepPartnerSearch({- after: "pge_100",- batchSize: 2,- searchProvider,- });-- expect(mocks.findMany.mock.calls[0][0].where).toEqual({- id: { gt: "pge_100" },- });- });-- it("never deletes, because an out-of-range id looks the same as a missing one", async () => {- const searchProvider = createProvider();- mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);-- await sweepPartnerSearch({ batchSize: 2, searchProvider });-- expect(searchProvider.delete).not.toHaveBeenCalled();- expect(searchProvider.upsert).toHaveBeenCalledTimes(1);- });-- it("requires a configured provider", async () => {- await expect(sweepPartnerSearch({ searchProvider: null })).rejects.toThrow(- "Partner search provider is not configured.",- );-- expect(mocks.findMany).not.toHaveBeenCalled();- });-- it("rejects a non-positive time budget", async () => {- const searchProvider = createProvider();-- await expect(- sweepPartnerSearch({ timeBudgetMs: 0, searchProvider }),- ).rejects.toThrow("Time budget must be a positive integer.");- });-- // The hop budget is only meaningful if it finishes well inside the function- // limit, since the batch in flight when it expires cannot be interrupted.- it("leaves the job route room to finish the batch in flight", () => {- const JOB_ROUTE_MAX_DURATION_MS = 600_000;-- expect(PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS).toBeLessThanOrEqual(- JOB_ROUTE_MAX_DURATION_MS / 2,- );- });-});diff --git a/apps/web/vercel.json b/apps/web/vercel.jsonindex e6200d1dbc..e59dfb5eda 100644--- a/apps/web/vercel.json+++ b/apps/web/vercel.json@@ -95,10 +95,6 @@{"path": "/api/cron/campaigns/queue-scheduled","schedule": "* * * * *"- },- {- "path": "/api/cron/partners/search-sweep",- "schedule": "0 3 * * 0"}],"functions": {