The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Add the partner search reconciliation sweep
Rafi922fae589c29
diff --git a/apps/web/app/(ee)/api/cron/partners/search-sweep/route.ts b/apps/web/app/(ee)/api/cron/partners/search-sweep/route.tsnew file mode 100644index 0000000000..830760ad06--- /dev/null+++ b/apps/web/app/(ee)/api/cron/partners/search-sweep/route.ts@@ -0,0 +1,30 @@+import { getPartnerSearchProvider } from "@/lib/api/partners/search";+import { withCron } from "@/lib/cron/with-cron";+import { partnerSearchSweepJob } from "@/lib/jobs/handlers/partner-search-sweep-job";+import { logAndRespond } from "../../utils";++export const dynamic = "force-dynamic";++// Kicks off a full re-index of the partner search corpus, which the job then+// carries to completion by re-dispatching itself with a cursor.+//+// This is the backstop for call-site drift: a mutation path that never queued a+// sync, or one added by someone unaware the index exists, is corrected by the+// next pass rather than staying wrong indefinitely. The pass interval is+// therefore the worst-case staleness of any document.+//+// Runs once a week at 03:00 AM UTC on Sunday (0 3 * * 0)+// GET /api/cron/partners/search-sweep+export const GET = withCron(async () => {+ if (!getPartnerSearchProvider()) {+ return logAndRespond(+ "Partner search provider is not configured, skipping sweep.",+ );+ }++ // No cursor: every run starts a fresh pass rather than resuming an+ // interrupted one, so a stalled pass cannot wedge the schedule.+ await partnerSearchSweepJob.dispatch({});++ return logAndRespond("Partner search sweep started.");+});diff --git a/apps/web/lib/api/partners/search/backfill.ts b/apps/web/lib/api/partners/search/backfill.tsindex 1f986c2d4b..311399febd 100644--- a/apps/web/lib/api/partners/search/backfill.ts+++ b/apps/web/lib/api/partners/search/backfill.ts@@ -1,18 +1,13 @@-import { prisma } from "@/lib/prisma";-import { getPartnerSearchProvider } from "./provider";import {- partnerSearchDocumentSelect,- serializePartnerSearchDocument,-} from "./serialize-document";+ indexPartnerSearchEnrollments,+ type PartnerSearchIndexProgress,+} from "./index-enrollments";+import { getPartnerSearchProvider } from "./provider";import type { PartnerSearchProvider } from "./types";const DEFAULT_BATCH_SIZE = 500;-export interface PartnerSearchBackfillProgress {- batchSize: number;- processed: number;- lastDocumentId: string;-}+export type PartnerSearchBackfillProgress = PartnerSearchIndexProgress;interface BackfillPartnerSearchOptions {programId: string;@@ -37,47 +32,16 @@ export async function backfillPartnerSearch({throw new Error("Batch size must be a positive integer.");}- let lastDocumentId = after;- let processed = 0;-- while (true) {- const enrollments = await prisma.programEnrollment.findMany({- where: {- programId,- ...(lastDocumentId && {- id: { gt: lastDocumentId },- }),- },- select: partnerSearchDocumentSelect,- orderBy: {- id: "asc",- },- take: batchSize,- });-- if (enrollments.length === 0) {- break;- }-- await searchProvider.upsert(- enrollments.map(serializePartnerSearchDocument),- );-- lastDocumentId = enrollments[enrollments.length - 1].id;- processed += enrollments.length;- onProgress?.({- batchSize: enrollments.length,- processed,- lastDocumentId,- });-- if (enrollments.length < batchSize) {- break;- }- }+ const { processed, lastDocumentId } = await indexPartnerSearchEnrollments({+ searchProvider,+ where: { programId },+ after,+ batchSize,+ onProgress,+ });return {processed,- lastDocumentId: lastDocumentId ?? null,+ lastDocumentId,};}diff --git a/apps/web/lib/api/partners/search/index-enrollments.ts b/apps/web/lib/api/partners/search/index-enrollments.tsnew file mode 100644index 0000000000..8f8c73bae7--- /dev/null+++ b/apps/web/lib/api/partners/search/index-enrollments.ts@@ -0,0 +1,103 @@+import { prisma } from "@/lib/prisma";+import { Prisma } from "@prisma/client";+import {+ partnerSearchDocumentSelect,+ serializePartnerSearchDocument,+} from "./serialize-document";+import type { PartnerSearchProvider } from "./types";++export interface PartnerSearchIndexProgress {+ batchSize: number;+ processed: number;+ lastDocumentId: string;+}++interface IndexPartnerSearchEnrollmentsOptions {+ searchProvider: PartnerSearchProvider;+ /**+ * Narrows what gets indexed. Must not constrain `id`, which is reserved for+ * the keyset cursor.+ */+ where?: Prisma.ProgramEnrollmentWhereInput;+ after?: string;+ batchSize: number;+ /**+ * Batches to run before returning, so a caller on a request timeout can stop+ * and resume from the returned cursor. Runs to exhaustion when omitted.+ */+ maxBatches?: number;+ onProgress?: (progress: PartnerSearchIndexProgress) => void;+}++/**+ * Pages enrollments by ID and writes them to the index.+ *+ * Keyset paging rather than offset, so per-batch cost stays flat however deep+ * the run gets. Shared by the backfill, which narrows to one program and runs+ * to exhaustion, and the reconciliation sweep, which runs unnarrowed in bounded+ * slices.+ *+ * Upserts only. A document whose enrollment was deleted is not removed here,+ * because an ID absent from a range scan is indistinguishable from one that was+ * never in range. Deletions are the call sites' responsibility.+ */+export async function indexPartnerSearchEnrollments({+ searchProvider,+ where,+ after,+ batchSize,+ maxBatches,+ onProgress,+}: IndexPartnerSearchEnrollmentsOptions) {+ let lastDocumentId = after;+ let processed = 0;+ let batches = 0;+ let done = false;++ while (maxBatches === undefined || batches < maxBatches) {+ const enrollments = await prisma.programEnrollment.findMany({+ where: {+ ...where,+ ...(lastDocumentId && {+ id: { gt: lastDocumentId },+ }),+ },+ select: partnerSearchDocumentSelect,+ orderBy: {+ id: "asc",+ },+ take: batchSize,+ });++ if (enrollments.length === 0) {+ done = true;+ break;+ }++ await searchProvider.upsert(+ enrollments.map(serializePartnerSearchDocument),+ );++ lastDocumentId = enrollments[enrollments.length - 1].id;+ processed += enrollments.length;+ batches++;++ onProgress?.({+ batchSize: enrollments.length,+ processed,+ lastDocumentId,+ });++ // A short page means the range is exhausted.+ if (enrollments.length < batchSize) {+ done = true;+ break;+ }+ }++ return {+ processed,+ lastDocumentId: lastDocumentId ?? null,+ done,+ };+}diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.tsindex b1ea92b8f9..15120f4d8b 100644--- a/apps/web/lib/api/partners/search/index.ts+++ b/apps/web/lib/api/partners/search/index.ts@@ -1,9 +1,11 @@export * from "./backfill";export * from "./build-search-query";export * from "./find-candidates";+export * from "./index-enrollments";export * from "./order-search-results";export * from "./provider";export * from "./searchable-values";export * from "./serialize-document";+export * from "./sweep";export * from "./sync";export * from "./types";diff --git a/apps/web/lib/api/partners/search/sweep.ts b/apps/web/lib/api/partners/search/sweep.tsnew file mode 100644index 0000000000..6dafab3dbd--- /dev/null+++ b/apps/web/lib/api/partners/search/sweep.ts@@ -0,0 +1,92 @@+import { Prisma } from "@prisma/client";+import { indexPartnerSearchEnrollments } from "./index-enrollments";+import { getPartnerSearchProvider } from "./provider";+import type { PartnerSearchProvider } from "./types";++export const PARTNER_SEARCH_SWEEP_BATCH_SIZE = 500;++/**+ * Batches per invocation. At 500 documents each this is 20K per hop, which+ * leaves room under the job route's duration limit while keeping the number of+ * hops for a full pass in the dozens rather than the thousands.+ */+export const PARTNER_SEARCH_SWEEP_MAX_BATCHES = 40;++interface SweepPartnerSearchOptions {+ after?: string;+ /**+ * Restricts the pass to enrollments touched at or after this moment.+ *+ * Deliberately not the scheduled mode. Neither ProgramEnrollment.updatedAt+ * nor Partner.updatedAt is indexed, so this filter is a scan and belongs to+ * one-shot use — most usefully the catch-up after a long backfill, which has+ * to re-index whatever changed while it was running.+ *+ * It also cannot see everything. ProgramPartnerTag carries no timestamps at+ * all, and a link edit moves neither of these columns, so a watermark pass is+ * strictly narrower than the full pass below.+ */+ since?: Date;+ batchSize?: number;+ maxBatches?: number;+ searchProvider?: PartnerSearchProvider | null;+}++function buildSweepWhere(+ since?: Date,+): Prisma.ProgramEnrollmentWhereInput | undefined {+ if (!since) {+ return undefined;+ }++ return {+ OR: [{ updatedAt: { gte: since } }, { partner: { updatedAt: { gte: since } } }],+ };+}++/**+ * Re-indexes a slice of the corpus, resuming from `after`.+ *+ * This is the backstop for call-site drift. Every hook that fires is an+ * optimization on top of it: a mutation path that forgets to queue a sync, or+ * one added later by someone who does not know the index exists, is corrected+ * by the next full pass rather than staying wrong until someone notices.+ *+ * Runs unfiltered by default, re-indexing everything rather than only what a+ * timestamp says changed. That costs writes the watermark version would skip,+ * and buys the one guarantee the watermark cannot give: no document is stale+ * for longer than a full pass, whatever drifted and whether or not the column+ * that drifted has a timestamp. Tags have none, which is what makes this the+ * only mechanism that covers them.+ *+ * Deletions stay outside this. A pass only sees rows that still exist, so an+ * orphaned document is invisible to it and is cleared by its call-site hook or+ * by rebuilding the namespace.+ */+export async function sweepPartnerSearch({+ after,+ since,+ batchSize = PARTNER_SEARCH_SWEEP_BATCH_SIZE,+ maxBatches = PARTNER_SEARCH_SWEEP_MAX_BATCHES,+ searchProvider = getPartnerSearchProvider(),+}: SweepPartnerSearchOptions = {}) {+ if (!searchProvider) {+ throw new Error("Partner search provider is not configured.");+ }++ if (!Number.isSafeInteger(batchSize) || batchSize <= 0) {+ throw new Error("Batch size must be a positive integer.");+ }++ if (!Number.isSafeInteger(maxBatches) || maxBatches <= 0) {+ throw new Error("Max batches must be a positive integer.");+ }++ return await indexPartnerSearchEnrollments({+ searchProvider,+ where: buildSweepWhere(since),+ after,+ batchSize,+ maxBatches,+ });+}diff --git a/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts b/apps/web/lib/jobs/handlers/partner-search-sweep-job.tsnew file mode 100644index 0000000000..53db867c86--- /dev/null+++ b/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts@@ -0,0 +1,73 @@+import {+ getPartnerSearchProvider,+ sweepPartnerSearch,+} from "@/lib/api/partners/search";+import * as z from "zod/v4";+import { defineJob } from "../index";++const inputSchema = z.object({+ /** Keyset cursor, carried across hops. Absent starts a fresh pass. */+ after: z.string().optional(),+ /**+ * Documents indexed so far in this pass, for logging continuity. Defaulted in+ * the handler rather than by the schema, because defineJob types the dispatch+ * payload from the schema's output and a zod default would make this required+ * at every call site.+ */+ processed: z.number().int().nonnegative().optional(),+ /** One-shot watermark mode. See sweepPartnerSearch. */+ since: z.iso.datetime().optional(),+});++export const partnerSearchSweepJob = defineJob({+ name: "partner-search-sweep-job",+ schema: inputSchema,+ defaults: {+ retries: 3,+ flowControl: {+ // A pass is inherently sequential, and holding it to one in flight keeps+ // a long sweep from competing with interactive syncs for write capacity.+ key: "partner-search-sweep",+ parallelism: 1,+ },+ },+ async handle({ after, processed: alreadyProcessed = 0, since }) {+ if (!getPartnerSearchProvider()) {+ console.log(+ "[partnerSearchSweepJob] No search provider configured. Skipping...",+ );+ return;+ }++ const { processed, lastDocumentId, done } = await sweepPartnerSearch({+ after,+ ...(since && { since: new Date(since) }),+ });++ const totalProcessed = alreadyProcessed + processed;++ if (done) {+ console.log(+ `[partnerSearchSweepJob] Pass complete. Re-indexed ${totalProcessed} documents.`,+ );+ return;+ }++ console.log(+ `[partnerSearchSweepJob] Re-indexed ${totalProcessed} documents so far, continuing after ${lastDocumentId}.`,+ );++ // Cursor lives in the payload rather than in storage, so a pass needs no+ // state of its own and an interrupted one simply restarts on the next cron.+ await partnerSearchSweepJob.dispatch(+ {+ ...(lastDocumentId && { after: lastDocumentId }),+ processed: totalProcessed,+ ...(since && { since }),+ },+ {+ delay: 1,+ },+ );+ },+});diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.tsindex 078257b5c9..89e5f8c809 100644--- a/apps/web/lib/jobs/registry.ts+++ b/apps/web/lib/jobs/registry.ts@@ -34,6 +34,11 @@ const jobLoaders = {import("./handlers/partner-search-sync-job").then((m) => m.partnerSearchSyncJob,),++ "partner-search-sweep-job": () =>+ import("./handlers/partner-search-sweep-job").then(+ (m) => m.partnerSearchSweepJob,+ ),} as const satisfies Record<string, () => Promise<JobDefinition>>;const jobCache = new Map<string, JobDefinition>();diff --git a/apps/web/tests/partners/partner-search-sweep-job.test.ts b/apps/web/tests/partners/partner-search-sweep-job.test.tsnew file mode 100644index 0000000000..8483d96470--- /dev/null+++ b/apps/web/tests/partners/partner-search-sweep-job.test.ts@@ -0,0 +1,100 @@+import { partnerSearchSweepJob } from "@/lib/jobs/handlers/partner-search-sweep-job";+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ getPartnerSearchProvider: vi.fn(),+ sweepPartnerSearch: vi.fn(),+}));++vi.mock("@/lib/api/partners/search", () => ({+ getPartnerSearchProvider: mocks.getPartnerSearchProvider,+ sweepPartnerSearch: mocks.sweepPartnerSearch,+}));++describe("partnerSearchSweepJob", () => {+ beforeEach(() => {+ vi.restoreAllMocks();+ mocks.getPartnerSearchProvider+ .mockReset()+ .mockReturnValue({ name: "turbopuffer" });+ mocks.sweepPartnerSearch.mockReset().mockResolvedValue({+ processed: 0,+ lastDocumentId: null,+ done: true,+ });+ });++ it("skips entirely when no provider is configured", async () => {+ mocks.getPartnerSearchProvider.mockReturnValue(null);++ await partnerSearchSweepJob.execute({});++ expect(mocks.sweepPartnerSearch).not.toHaveBeenCalled();+ });++ it("stops when the pass is done", async () => {+ const dispatch = vi+ .spyOn(partnerSearchSweepJob, "dispatch")+ .mockResolvedValue({ status: "published", messageId: "msg_1" });++ mocks.sweepPartnerSearch.mockResolvedValue({+ processed: 12,+ lastDocumentId: "pge_12",+ done: true,+ });++ await partnerSearchSweepJob.execute({});++ expect(dispatch).not.toHaveBeenCalled();+ });++ it("carries the cursor and running total to the next hop", async () => {+ const dispatch = vi+ .spyOn(partnerSearchSweepJob, "dispatch")+ .mockResolvedValue({ status: "published", messageId: "msg_1" });++ mocks.sweepPartnerSearch.mockResolvedValue({+ processed: 20,+ lastDocumentId: "pge_20",+ done: false,+ });++ await partnerSearchSweepJob.execute({ after: "pge_10", processed: 30 });++ expect(mocks.sweepPartnerSearch).toHaveBeenCalledWith({ after: "pge_10" });+ expect(dispatch).toHaveBeenCalledWith(+ { after: "pge_20", processed: 50 },+ { delay: 1 },+ );+ });++ it("keeps the watermark across hops so a catch-up stays narrowed", async () => {+ const dispatch = vi+ .spyOn(partnerSearchSweepJob, "dispatch")+ .mockResolvedValue({ status: "published", messageId: "msg_1" });++ const since = "2026-08-01T00:00:00.000Z";+ mocks.sweepPartnerSearch.mockResolvedValue({+ processed: 20,+ lastDocumentId: "pge_20",+ done: false,+ });++ await partnerSearchSweepJob.execute({ since });++ expect(mocks.sweepPartnerSearch).toHaveBeenCalledWith({+ after: undefined,+ since: new Date(since),+ });+ expect(dispatch).toHaveBeenCalledWith(+ { after: "pge_20", processed: 20, since },+ { delay: 1 },+ );+ });++ it("rejects a malformed watermark rather than sweeping everything", async () => {+ await expect(+ partnerSearchSweepJob.execute({ since: "last tuesday" }),+ ).rejects.toThrow();+ });+});diff --git a/apps/web/tests/partners/partner-search-sweep.test.ts b/apps/web/tests/partners/partner-search-sweep.test.tsnew file mode 100644index 0000000000..52b606f482--- /dev/null+++ b/apps/web/tests/partners/partner-search-sweep.test.ts@@ -0,0 +1,178 @@+import {+ partnerSearchDocumentSelect,+ sweepPartnerSearch,+ type PartnerSearchDocumentSource,+ type PartnerSearchProvider,+} from "@/lib/api/partners/search";+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ findMany: vi.fn(),+}));++vi.mock("@/lib/prisma", () => ({+ prisma: {+ programEnrollment: {+ findMany: mocks.findMany,+ },+ },+}));++function createSource(id: string): PartnerSearchDocumentSource {+ return {+ id,+ programId: "prog_test",+ partnerId: `pn_${id}`,+ status: "approved" as const,+ groupId: null,+ partner: {+ name: "Rafi Hasan",+ email: "[email protected]",+ companyName: null,+ description: null,+ country: null,+ programPartnerTags: [],+ platforms: [],+ },+ links: [],+ };+}++function createProvider(): PartnerSearchProvider {+ return {+ searchCandidates: vi.fn(),+ countCandidates: vi.fn(),+ upsert: vi.fn(),+ delete: vi.fn(),+ };+}++describe("sweepPartnerSearch", () => {+ beforeEach(() => {+ mocks.findMany.mockReset();+ });++ it("re-indexes the whole corpus rather than a filtered slice", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);++ await sweepPartnerSearch({ batchSize: 2, searchProvider });++ expect(mocks.findMany).toHaveBeenCalledWith({+ where: {},+ select: partnerSearchDocumentSelect,+ orderBy: { id: "asc" },+ take: 2,+ });+ });++ it("reports the pass as done when a page comes back short", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);++ const result = await sweepPartnerSearch({ batchSize: 2, searchProvider });++ expect(result).toEqual({+ processed: 1,+ lastDocumentId: "pge_1",+ done: true,+ });+ });++ it("stops at the batch ceiling and returns a resumable cursor", async () => {+ const searchProvider = createProvider();+ mocks.findMany+ .mockResolvedValueOnce([createSource("pge_1"), createSource("pge_2")])+ .mockResolvedValueOnce([createSource("pge_3"), createSource("pge_4")]);++ const result = await sweepPartnerSearch({+ batchSize: 2,+ maxBatches: 2,+ searchProvider,+ });++ expect(result).toEqual({+ processed: 4,+ lastDocumentId: "pge_4",+ done: false,+ });+ expect(mocks.findMany).toHaveBeenCalledTimes(2);+ });++ it("resumes from the cursor it was given", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([]);++ await sweepPartnerSearch({+ after: "pge_100",+ batchSize: 2,+ searchProvider,+ });++ expect(mocks.findMany.mock.calls[0][0].where).toEqual({+ id: { gt: "pge_100" },+ });+ });++ it("narrows to enrollment and partner timestamps in watermark mode", async () => {+ const searchProvider = createProvider();+ const since = new Date("2026-08-01T00:00:00.000Z");+ mocks.findMany.mockResolvedValueOnce([]);++ await sweepPartnerSearch({ since, batchSize: 2, searchProvider });++ expect(mocks.findMany.mock.calls[0][0].where).toEqual({+ OR: [+ { updatedAt: { gte: since } },+ { partner: { updatedAt: { gte: since } } },+ ],+ });+ });++ it("keeps the watermark filter alongside the cursor when resuming", async () => {+ const searchProvider = createProvider();+ const since = new Date("2026-08-01T00:00:00.000Z");+ mocks.findMany.mockResolvedValueOnce([]);++ await sweepPartnerSearch({+ since,+ after: "pge_5",+ batchSize: 2,+ searchProvider,+ });++ expect(mocks.findMany.mock.calls[0][0].where).toEqual({+ OR: [+ { updatedAt: { gte: since } },+ { partner: { updatedAt: { gte: since } } },+ ],+ id: { gt: "pge_5" },+ });+ });++ it("never deletes, because an out-of-range id looks the same as a missing one", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);++ await sweepPartnerSearch({ batchSize: 2, searchProvider });++ expect(searchProvider.delete).not.toHaveBeenCalled();+ expect(searchProvider.upsert).toHaveBeenCalledTimes(1);+ });++ it("requires a configured provider", async () => {+ await expect(sweepPartnerSearch({ searchProvider: null })).rejects.toThrow(+ "Partner search provider is not configured.",+ );++ expect(mocks.findMany).not.toHaveBeenCalled();+ });++ it("rejects a non-positive batch ceiling", async () => {+ const searchProvider = createProvider();++ await expect(+ sweepPartnerSearch({ maxBatches: 0, searchProvider }),+ ).rejects.toThrow("Max batches must be a positive integer.");+ });+});diff --git a/apps/web/vercel.json b/apps/web/vercel.jsonindex 2627027c59..82212ce77d 100644--- a/apps/web/vercel.json+++ b/apps/web/vercel.json@@ -91,6 +91,10 @@{"path": "/api/cron/queue/retry","schedule": "* * * * *"+ },+ {+ "path": "/api/cron/partners/search-sweep",+ "schedule": "0 3 * * 0"}],"functions": {