The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Remove the sweep's unused watermark mode
Rafi84238e42fd73
diff --git a/apps/web/lib/api/partners/search/sweep.ts b/apps/web/lib/api/partners/search/sweep.tsindex 4a3f1b0bfb..3883fa7890 100644--- a/apps/web/lib/api/partners/search/sweep.ts+++ b/apps/web/lib/api/partners/search/sweep.ts@@ -1,4 +1,3 @@-import { Prisma } from "@prisma/client";import { indexPartnerSearchEnrollments } from "./index-enrollments";import { getPartnerSearchProvider } from "./provider";import type { PartnerSearchProvider } from "./types";@@ -21,51 +20,27 @@ export const PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS = 240_000;interface SweepPartnerSearchOptions {after?: string;- /**- * Restricts the pass to enrollments touched at or after this moment.- *- * Deliberately not the scheduled mode. Neither ProgramEnrollment.updatedAt- * nor Partner.updatedAt is indexed, so this filter is a scan and belongs to- * one-shot use — most usefully the catch-up after a long backfill, which has- * to re-index whatever changed while it was running.- *- * It also cannot see everything. ProgramPartnerTag carries no timestamps at- * all, and a link edit moves neither of these columns, so a watermark pass is- * strictly narrower than the full pass below.- */- since?: Date;batchSize?: number;timeBudgetMs?: number;now?: () => number;searchProvider?: PartnerSearchProvider | null;}-function buildSweepWhere(- since?: Date,-): Prisma.ProgramEnrollmentWhereInput | undefined {- if (!since) {- return undefined;- }-- return {- OR: [{ updatedAt: { gte: since } }, { partner: { updatedAt: { gte: since } } }],- };-}-/*** Re-indexes a slice of the corpus, resuming from `after`.** This is the backstop for call-site drift. Every hook that fires is an* optimization on top of it: a mutation path that forgets to queue a sync, or* one added later by someone who does not know the index exists, is corrected- * by the next full pass rather than staying wrong until someone notices.+ * by the next pass rather than staying wrong until someone notices.*- * Runs unfiltered by default, re-indexing everything rather than only what a- * timestamp says changed. That costs writes the watermark version would skip,- * and buys the one guarantee the watermark cannot give: no document is stale- * for longer than a full pass, whatever drifted and whether or not the column- * that drifted has a timestamp. Tags have none, which is what makes this the- * only mechanism that covers them.+ * Re-indexes everything rather than only what a timestamp says changed. A+ * watermark would skip writes, but it cannot see a tag change, because+ * ProgramPartnerTag carries no timestamps, nor a link edit, because that moves+ * neither the enrollment's nor the partner's. Passing over everything is what+ * makes the guarantee unconditional: no document is stale for longer than one+ * pass, whatever drifted and whether or not the column that drifted has a+ * timestamp.** Deletions stay outside this. A pass only sees rows that still exist, so an* orphaned document is invisible to it and is cleared by its call-site hook or@@ -73,7 +48,6 @@ function buildSweepWhere(*/export async function sweepPartnerSearch({after,- since,batchSize = PARTNER_SEARCH_SWEEP_BATCH_SIZE,timeBudgetMs = PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,now,@@ -93,7 +67,6 @@ export async function sweepPartnerSearch({return await indexPartnerSearchEnrollments({searchProvider,- where: buildSweepWhere(since),after,batchSize,timeBudgetMs,diff --git a/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts b/apps/web/lib/jobs/handlers/partner-search-sweep-job.tsindex 53db867c86..a3b5fc04ec 100644--- a/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts+++ b/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts@@ -15,8 +15,6 @@ const inputSchema = z.object({* at every call site.*/processed: z.number().int().nonnegative().optional(),- /** One-shot watermark mode. See sweepPartnerSearch. */- since: z.iso.datetime().optional(),});export const partnerSearchSweepJob = defineJob({@@ -31,7 +29,7 @@ export const partnerSearchSweepJob = defineJob({parallelism: 1,},},- async handle({ after, processed: alreadyProcessed = 0, since }) {+ async handle({ after, processed: alreadyProcessed = 0 }) {if (!getPartnerSearchProvider()) {console.log("[partnerSearchSweepJob] No search provider configured. Skipping...",@@ -41,7 +39,6 @@ export const partnerSearchSweepJob = defineJob({const { processed, lastDocumentId, done } = await sweepPartnerSearch({after,- ...(since && { since: new Date(since) }),});const totalProcessed = alreadyProcessed + processed;@@ -63,7 +60,6 @@ export const partnerSearchSweepJob = defineJob({{...(lastDocumentId && { after: lastDocumentId }),processed: totalProcessed,- ...(since && { since }),},{delay: 1,diff --git a/apps/web/tests/partners/partner-search-sweep-job.test.ts b/apps/web/tests/partners/partner-search-sweep-job.test.tsindex 8483d96470..25f43ac1ef 100644--- a/apps/web/tests/partners/partner-search-sweep-job.test.ts+++ b/apps/web/tests/partners/partner-search-sweep-job.test.ts@@ -68,33 +68,4 @@ describe("partnerSearchSweepJob", () => {);});- it("keeps the watermark across hops so a catch-up stays narrowed", async () => {- const dispatch = vi- .spyOn(partnerSearchSweepJob, "dispatch")- .mockResolvedValue({ status: "published", messageId: "msg_1" });-- const since = "2026-08-01T00:00:00.000Z";- mocks.sweepPartnerSearch.mockResolvedValue({- processed: 20,- lastDocumentId: "pge_20",- done: false,- });-- await partnerSearchSweepJob.execute({ since });-- expect(mocks.sweepPartnerSearch).toHaveBeenCalledWith({- after: undefined,- since: new Date(since),- });- expect(dispatch).toHaveBeenCalledWith(- { after: "pge_20", processed: 20, since },- { delay: 1 },- );- });-- it("rejects a malformed watermark rather than sweeping everything", async () => {- await expect(- partnerSearchSweepJob.execute({ since: "last tuesday" }),- ).rejects.toThrow();- });});diff --git a/apps/web/tests/partners/partner-search-sweep.test.ts b/apps/web/tests/partners/partner-search-sweep.test.tsindex d3304bfcef..5607c3c658 100644--- a/apps/web/tests/partners/partner-search-sweep.test.ts+++ b/apps/web/tests/partners/partner-search-sweep.test.ts@@ -159,42 +159,6 @@ describe("sweepPartnerSearch", () => {});});- it("narrows to enrollment and partner timestamps in watermark mode", async () => {- const searchProvider = createProvider();- const since = new Date("2026-08-01T00:00:00.000Z");- mocks.findMany.mockResolvedValueOnce([]);-- await sweepPartnerSearch({ since, batchSize: 2, searchProvider });-- expect(mocks.findMany.mock.calls[0][0].where).toEqual({- OR: [- { updatedAt: { gte: since } },- { partner: { updatedAt: { gte: since } } },- ],- });- });-- it("keeps the watermark filter alongside the cursor when resuming", async () => {- const searchProvider = createProvider();- const since = new Date("2026-08-01T00:00:00.000Z");- mocks.findMany.mockResolvedValueOnce([]);-- await sweepPartnerSearch({- since,- after: "pge_5",- batchSize: 2,- searchProvider,- });-- expect(mocks.findMany.mock.calls[0][0].where).toEqual({- OR: [- { updatedAt: { gte: since } },- { partner: { updatedAt: { gte: since } } },- ],- id: { gt: "pge_5" },- });- });-it("never deletes, because an out-of-range id looks the same as a missing one", async () => {const searchProvider = createProvider();mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);