Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Remove the sweep's unused watermark mode

Rafi84238e42fd73
diff --git a/apps/web/lib/api/partners/search/sweep.ts b/apps/web/lib/api/partners/search/sweep.ts
index 4a3f1b0bfb..3883fa7890 100644
--- a/apps/web/lib/api/partners/search/sweep.ts
+++ b/apps/web/lib/api/partners/search/sweep.ts
@@ -1,4 +1,3 @@
-import { Prisma } from "@prisma/client";
import { indexPartnerSearchEnrollments } from "./index-enrollments";
import { getPartnerSearchProvider } from "./provider";
import type { PartnerSearchProvider } from "./types";
@@ -21,51 +20,27 @@ export const PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS = 240_000;
interface SweepPartnerSearchOptions {
after?: string;
- /**
- * Restricts the pass to enrollments touched at or after this moment.
- *
- * Deliberately not the scheduled mode. Neither ProgramEnrollment.updatedAt
- * nor Partner.updatedAt is indexed, so this filter is a scan and belongs to
- * one-shot use — most usefully the catch-up after a long backfill, which has
- * to re-index whatever changed while it was running.
- *
- * It also cannot see everything. ProgramPartnerTag carries no timestamps at
- * all, and a link edit moves neither of these columns, so a watermark pass is
- * strictly narrower than the full pass below.
- */
- since?: Date;
batchSize?: number;
timeBudgetMs?: number;
now?: () => number;
searchProvider?: PartnerSearchProvider | null;
}
-function buildSweepWhere(
- since?: Date,
-): Prisma.ProgramEnrollmentWhereInput | undefined {
- if (!since) {
- return undefined;
- }
-
- return {
- OR: [{ updatedAt: { gte: since } }, { partner: { updatedAt: { gte: since } } }],
- };
-}
-
/**
* Re-indexes a slice of the corpus, resuming from `after`.
*
* This is the backstop for call-site drift. Every hook that fires is an
* optimization on top of it: a mutation path that forgets to queue a sync, or
* one added later by someone who does not know the index exists, is corrected
- * by the next full pass rather than staying wrong until someone notices.
+ * by the next pass rather than staying wrong until someone notices.
*
- * Runs unfiltered by default, re-indexing everything rather than only what a
- * timestamp says changed. That costs writes the watermark version would skip,
- * and buys the one guarantee the watermark cannot give: no document is stale
- * for longer than a full pass, whatever drifted and whether or not the column
- * that drifted has a timestamp. Tags have none, which is what makes this the
- * only mechanism that covers them.
+ * Re-indexes everything rather than only what a timestamp says changed. A
+ * watermark would skip writes, but it cannot see a tag change, because
+ * ProgramPartnerTag carries no timestamps, nor a link edit, because that moves
+ * neither the enrollment's nor the partner's. Passing over everything is what
+ * makes the guarantee unconditional: no document is stale for longer than one
+ * pass, whatever drifted and whether or not the column that drifted has a
+ * timestamp.
*
* Deletions stay outside this. A pass only sees rows that still exist, so an
* orphaned document is invisible to it and is cleared by its call-site hook or
@@ -73,7 +48,6 @@ function buildSweepWhere(
*/
export async function sweepPartnerSearch({
after,
- since,
batchSize = PARTNER_SEARCH_SWEEP_BATCH_SIZE,
timeBudgetMs = PARTNER_SEARCH_SWEEP_TIME_BUDGET_MS,
now,
@@ -93,7 +67,6 @@ export async function sweepPartnerSearch({
return await indexPartnerSearchEnrollments({
searchProvider,
- where: buildSweepWhere(since),
after,
batchSize,
timeBudgetMs,
diff --git a/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts b/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts
index 53db867c86..a3b5fc04ec 100644
--- a/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts
+++ b/apps/web/lib/jobs/handlers/partner-search-sweep-job.ts
@@ -15,8 +15,6 @@ const inputSchema = z.object({
* at every call site.
*/
processed: z.number().int().nonnegative().optional(),
- /** One-shot watermark mode. See sweepPartnerSearch. */
- since: z.iso.datetime().optional(),
});
export const partnerSearchSweepJob = defineJob({
@@ -31,7 +29,7 @@ export const partnerSearchSweepJob = defineJob({
parallelism: 1,
},
},
- async handle({ after, processed: alreadyProcessed = 0, since }) {
+ async handle({ after, processed: alreadyProcessed = 0 }) {
if (!getPartnerSearchProvider()) {
console.log(
"[partnerSearchSweepJob] No search provider configured. Skipping...",
@@ -41,7 +39,6 @@ export const partnerSearchSweepJob = defineJob({
const { processed, lastDocumentId, done } = await sweepPartnerSearch({
after,
- ...(since && { since: new Date(since) }),
});
const totalProcessed = alreadyProcessed + processed;
@@ -63,7 +60,6 @@ export const partnerSearchSweepJob = defineJob({
{
...(lastDocumentId && { after: lastDocumentId }),
processed: totalProcessed,
- ...(since && { since }),
},
{
delay: 1,
diff --git a/apps/web/tests/partners/partner-search-sweep-job.test.ts b/apps/web/tests/partners/partner-search-sweep-job.test.ts
index 8483d96470..25f43ac1ef 100644
--- a/apps/web/tests/partners/partner-search-sweep-job.test.ts
+++ b/apps/web/tests/partners/partner-search-sweep-job.test.ts
@@ -68,33 +68,4 @@ describe("partnerSearchSweepJob", () => {
);
});
- it("keeps the watermark across hops so a catch-up stays narrowed", async () => {
- const dispatch = vi
- .spyOn(partnerSearchSweepJob, "dispatch")
- .mockResolvedValue({ status: "published", messageId: "msg_1" });
-
- const since = "2026-08-01T00:00:00.000Z";
- mocks.sweepPartnerSearch.mockResolvedValue({
- processed: 20,
- lastDocumentId: "pge_20",
- done: false,
- });
-
- await partnerSearchSweepJob.execute({ since });
-
- expect(mocks.sweepPartnerSearch).toHaveBeenCalledWith({
- after: undefined,
- since: new Date(since),
- });
- expect(dispatch).toHaveBeenCalledWith(
- { after: "pge_20", processed: 20, since },
- { delay: 1 },
- );
- });
-
- it("rejects a malformed watermark rather than sweeping everything", async () => {
- await expect(
- partnerSearchSweepJob.execute({ since: "last tuesday" }),
- ).rejects.toThrow();
- });
});
diff --git a/apps/web/tests/partners/partner-search-sweep.test.ts b/apps/web/tests/partners/partner-search-sweep.test.ts
index d3304bfcef..5607c3c658 100644
--- a/apps/web/tests/partners/partner-search-sweep.test.ts
+++ b/apps/web/tests/partners/partner-search-sweep.test.ts
@@ -159,42 +159,6 @@ describe("sweepPartnerSearch", () => {
});
});
- it("narrows to enrollment and partner timestamps in watermark mode", async () => {
- const searchProvider = createProvider();
- const since = new Date("2026-08-01T00:00:00.000Z");
- mocks.findMany.mockResolvedValueOnce([]);
-
- await sweepPartnerSearch({ since, batchSize: 2, searchProvider });
-
- expect(mocks.findMany.mock.calls[0][0].where).toEqual({
- OR: [
- { updatedAt: { gte: since } },
- { partner: { updatedAt: { gte: since } } },
- ],
- });
- });
-
- it("keeps the watermark filter alongside the cursor when resuming", async () => {
- const searchProvider = createProvider();
- const since = new Date("2026-08-01T00:00:00.000Z");
- mocks.findMany.mockResolvedValueOnce([]);
-
- await sweepPartnerSearch({
- since,
- after: "pge_5",
- batchSize: 2,
- searchProvider,
- });
-
- expect(mocks.findMany.mock.calls[0][0].where).toEqual({
- OR: [
- { updatedAt: { gte: since } },
- { partner: { updatedAt: { gte: since } } },
- ],
- id: { gt: "pge_5" },
- });
- });
-
it("never deletes, because an out-of-range id looks the same as a missing one", async () => {
const searchProvider = createProvider();
mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);