The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Add the partner search sync job
Rafi711a8bf4f1c1
diff --git a/apps/web/lib/jobs/handlers/partner-search-sync-job.ts b/apps/web/lib/jobs/handlers/partner-search-sync-job.tsnew file mode 100644index 0000000000..80c9f2347e--- /dev/null+++ b/apps/web/lib/jobs/handlers/partner-search-sync-job.ts@@ -0,0 +1,118 @@+import {+ findPartnerSearchSyncEnrollmentIds,+ getPartnerSearchProvider,+ PARTNER_SEARCH_SYNC_BATCH_SIZE,+ syncPartnerSearchDocuments,+} from "@/lib/api/partners/search";+import * as z from "zod/v4";+import { defineJob } from "../index";++/**+ * How many concurrent syncs may aim writes at the provider at once. A bulk+ * group move fans out to dozens of jobs, and without a ceiling they would all+ * write at the same moment. Interactive single-partner edits queue behind that+ * work, which is the tradeoff: a search index a few seconds stale is cheaper+ * than a rate-limited provider. Tune against the provider's write limits.+ */+const SYNC_PARALLELISM = 20;++/**+ * Two shapes, because the two kinds of change have different blast radii.+ *+ * `enrollments` carries IDs the caller already knows, and is the only shape+ * that can express a deletion — the handler discovers an enrollment is gone by+ * failing to read it back.+ *+ * `partners` is for changes that fan out beyond one enrollment (a profile edit,+ * a platform change), where the caller knows the partner but not how many+ * programs it reaches.+ */+const inputSchema = z.discriminatedUnion("type", [+ z.object({+ type: z.literal("enrollments"),+ enrollmentIds: z+ .array(z.string())+ .min(1)+ .max(PARTNER_SEARCH_SYNC_BATCH_SIZE),+ }),+ z.object({+ type: z.literal("partners"),+ partnerIds: z.array(z.string()).min(1).max(PARTNER_SEARCH_SYNC_BATCH_SIZE),+ programId: z.string().optional(),+ after: z.string().optional(),+ }),+]);++export const partnerSearchSyncJob = defineJob({+ name: "partner-search-sync-job",+ schema: inputSchema,+ defaults: {+ retries: 3,+ flowControl: {+ key: "partner-search-sync",+ parallelism: SYNC_PARALLELISM,+ },+ },+ async handle(input) {+ const searchProvider = getPartnerSearchProvider();++ // Matches the read path, which falls back to the database search rather+ // than failing. An environment without the key should not accumulate a+ // backlog of jobs that can never succeed.+ if (!searchProvider) {+ console.log(+ "[partnerSearchSyncJob] No search provider configured. Skipping...",+ );+ return;+ }++ if (input.type === "enrollments") {+ const { upserted, deleted } = await syncPartnerSearchDocuments({+ enrollmentIds: input.enrollmentIds,+ searchProvider,+ });++ console.log(+ `[partnerSearchSyncJob] Synced ${upserted} and removed ${deleted} enrollment documents.`,+ );++ return;+ }++ const enrollmentIds = await findPartnerSearchSyncEnrollmentIds({+ partnerIds: input.partnerIds,+ programId: input.programId,+ after: input.after,+ take: PARTNER_SEARCH_SYNC_BATCH_SIZE,+ });++ if (enrollmentIds.length === 0) {+ return;+ }++ // These IDs came from the database a moment ago, so the sync will upsert+ // rather than delete them — unless the enrollment disappeared in between,+ // in which case removing it from the index is the right outcome anyway.+ const { upserted, deleted } = await syncPartnerSearchDocuments({+ enrollmentIds,+ searchProvider,+ });++ console.log(+ `[partnerSearchSyncJob] Synced ${upserted} and removed ${deleted} enrollment documents for ${input.partnerIds.length} partner(s).`,+ );++ // A full page means there may be more enrollments for these partners.+ if (enrollmentIds.length === PARTNER_SEARCH_SYNC_BATCH_SIZE) {+ await partnerSearchSyncJob.dispatch(+ {+ ...input,+ after: enrollmentIds[enrollmentIds.length - 1],+ },+ {+ delay: 1,+ },+ );+ }+ },+});diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.tsindex a93e0a52d1..078257b5c9 100644--- a/apps/web/lib/jobs/registry.ts+++ b/apps/web/lib/jobs/registry.ts@@ -29,6 +29,11 @@ const jobLoaders = {import("./handlers/create-tremendous-campaign-job").then((m) => m.createTremendousCampaignJob,),++ "partner-search-sync-job": () =>+ import("./handlers/partner-search-sync-job").then(+ (m) => m.partnerSearchSyncJob,+ ),} as const satisfies Record<string, () => Promise<JobDefinition>>;const jobCache = new Map<string, JobDefinition>();diff --git a/apps/web/tests/partners/partner-search-sync-job.test.ts b/apps/web/tests/partners/partner-search-sync-job.test.tsnew file mode 100644index 0000000000..89062a1266--- /dev/null+++ b/apps/web/tests/partners/partner-search-sync-job.test.ts@@ -0,0 +1,166 @@+import { partnerSearchSyncJob } from "@/lib/jobs/handlers/partner-search-sync-job";+import { beforeEach, describe, expect, it, vi } from "vitest";++// A small batch size keeps the pagination cases readable.+const BATCH_SIZE = 3;++const mocks = vi.hoisted(() => ({+ getPartnerSearchProvider: vi.fn(),+ syncPartnerSearchDocuments: vi.fn(),+ findPartnerSearchSyncEnrollmentIds: vi.fn(),+}));++vi.mock("@/lib/api/partners/search", () => ({+ PARTNER_SEARCH_SYNC_BATCH_SIZE: 3,+ getPartnerSearchProvider: mocks.getPartnerSearchProvider,+ syncPartnerSearchDocuments: mocks.syncPartnerSearchDocuments,+ findPartnerSearchSyncEnrollmentIds: mocks.findPartnerSearchSyncEnrollmentIds,+}));++const searchProvider = { name: "turbopuffer" };++describe("partnerSearchSyncJob", () => {+ beforeEach(() => {+ vi.restoreAllMocks();+ mocks.getPartnerSearchProvider.mockReset().mockReturnValue(searchProvider);+ mocks.syncPartnerSearchDocuments+ .mockReset()+ .mockResolvedValue({ upserted: 0, deleted: 0 });+ mocks.findPartnerSearchSyncEnrollmentIds.mockReset().mockResolvedValue([]);+ });++ it("skips entirely when no provider is configured", async () => {+ mocks.getPartnerSearchProvider.mockReturnValue(null);++ await partnerSearchSyncJob.execute({+ type: "enrollments",+ enrollmentIds: ["pge_1"],+ });++ expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();+ expect(mocks.findPartnerSearchSyncEnrollmentIds).not.toHaveBeenCalled();+ });++ it("syncs the enrollment ids it is given", async () => {+ await partnerSearchSyncJob.execute({+ type: "enrollments",+ enrollmentIds: ["pge_1", "pge_2"],+ });++ expect(mocks.syncPartnerSearchDocuments).toHaveBeenCalledWith({+ enrollmentIds: ["pge_1", "pge_2"],+ searchProvider,+ });+ expect(mocks.findPartnerSearchSyncEnrollmentIds).not.toHaveBeenCalled();+ });++ it("resolves a partner fan-out before syncing", async () => {+ mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([+ "pge_1",+ "pge_2",+ ]);++ await partnerSearchSyncJob.execute({+ type: "partners",+ partnerIds: ["pn_1"],+ });++ expect(mocks.findPartnerSearchSyncEnrollmentIds).toHaveBeenCalledWith({+ partnerIds: ["pn_1"],+ programId: undefined,+ after: undefined,+ take: BATCH_SIZE,+ });+ expect(mocks.syncPartnerSearchDocuments).toHaveBeenCalledWith({+ enrollmentIds: ["pge_1", "pge_2"],+ searchProvider,+ });+ });++ it("continues from the last enrollment when a page comes back full", async () => {+ const dispatch = vi+ .spyOn(partnerSearchSyncJob, "dispatch")+ .mockResolvedValue({ status: "published", messageId: "msg_1" });++ mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([+ "pge_1",+ "pge_2",+ "pge_3",+ ]);++ await partnerSearchSyncJob.execute({+ type: "partners",+ partnerIds: ["pn_1"],+ programId: "prog_1",+ });++ expect(dispatch).toHaveBeenCalledWith(+ {+ type: "partners",+ partnerIds: ["pn_1"],+ programId: "prog_1",+ after: "pge_3",+ },+ { delay: 1 },+ );+ });++ it("stops when a page comes back short", async () => {+ const dispatch = vi+ .spyOn(partnerSearchSyncJob, "dispatch")+ .mockResolvedValue({ status: "published", messageId: "msg_1" });++ mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([+ "pge_1",+ "pge_2",+ ]);++ await partnerSearchSyncJob.execute({+ type: "partners",+ partnerIds: ["pn_1"],+ });++ expect(dispatch).not.toHaveBeenCalled();+ });++ it("does not sync or continue when a partner has no enrollments", async () => {+ const dispatch = vi+ .spyOn(partnerSearchSyncJob, "dispatch")+ .mockResolvedValue({ status: "published", messageId: "msg_1" });++ mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([]);++ await partnerSearchSyncJob.execute({+ type: "partners",+ partnerIds: ["pn_1"],+ });++ expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();+ expect(dispatch).not.toHaveBeenCalled();+ });++ it("rejects a payload with no ids, so an empty fan-out cannot be queued", async () => {+ await expect(+ partnerSearchSyncJob.execute({ type: "enrollments", enrollmentIds: [] }),+ ).rejects.toThrow();++ await expect(+ partnerSearchSyncJob.execute({ type: "partners", partnerIds: [] }),+ ).rejects.toThrow();+ });++ it("rejects a payload larger than one batch", async () => {+ await expect(+ partnerSearchSyncJob.execute({+ type: "enrollments",+ enrollmentIds: ["pge_1", "pge_2", "pge_3", "pge_4"],+ }),+ ).rejects.toThrow();+ });++ it("rejects an unknown payload shape", async () => {+ await expect(+ partnerSearchSyncJob.execute({ enrollmentIds: ["pge_1"] }),+ ).rejects.toThrow();+ });+});