Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Add the partner search sync job

Rafi711a8bf4f1c1
diff --git a/apps/web/lib/jobs/handlers/partner-search-sync-job.ts b/apps/web/lib/jobs/handlers/partner-search-sync-job.ts
new file mode 100644
index 0000000000..80c9f2347e
--- /dev/null
+++ b/apps/web/lib/jobs/handlers/partner-search-sync-job.ts
@@ -0,0 +1,118 @@
+import {
+ findPartnerSearchSyncEnrollmentIds,
+ getPartnerSearchProvider,
+ PARTNER_SEARCH_SYNC_BATCH_SIZE,
+ syncPartnerSearchDocuments,
+} from "@/lib/api/partners/search";
+import * as z from "zod/v4";
+import { defineJob } from "../index";
+
+/**
+ * How many concurrent syncs may aim writes at the provider at once. A bulk
+ * group move fans out to dozens of jobs, and without a ceiling they would all
+ * write at the same moment. Interactive single-partner edits queue behind that
+ * work, which is the tradeoff: a search index a few seconds stale is cheaper
+ * than a rate-limited provider. Tune against the provider's write limits.
+ */
+const SYNC_PARALLELISM = 20;
+
+/**
+ * Two shapes, because the two kinds of change have different blast radii.
+ *
+ * `enrollments` carries IDs the caller already knows, and is the only shape
+ * that can express a deletion — the handler discovers an enrollment is gone by
+ * failing to read it back.
+ *
+ * `partners` is for changes that fan out beyond one enrollment (a profile edit,
+ * a platform change), where the caller knows the partner but not how many
+ * programs it reaches.
+ */
+const inputSchema = z.discriminatedUnion("type", [
+ z.object({
+ type: z.literal("enrollments"),
+ enrollmentIds: z
+ .array(z.string())
+ .min(1)
+ .max(PARTNER_SEARCH_SYNC_BATCH_SIZE),
+ }),
+ z.object({
+ type: z.literal("partners"),
+ partnerIds: z.array(z.string()).min(1).max(PARTNER_SEARCH_SYNC_BATCH_SIZE),
+ programId: z.string().optional(),
+ after: z.string().optional(),
+ }),
+]);
+
+export const partnerSearchSyncJob = defineJob({
+ name: "partner-search-sync-job",
+ schema: inputSchema,
+ defaults: {
+ retries: 3,
+ flowControl: {
+ key: "partner-search-sync",
+ parallelism: SYNC_PARALLELISM,
+ },
+ },
+ async handle(input) {
+ const searchProvider = getPartnerSearchProvider();
+
+ // Matches the read path, which falls back to the database search rather
+ // than failing. An environment without the key should not accumulate a
+ // backlog of jobs that can never succeed.
+ if (!searchProvider) {
+ console.log(
+ "[partnerSearchSyncJob] No search provider configured. Skipping...",
+ );
+ return;
+ }
+
+ if (input.type === "enrollments") {
+ const { upserted, deleted } = await syncPartnerSearchDocuments({
+ enrollmentIds: input.enrollmentIds,
+ searchProvider,
+ });
+
+ console.log(
+ `[partnerSearchSyncJob] Synced ${upserted} and removed ${deleted} enrollment documents.`,
+ );
+
+ return;
+ }
+
+ const enrollmentIds = await findPartnerSearchSyncEnrollmentIds({
+ partnerIds: input.partnerIds,
+ programId: input.programId,
+ after: input.after,
+ take: PARTNER_SEARCH_SYNC_BATCH_SIZE,
+ });
+
+ if (enrollmentIds.length === 0) {
+ return;
+ }
+
+ // These IDs came from the database a moment ago, so the sync will upsert
+ // rather than delete them — unless the enrollment disappeared in between,
+ // in which case removing it from the index is the right outcome anyway.
+ const { upserted, deleted } = await syncPartnerSearchDocuments({
+ enrollmentIds,
+ searchProvider,
+ });
+
+ console.log(
+ `[partnerSearchSyncJob] Synced ${upserted} and removed ${deleted} enrollment documents for ${input.partnerIds.length} partner(s).`,
+ );
+
+ // A full page means there may be more enrollments for these partners.
+ if (enrollmentIds.length === PARTNER_SEARCH_SYNC_BATCH_SIZE) {
+ await partnerSearchSyncJob.dispatch(
+ {
+ ...input,
+ after: enrollmentIds[enrollmentIds.length - 1],
+ },
+ {
+ delay: 1,
+ },
+ );
+ }
+ },
+});
diff --git a/apps/web/lib/jobs/registry.ts b/apps/web/lib/jobs/registry.ts
index a93e0a52d1..078257b5c9 100644
--- a/apps/web/lib/jobs/registry.ts
+++ b/apps/web/lib/jobs/registry.ts
@@ -29,6 +29,11 @@ const jobLoaders = {
import("./handlers/create-tremendous-campaign-job").then(
(m) => m.createTremendousCampaignJob,
),
+
+ "partner-search-sync-job": () =>
+ import("./handlers/partner-search-sync-job").then(
+ (m) => m.partnerSearchSyncJob,
+ ),
} as const satisfies Record<string, () => Promise<JobDefinition>>;
const jobCache = new Map<string, JobDefinition>();
diff --git a/apps/web/tests/partners/partner-search-sync-job.test.ts b/apps/web/tests/partners/partner-search-sync-job.test.ts
new file mode 100644
index 0000000000..89062a1266
--- /dev/null
+++ b/apps/web/tests/partners/partner-search-sync-job.test.ts
@@ -0,0 +1,166 @@
+import { partnerSearchSyncJob } from "@/lib/jobs/handlers/partner-search-sync-job";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+// A small batch size keeps the pagination cases readable.
+const BATCH_SIZE = 3;
+
+const mocks = vi.hoisted(() => ({
+ getPartnerSearchProvider: vi.fn(),
+ syncPartnerSearchDocuments: vi.fn(),
+ findPartnerSearchSyncEnrollmentIds: vi.fn(),
+}));
+
+vi.mock("@/lib/api/partners/search", () => ({
+ PARTNER_SEARCH_SYNC_BATCH_SIZE: 3,
+ getPartnerSearchProvider: mocks.getPartnerSearchProvider,
+ syncPartnerSearchDocuments: mocks.syncPartnerSearchDocuments,
+ findPartnerSearchSyncEnrollmentIds: mocks.findPartnerSearchSyncEnrollmentIds,
+}));
+
+const searchProvider = { name: "turbopuffer" };
+
+describe("partnerSearchSyncJob", () => {
+ beforeEach(() => {
+ vi.restoreAllMocks();
+ mocks.getPartnerSearchProvider.mockReset().mockReturnValue(searchProvider);
+ mocks.syncPartnerSearchDocuments
+ .mockReset()
+ .mockResolvedValue({ upserted: 0, deleted: 0 });
+ mocks.findPartnerSearchSyncEnrollmentIds.mockReset().mockResolvedValue([]);
+ });
+
+ it("skips entirely when no provider is configured", async () => {
+ mocks.getPartnerSearchProvider.mockReturnValue(null);
+
+ await partnerSearchSyncJob.execute({
+ type: "enrollments",
+ enrollmentIds: ["pge_1"],
+ });
+
+ expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();
+ expect(mocks.findPartnerSearchSyncEnrollmentIds).not.toHaveBeenCalled();
+ });
+
+ it("syncs the enrollment ids it is given", async () => {
+ await partnerSearchSyncJob.execute({
+ type: "enrollments",
+ enrollmentIds: ["pge_1", "pge_2"],
+ });
+
+ expect(mocks.syncPartnerSearchDocuments).toHaveBeenCalledWith({
+ enrollmentIds: ["pge_1", "pge_2"],
+ searchProvider,
+ });
+ expect(mocks.findPartnerSearchSyncEnrollmentIds).not.toHaveBeenCalled();
+ });
+
+ it("resolves a partner fan-out before syncing", async () => {
+ mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([
+ "pge_1",
+ "pge_2",
+ ]);
+
+ await partnerSearchSyncJob.execute({
+ type: "partners",
+ partnerIds: ["pn_1"],
+ });
+
+ expect(mocks.findPartnerSearchSyncEnrollmentIds).toHaveBeenCalledWith({
+ partnerIds: ["pn_1"],
+ programId: undefined,
+ after: undefined,
+ take: BATCH_SIZE,
+ });
+ expect(mocks.syncPartnerSearchDocuments).toHaveBeenCalledWith({
+ enrollmentIds: ["pge_1", "pge_2"],
+ searchProvider,
+ });
+ });
+
+ it("continues from the last enrollment when a page comes back full", async () => {
+ const dispatch = vi
+ .spyOn(partnerSearchSyncJob, "dispatch")
+ .mockResolvedValue({ status: "published", messageId: "msg_1" });
+
+ mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([
+ "pge_1",
+ "pge_2",
+ "pge_3",
+ ]);
+
+ await partnerSearchSyncJob.execute({
+ type: "partners",
+ partnerIds: ["pn_1"],
+ programId: "prog_1",
+ });
+
+ expect(dispatch).toHaveBeenCalledWith(
+ {
+ type: "partners",
+ partnerIds: ["pn_1"],
+ programId: "prog_1",
+ after: "pge_3",
+ },
+ { delay: 1 },
+ );
+ });
+
+ it("stops when a page comes back short", async () => {
+ const dispatch = vi
+ .spyOn(partnerSearchSyncJob, "dispatch")
+ .mockResolvedValue({ status: "published", messageId: "msg_1" });
+
+ mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([
+ "pge_1",
+ "pge_2",
+ ]);
+
+ await partnerSearchSyncJob.execute({
+ type: "partners",
+ partnerIds: ["pn_1"],
+ });
+
+ expect(dispatch).not.toHaveBeenCalled();
+ });
+
+ it("does not sync or continue when a partner has no enrollments", async () => {
+ const dispatch = vi
+ .spyOn(partnerSearchSyncJob, "dispatch")
+ .mockResolvedValue({ status: "published", messageId: "msg_1" });
+
+ mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([]);
+
+ await partnerSearchSyncJob.execute({
+ type: "partners",
+ partnerIds: ["pn_1"],
+ });
+
+ expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();
+ expect(dispatch).not.toHaveBeenCalled();
+ });
+
+ it("rejects a payload with no ids, so an empty fan-out cannot be queued", async () => {
+ await expect(
+ partnerSearchSyncJob.execute({ type: "enrollments", enrollmentIds: [] }),
+ ).rejects.toThrow();
+
+ await expect(
+ partnerSearchSyncJob.execute({ type: "partners", partnerIds: [] }),
+ ).rejects.toThrow();
+ });
+
+ it("rejects a payload larger than one batch", async () => {
+ await expect(
+ partnerSearchSyncJob.execute({
+ type: "enrollments",
+ enrollmentIds: ["pge_1", "pge_2", "pge_3", "pge_4"],
+ }),
+ ).rejects.toThrow();
+ });
+
+ it("rejects an unknown payload shape", async () => {
+ await expect(
+ partnerSearchSyncJob.execute({ enrollmentIds: ["pge_1"] }),
+ ).rejects.toThrow();
+ });
+});