The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Add the partner search sync dispatcher
Rafi05df6d8eb5b8
diff --git a/apps/web/lib/api/partners/queue-partner-search-sync.ts b/apps/web/lib/api/partners/queue-partner-search-sync.tsnew file mode 100644index 0000000000..114a98ca70--- /dev/null+++ b/apps/web/lib/api/partners/queue-partner-search-sync.ts@@ -0,0 +1,137 @@+import { partnerSearchSyncJob } from "@/lib/jobs/handlers/partner-search-sync-job";+import { chunk } from "@dub/utils";+import {+ getPartnerSearchProvider,+ PARTNER_SEARCH_SYNC_BATCH_SIZE,+} from "./search";++/**+ * How long a sync waits before it runs. The delay is what gives deduplication+ * something to collapse: an admin editing one partner three times in a row+ * should produce one write, not three.+ */+export const PARTNER_SEARCH_SYNC_DELAY_SECONDS = 5;++/**+ * Links change far more often than the other document sources, and a link edit+ * only moves `shortLinks` and `destinationUrls`, so they are worth batching+ * harder. Nothing is lost by waiting: the handler re-reads the whole document,+ * so a link sync that runs ten minutes late still picks up every other change+ * that landed in the meantime.+ *+ * Link *creation* should use the default delay instead — a partner's short link+ * is how people search for them, so a newly enrolled partner being unfindable+ * by link for ten minutes is a worse trade than the extra write.+ */+export const PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS = 600;++type PartnerSearchSyncPayload =+ | { type: "enrollments"; enrollmentIds: string[] }+ | { type: "partners"; partnerIds: string[]; programId?: string };++interface QueuePartnerSearchSyncInput {+ /**+ * Enrollments the caller already has IDs for. The only shape that can express+ * a deletion, since the job removes what it cannot read back.+ */+ enrollmentIds?: string[];+ /**+ * Partners whose change fans out to every enrollment they hold, unless+ * `programId` narrows it to one.+ */+ partnerIds?: string[];+ programId?: string;+ delay?: number;+}++function unique(values: string[] | undefined): string[] {+ return Array.from(new Set((values ?? []).filter(Boolean)));+}++/**+ * Collapses repeat syncs of the same subject while one is still pending.+ *+ * Only for single-subject payloads: bulk operations produce chunks whose exact+ * composition never repeats, so a key built from them would never match, and+ * they are not the traffic worth collapsing anyway. Repeated edits to one+ * partner are.+ *+ * The delay is part of the key so a slow link sync cannot suppress a fast one+ * queued behind it, which would drag an interactive edit out to the link delay.+ * The cost is at most one redundant write, and the job is idempotent.+ *+ * If the delay ever exceeds QStash's deduplication retention the collapse stops+ * happening and every edit writes, which is wasteful but still correct.+ */+function buildDeduplicationId(+ payload: PartnerSearchSyncPayload,+ delay: number,+): string | undefined {+ if (payload.type === "enrollments") {+ return payload.enrollmentIds.length === 1+ ? `enrollment:${payload.enrollmentIds[0]}:${delay}`+ : undefined;+ }++ return payload.partnerIds.length === 1+ ? `partner:${payload.partnerIds[0]}:${payload.programId ?? "all"}:${delay}`+ : undefined;+}++/**+ * Queues an index sync for whatever the caller just changed.+ *+ * Never throws. A search index that falls behind costs relevance ranking and+ * the wider field coverage, which the read path already degrades to when no+ * provider is configured; a partner mutation failing because a queue was+ * unreachable would be a far worse outcome. Job dispatch already persists to+ * the database when QStash is unavailable, so reaching the catch here means+ * both paths failed, and the reconciliation sweep is what recovers from it.+ */+export async function queuePartnerSearchSync({+ enrollmentIds,+ partnerIds,+ programId,+ delay = PARTNER_SEARCH_SYNC_DELAY_SECONDS,+}: QueuePartnerSearchSyncInput) {+ // Checked here as well as in the handler so an unconfigured environment does+ // not pay for queueing work that would immediately no-op.+ if (!getPartnerSearchProvider()) {+ return;+ }++ const enrollments = unique(enrollmentIds);+ const partners = unique(partnerIds);++ if (enrollments.length === 0 && partners.length === 0) {+ return;+ }++ const payloads: PartnerSearchSyncPayload[] = [+ ...chunk(enrollments, PARTNER_SEARCH_SYNC_BATCH_SIZE).map((ids) => ({+ type: "enrollments" as const,+ enrollmentIds: ids,+ })),+ ...chunk(partners, PARTNER_SEARCH_SYNC_BATCH_SIZE).map((ids) => ({+ type: "partners" as const,+ partnerIds: ids,+ ...(programId && { programId }),+ })),+ ];++ try {+ await partnerSearchSyncJob.dispatchBatch(payloads, (payload) => {+ const deduplicationId = buildDeduplicationId(payload, delay);++ return {+ delay,+ ...(deduplicationId && { deduplicationId }),+ };+ });+ } catch (error) {+ console.error(+ "[Partner Search] Failed to queue an index sync. The reconciliation sweep will pick this up.",+ error,+ );+ }+}diff --git a/apps/web/tests/partners/queue-partner-search-sync.test.ts b/apps/web/tests/partners/queue-partner-search-sync.test.tsnew file mode 100644index 0000000000..316a82eab1--- /dev/null+++ b/apps/web/tests/partners/queue-partner-search-sync.test.ts@@ -0,0 +1,165 @@+import {+ PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS,+ PARTNER_SEARCH_SYNC_DELAY_SECONDS,+ queuePartnerSearchSync,+} from "@/lib/api/partners/queue-partner-search-sync";+import { beforeEach, describe, expect, it, vi } from "vitest";++const BATCH_SIZE = 3;++const mocks = vi.hoisted(() => ({+ getPartnerSearchProvider: vi.fn(),+ dispatchBatch: vi.fn(),+}));++vi.mock("@/lib/api/partners/search", () => ({+ PARTNER_SEARCH_SYNC_BATCH_SIZE: 3,+ getPartnerSearchProvider: mocks.getPartnerSearchProvider,+}));++vi.mock("@/lib/jobs/handlers/partner-search-sync-job", () => ({+ partnerSearchSyncJob: {+ dispatchBatch: mocks.dispatchBatch,+ },+}));++/** The options the dispatcher computed for each payload it queued. */+function dispatchedWithOptions() {+ const [payloads, getOptions] = mocks.dispatchBatch.mock.calls[0];++ return payloads.map((payload: unknown, index: number) => ({+ payload,+ options: getOptions?.(payload, index),+ }));+}++describe("queuePartnerSearchSync", () => {+ beforeEach(() => {+ mocks.getPartnerSearchProvider+ .mockReset()+ .mockReturnValue({ name: "turbopuffer" });+ mocks.dispatchBatch.mockReset().mockResolvedValue({+ published: 1,+ deferred: 0,+ failed: 0,+ results: [],+ });+ });++ it("queues nothing when no provider is configured", async () => {+ mocks.getPartnerSearchProvider.mockReturnValue(null);++ await queuePartnerSearchSync({ enrollmentIds: ["pge_1"] });++ expect(mocks.dispatchBatch).not.toHaveBeenCalled();+ });++ it("queues nothing when there is nothing to sync", async () => {+ await queuePartnerSearchSync({ enrollmentIds: [], partnerIds: [] });++ expect(mocks.dispatchBatch).not.toHaveBeenCalled();+ });++ it("deduplicates ids before queueing", async () => {+ await queuePartnerSearchSync({ enrollmentIds: ["pge_1", "pge_1"] });++ expect(mocks.dispatchBatch.mock.calls[0][0]).toEqual([+ { type: "enrollments", enrollmentIds: ["pge_1"] },+ ]);+ });++ it("collapses repeat syncs of a single enrollment", async () => {+ await queuePartnerSearchSync({ enrollmentIds: ["pge_1"] });++ const [{ options }] = dispatchedWithOptions();++ expect(options).toEqual({+ delay: PARTNER_SEARCH_SYNC_DELAY_SECONDS,+ deduplicationId: `enrollment:pge_1:${PARTNER_SEARCH_SYNC_DELAY_SECONDS}`,+ });+ });++ it("scopes a single partner's dedup key by program", async () => {+ await queuePartnerSearchSync({+ partnerIds: ["pn_1"],+ programId: "prog_1",+ });++ const [{ options }] = dispatchedWithOptions();++ expect(options.deduplicationId).toBe(+ `partner:pn_1:prog_1:${PARTNER_SEARCH_SYNC_DELAY_SECONDS}`,+ );+ });++ it("does not deduplicate bulk chunks, whose composition never repeats", async () => {+ await queuePartnerSearchSync({ enrollmentIds: ["pge_1", "pge_2"] });++ const [{ options }] = dispatchedWithOptions();++ expect(options.deduplicationId).toBeUndefined();+ });++ it("keeps a slow link sync from suppressing a fast one", async () => {+ await queuePartnerSearchSync({+ enrollmentIds: ["pge_1"],+ delay: PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS,+ });+ await queuePartnerSearchSync({ enrollmentIds: ["pge_1"] });++ const slow = mocks.dispatchBatch.mock.calls[0][1](+ { type: "enrollments", enrollmentIds: ["pge_1"] },+ 0,+ );+ const fast = mocks.dispatchBatch.mock.calls[1][1](+ { type: "enrollments", enrollmentIds: ["pge_1"] },+ 0,+ );++ expect(slow.delay).toBe(PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS);+ expect(fast.delay).toBe(PARTNER_SEARCH_SYNC_DELAY_SECONDS);+ expect(slow.deduplicationId).not.toBe(fast.deduplicationId);+ });++ it("chunks past the batch size so no payload exceeds what the job accepts", async () => {+ await queuePartnerSearchSync({+ enrollmentIds: ["pge_1", "pge_2", "pge_3", "pge_4"],+ });++ expect(mocks.dispatchBatch.mock.calls[0][0]).toEqual([+ { type: "enrollments", enrollmentIds: ["pge_1", "pge_2", "pge_3"] },+ { type: "enrollments", enrollmentIds: ["pge_4"] },+ ]);+ });++ it("queues both shapes when a caller has enrollments and partners", async () => {+ await queuePartnerSearchSync({+ enrollmentIds: ["pge_1"],+ partnerIds: ["pn_1"],+ programId: "prog_1",+ });++ expect(mocks.dispatchBatch.mock.calls[0][0]).toEqual([+ { type: "enrollments", enrollmentIds: ["pge_1"] },+ { type: "partners", partnerIds: ["pn_1"], programId: "prog_1" },+ ]);+ });++ it("swallows a dispatch failure so it cannot break the mutation that queued it", async () => {+ mocks.dispatchBatch.mockRejectedValue(new Error("qstash unreachable"));++ await expect(+ queuePartnerSearchSync({ enrollmentIds: ["pge_1"] }),+ ).resolves.toBeUndefined();+ });++ it("never builds a payload larger than the job's batch size", async () => {+ await queuePartnerSearchSync({+ partnerIds: Array.from({ length: 7 }, (_, index) => `pn_${index}`),+ });++ for (const payload of mocks.dispatchBatch.mock.calls[0][0]) {+ expect(payload.partnerIds.length).toBeLessThanOrEqual(BATCH_SIZE);+ }+ });+});