Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Add the partner search sync dispatcher

Rafi05df6d8eb5b8
diff --git a/apps/web/lib/api/partners/queue-partner-search-sync.ts b/apps/web/lib/api/partners/queue-partner-search-sync.ts
new file mode 100644
index 0000000000..114a98ca70
--- /dev/null
+++ b/apps/web/lib/api/partners/queue-partner-search-sync.ts
@@ -0,0 +1,137 @@
+import { partnerSearchSyncJob } from "@/lib/jobs/handlers/partner-search-sync-job";
+import { chunk } from "@dub/utils";
+import {
+ getPartnerSearchProvider,
+ PARTNER_SEARCH_SYNC_BATCH_SIZE,
+} from "./search";
+
+/**
+ * How long a sync waits before it runs. The delay is what gives deduplication
+ * something to collapse: an admin editing one partner three times in a row
+ * should produce one write, not three.
+ */
+export const PARTNER_SEARCH_SYNC_DELAY_SECONDS = 5;
+
+/**
+ * Links change far more often than the other document sources, and a link edit
+ * only moves `shortLinks` and `destinationUrls`, so they are worth batching
+ * harder. Nothing is lost by waiting: the handler re-reads the whole document,
+ * so a link sync that runs ten minutes late still picks up every other change
+ * that landed in the meantime.
+ *
+ * Link *creation* should use the default delay instead — a partner's short link
+ * is how people search for them, so a newly enrolled partner being unfindable
+ * by link for ten minutes is a worse trade than the extra write.
+ */
+export const PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS = 600;
+
+type PartnerSearchSyncPayload =
+ | { type: "enrollments"; enrollmentIds: string[] }
+ | { type: "partners"; partnerIds: string[]; programId?: string };
+
+interface QueuePartnerSearchSyncInput {
+ /**
+ * Enrollments the caller already has IDs for. The only shape that can express
+ * a deletion, since the job removes what it cannot read back.
+ */
+ enrollmentIds?: string[];
+ /**
+ * Partners whose change fans out to every enrollment they hold, unless
+ * `programId` narrows it to one.
+ */
+ partnerIds?: string[];
+ programId?: string;
+ delay?: number;
+}
+
+function unique(values: string[] | undefined): string[] {
+ return Array.from(new Set((values ?? []).filter(Boolean)));
+}
+
+/**
+ * Collapses repeat syncs of the same subject while one is still pending.
+ *
+ * Only for single-subject payloads: bulk operations produce chunks whose exact
+ * composition never repeats, so a key built from them would never match, and
+ * they are not the traffic worth collapsing anyway. Repeated edits to one
+ * partner are.
+ *
+ * The delay is part of the key so a slow link sync cannot suppress a fast one
+ * queued behind it, which would drag an interactive edit out to the link delay.
+ * The cost is at most one redundant write, and the job is idempotent.
+ *
+ * If the delay ever exceeds QStash's deduplication retention the collapse stops
+ * happening and every edit writes, which is wasteful but still correct.
+ */
+function buildDeduplicationId(
+ payload: PartnerSearchSyncPayload,
+ delay: number,
+): string | undefined {
+ if (payload.type === "enrollments") {
+ return payload.enrollmentIds.length === 1
+ ? `enrollment:${payload.enrollmentIds[0]}:${delay}`
+ : undefined;
+ }
+
+ return payload.partnerIds.length === 1
+ ? `partner:${payload.partnerIds[0]}:${payload.programId ?? "all"}:${delay}`
+ : undefined;
+}
+
+/**
+ * Queues an index sync for whatever the caller just changed.
+ *
+ * Never throws. A search index that falls behind costs relevance ranking and
+ * the wider field coverage, which the read path already degrades to when no
+ * provider is configured; a partner mutation failing because a queue was
+ * unreachable would be a far worse outcome. Job dispatch already persists to
+ * the database when QStash is unavailable, so reaching the catch here means
+ * both paths failed, and the reconciliation sweep is what recovers from it.
+ */
+export async function queuePartnerSearchSync({
+ enrollmentIds,
+ partnerIds,
+ programId,
+ delay = PARTNER_SEARCH_SYNC_DELAY_SECONDS,
+}: QueuePartnerSearchSyncInput) {
+ // Checked here as well as in the handler so an unconfigured environment does
+ // not pay for queueing work that would immediately no-op.
+ if (!getPartnerSearchProvider()) {
+ return;
+ }
+
+ const enrollments = unique(enrollmentIds);
+ const partners = unique(partnerIds);
+
+ if (enrollments.length === 0 && partners.length === 0) {
+ return;
+ }
+
+ const payloads: PartnerSearchSyncPayload[] = [
+ ...chunk(enrollments, PARTNER_SEARCH_SYNC_BATCH_SIZE).map((ids) => ({
+ type: "enrollments" as const,
+ enrollmentIds: ids,
+ })),
+ ...chunk(partners, PARTNER_SEARCH_SYNC_BATCH_SIZE).map((ids) => ({
+ type: "partners" as const,
+ partnerIds: ids,
+ ...(programId && { programId }),
+ })),
+ ];
+
+ try {
+ await partnerSearchSyncJob.dispatchBatch(payloads, (payload) => {
+ const deduplicationId = buildDeduplicationId(payload, delay);
+
+ return {
+ delay,
+ ...(deduplicationId && { deduplicationId }),
+ };
+ });
+ } catch (error) {
+ console.error(
+ "[Partner Search] Failed to queue an index sync. The reconciliation sweep will pick this up.",
+ error,
+ );
+ }
+}
diff --git a/apps/web/tests/partners/queue-partner-search-sync.test.ts b/apps/web/tests/partners/queue-partner-search-sync.test.ts
new file mode 100644
index 0000000000..316a82eab1
--- /dev/null
+++ b/apps/web/tests/partners/queue-partner-search-sync.test.ts
@@ -0,0 +1,165 @@
+import {
+ PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS,
+ PARTNER_SEARCH_SYNC_DELAY_SECONDS,
+ queuePartnerSearchSync,
+} from "@/lib/api/partners/queue-partner-search-sync";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const BATCH_SIZE = 3;
+
+const mocks = vi.hoisted(() => ({
+ getPartnerSearchProvider: vi.fn(),
+ dispatchBatch: vi.fn(),
+}));
+
+vi.mock("@/lib/api/partners/search", () => ({
+ PARTNER_SEARCH_SYNC_BATCH_SIZE: 3,
+ getPartnerSearchProvider: mocks.getPartnerSearchProvider,
+}));
+
+vi.mock("@/lib/jobs/handlers/partner-search-sync-job", () => ({
+ partnerSearchSyncJob: {
+ dispatchBatch: mocks.dispatchBatch,
+ },
+}));
+
+/** The options the dispatcher computed for each payload it queued. */
+function dispatchedWithOptions() {
+ const [payloads, getOptions] = mocks.dispatchBatch.mock.calls[0];
+
+ return payloads.map((payload: unknown, index: number) => ({
+ payload,
+ options: getOptions?.(payload, index),
+ }));
+}
+
+describe("queuePartnerSearchSync", () => {
+ beforeEach(() => {
+ mocks.getPartnerSearchProvider
+ .mockReset()
+ .mockReturnValue({ name: "turbopuffer" });
+ mocks.dispatchBatch.mockReset().mockResolvedValue({
+ published: 1,
+ deferred: 0,
+ failed: 0,
+ results: [],
+ });
+ });
+
+ it("queues nothing when no provider is configured", async () => {
+ mocks.getPartnerSearchProvider.mockReturnValue(null);
+
+ await queuePartnerSearchSync({ enrollmentIds: ["pge_1"] });
+
+ expect(mocks.dispatchBatch).not.toHaveBeenCalled();
+ });
+
+ it("queues nothing when there is nothing to sync", async () => {
+ await queuePartnerSearchSync({ enrollmentIds: [], partnerIds: [] });
+
+ expect(mocks.dispatchBatch).not.toHaveBeenCalled();
+ });
+
+ it("deduplicates ids before queueing", async () => {
+ await queuePartnerSearchSync({ enrollmentIds: ["pge_1", "pge_1"] });
+
+ expect(mocks.dispatchBatch.mock.calls[0][0]).toEqual([
+ { type: "enrollments", enrollmentIds: ["pge_1"] },
+ ]);
+ });
+
+ it("collapses repeat syncs of a single enrollment", async () => {
+ await queuePartnerSearchSync({ enrollmentIds: ["pge_1"] });
+
+ const [{ options }] = dispatchedWithOptions();
+
+ expect(options).toEqual({
+ delay: PARTNER_SEARCH_SYNC_DELAY_SECONDS,
+ deduplicationId: `enrollment:pge_1:${PARTNER_SEARCH_SYNC_DELAY_SECONDS}`,
+ });
+ });
+
+ it("scopes a single partner's dedup key by program", async () => {
+ await queuePartnerSearchSync({
+ partnerIds: ["pn_1"],
+ programId: "prog_1",
+ });
+
+ const [{ options }] = dispatchedWithOptions();
+
+ expect(options.deduplicationId).toBe(
+ `partner:pn_1:prog_1:${PARTNER_SEARCH_SYNC_DELAY_SECONDS}`,
+ );
+ });
+
+ it("does not deduplicate bulk chunks, whose composition never repeats", async () => {
+ await queuePartnerSearchSync({ enrollmentIds: ["pge_1", "pge_2"] });
+
+ const [{ options }] = dispatchedWithOptions();
+
+ expect(options.deduplicationId).toBeUndefined();
+ });
+
+ it("keeps a slow link sync from suppressing a fast one", async () => {
+ await queuePartnerSearchSync({
+ enrollmentIds: ["pge_1"],
+ delay: PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS,
+ });
+ await queuePartnerSearchSync({ enrollmentIds: ["pge_1"] });
+
+ const slow = mocks.dispatchBatch.mock.calls[0][1](
+ { type: "enrollments", enrollmentIds: ["pge_1"] },
+ 0,
+ );
+ const fast = mocks.dispatchBatch.mock.calls[1][1](
+ { type: "enrollments", enrollmentIds: ["pge_1"] },
+ 0,
+ );
+
+ expect(slow.delay).toBe(PARTNER_SEARCH_LINK_SYNC_DELAY_SECONDS);
+ expect(fast.delay).toBe(PARTNER_SEARCH_SYNC_DELAY_SECONDS);
+ expect(slow.deduplicationId).not.toBe(fast.deduplicationId);
+ });
+
+ it("chunks past the batch size so no payload exceeds what the job accepts", async () => {
+ await queuePartnerSearchSync({
+ enrollmentIds: ["pge_1", "pge_2", "pge_3", "pge_4"],
+ });
+
+ expect(mocks.dispatchBatch.mock.calls[0][0]).toEqual([
+ { type: "enrollments", enrollmentIds: ["pge_1", "pge_2", "pge_3"] },
+ { type: "enrollments", enrollmentIds: ["pge_4"] },
+ ]);
+ });
+
+ it("queues both shapes when a caller has enrollments and partners", async () => {
+ await queuePartnerSearchSync({
+ enrollmentIds: ["pge_1"],
+ partnerIds: ["pn_1"],
+ programId: "prog_1",
+ });
+
+ expect(mocks.dispatchBatch.mock.calls[0][0]).toEqual([
+ { type: "enrollments", enrollmentIds: ["pge_1"] },
+ { type: "partners", partnerIds: ["pn_1"], programId: "prog_1" },
+ ]);
+ });
+
+ it("swallows a dispatch failure so it cannot break the mutation that queued it", async () => {
+ mocks.dispatchBatch.mockRejectedValue(new Error("qstash unreachable"));
+
+ await expect(
+ queuePartnerSearchSync({ enrollmentIds: ["pge_1"] }),
+ ).resolves.toBeUndefined();
+ });
+
+ it("never builds a payload larger than the job's batch size", async () => {
+ await queuePartnerSearchSync({
+ partnerIds: Array.from({ length: 7 }, (_, index) => `pn_${index}`),
+ });
+
+ for (const payload of mocks.dispatchBatch.mock.calls[0][0]) {
+ expect(payload.partnerIds.length).toBeLessThanOrEqual(BATCH_SIZE);
+ }
+ });
+});