The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Add the partner search sync core
Rafi9ffd893972b1
diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.tsindex 05c9d4faf3..b1ea92b8f9 100644--- a/apps/web/lib/api/partners/search/index.ts+++ b/apps/web/lib/api/partners/search/index.ts@@ -5,4 +5,5 @@ export * from "./order-search-results";export * from "./provider";export * from "./searchable-values";export * from "./serialize-document";+export * from "./sync";export * from "./types";diff --git a/apps/web/lib/api/partners/search/sync.ts b/apps/web/lib/api/partners/search/sync.tsnew file mode 100644index 0000000000..bffef1a823--- /dev/null+++ b/apps/web/lib/api/partners/search/sync.ts@@ -0,0 +1,128 @@+import { prisma } from "@/lib/prisma";+import { getPartnerSearchProvider } from "./provider";+import {+ partnerSearchDocumentSelect,+ serializePartnerSearchDocument,+} from "./serialize-document";+import type { PartnerSearchProvider } from "./types";++/**+ * How many enrollments one sync resolves at a time, matching the backfill's+ * batch size. The provider chunks its own writes below this, so the ceiling+ * here is about how much the database hydrates at once, not the write size.+ */+export const PARTNER_SEARCH_SYNC_BATCH_SIZE = 500;++export interface PartnerSearchSyncResult {+ upserted: number;+ deleted: number;+}++interface SyncPartnerSearchDocumentsOptions {+ enrollmentIds: string[];+ searchProvider?: PartnerSearchProvider | null;+}++function unique(values: string[]): string[] {+ return Array.from(new Set(values.filter(Boolean)));+}++/**+ * Brings the index in line with the database for the given enrollments.+ *+ * Current state is read here rather than passed in by the caller, so a delayed+ * or replayed sync converges on the truth instead of re-applying whatever was+ * true when it was queued. That is also what makes deletes fall out for free:+ * an ID the database no longer has is removed from the index rather than+ * treated as an error, so a caller reacting to a deletion queues the same job+ * as one reacting to an edit.+ *+ * No-ops when no provider is configured, matching the read path, which falls+ * back to the database search rather than failing.+ */+export async function syncPartnerSearchDocuments({+ enrollmentIds,+ searchProvider = getPartnerSearchProvider(),+}: SyncPartnerSearchDocumentsOptions): Promise<PartnerSearchSyncResult> {+ const ids = unique(enrollmentIds);++ if (!searchProvider || ids.length === 0) {+ return { upserted: 0, deleted: 0 };+ }++ const enrollments = await prisma.programEnrollment.findMany({+ where: {+ id: {+ in: ids,+ },+ },+ select: partnerSearchDocumentSelect,+ });++ const foundIds = new Set(enrollments.map(({ id }) => id));+ const missingIds = ids.filter((id) => !foundIds.has(id));++ if (enrollments.length > 0) {+ await searchProvider.upsert(enrollments.map(serializePartnerSearchDocument));+ }++ if (missingIds.length > 0) {+ await searchProvider.delete(missingIds);+ }++ return {+ upserted: enrollments.length,+ deleted: missingIds.length,+ };+}++interface FindPartnerSearchSyncEnrollmentIdsOptions {+ partnerIds: string[];+ programId?: string;+ after?: string;+ take?: number;+}++/**+ * One page of enrollment IDs belonging to the given partners, for the changes+ * that fan out beyond a single enrollment: a profile edit or a platform change+ * touches every program the partner is enrolled in.+ *+ * Paged by enrollment ID so the caller can resume, because the fan-out is+ * unbounded — a partner enrolled in hundreds of programs produces hundreds of+ * documents from one profile write.+ *+ * `programId` narrows to a single enrollment per partner, which is what the+ * enrollment-scoped callers (group moves, status changes, tags) pass.+ */+export async function findPartnerSearchSyncEnrollmentIds({+ partnerIds,+ programId,+ after,+ take = PARTNER_SEARCH_SYNC_BATCH_SIZE,+}: FindPartnerSearchSyncEnrollmentIdsOptions): Promise<string[]> {+ const ids = unique(partnerIds);++ if (ids.length === 0) {+ return [];+ }++ const enrollments = await prisma.programEnrollment.findMany({+ where: {+ partnerId: {+ in: ids,+ },+ ...(programId && { programId }),+ ...(after && { id: { gt: after } }),+ },+ select: {+ id: true,+ },+ orderBy: {+ id: "asc",+ },+ take,+ });++ return enrollments.map(({ id }) => id);+}diff --git a/apps/web/tests/partners/partner-search-sync.test.ts b/apps/web/tests/partners/partner-search-sync.test.tsnew file mode 100644index 0000000000..6973a2e49f--- /dev/null+++ b/apps/web/tests/partners/partner-search-sync.test.ts@@ -0,0 +1,231 @@+import {+ findPartnerSearchSyncEnrollmentIds,+ syncPartnerSearchDocuments,+ type PartnerSearchDocumentSource,+ type PartnerSearchProvider,+} from "@/lib/api/partners/search";+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ findMany: vi.fn(),+}));++vi.mock("@/lib/prisma", () => ({+ prisma: {+ programEnrollment: {+ findMany: mocks.findMany,+ },+ },+}));++function createSource(+ id: string,+ overrides: Partial<PartnerSearchDocumentSource> = {},+): PartnerSearchDocumentSource {+ return {+ id,+ programId: "prog_test",+ partnerId: `pn_${id}`,+ status: "approved" as const,+ groupId: null,+ partner: {+ name: "Rafi Hasan",+ email: "[email protected]",+ companyName: "Dub Partners",+ description: "Developer tools educator",+ country: null,+ programPartnerTags: [],+ platforms: [],+ },+ links: [],+ ...overrides,+ };+}++function createProvider(): PartnerSearchProvider {+ return {+ searchCandidates: vi.fn(),+ countCandidates: vi.fn(),+ upsert: vi.fn(),+ delete: vi.fn(),+ };+}++describe("syncPartnerSearchDocuments", () => {+ beforeEach(() => {+ mocks.findMany.mockReset();+ });++ it("upserts the enrollments the database still has", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([+ createSource("pge_1"),+ createSource("pge_2"),+ ]);++ const result = await syncPartnerSearchDocuments({+ enrollmentIds: ["pge_1", "pge_2"],+ searchProvider,+ });++ expect(result).toEqual({ upserted: 2, deleted: 0 });+ expect(searchProvider.delete).not.toHaveBeenCalled();+ expect(searchProvider.upsert).toHaveBeenCalledTimes(1);+ expect(+ vi.mocked(searchProvider.upsert).mock.calls[0][0].map(({ id }) => id),+ ).toEqual(["pge_1", "pge_2"]);+ });++ it("deletes the enrollments the database no longer has", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);++ const result = await syncPartnerSearchDocuments({+ enrollmentIds: ["pge_1", "pge_deleted"],+ searchProvider,+ });++ expect(result).toEqual({ upserted: 1, deleted: 1 });+ expect(searchProvider.delete).toHaveBeenCalledWith(["pge_deleted"]);+ });++ it("deletes without upserting when every enrollment is gone", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([]);++ const result = await syncPartnerSearchDocuments({+ enrollmentIds: ["pge_gone"],+ searchProvider,+ });++ expect(result).toEqual({ upserted: 0, deleted: 1 });+ expect(searchProvider.upsert).not.toHaveBeenCalled();+ expect(searchProvider.delete).toHaveBeenCalledWith(["pge_gone"]);+ });++ it("deduplicates the requested ids so one enrollment is written once", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);++ await syncPartnerSearchDocuments({+ enrollmentIds: ["pge_1", "pge_1", "pge_1"],+ searchProvider,+ });++ expect(mocks.findMany.mock.calls[0][0].where.id.in).toEqual(["pge_1"]);+ expect(vi.mocked(searchProvider.upsert).mock.calls[0][0]).toHaveLength(1);+ });++ it("does not touch the database when no provider is configured", async () => {+ const result = await syncPartnerSearchDocuments({+ enrollmentIds: ["pge_1"],+ searchProvider: null,+ });++ expect(result).toEqual({ upserted: 0, deleted: 0 });+ expect(mocks.findMany).not.toHaveBeenCalled();+ });++ it("does not touch the database when there is nothing to sync", async () => {+ const searchProvider = createProvider();++ const result = await syncPartnerSearchDocuments({+ enrollmentIds: [],+ searchProvider,+ });++ expect(result).toEqual({ upserted: 0, deleted: 0 });+ expect(mocks.findMany).not.toHaveBeenCalled();+ });++ it("serializes the document the index stores", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([+ createSource("pge_1", {+ partner: {+ name: "Rafi Hasan",+ email: "[email protected]",+ companyName: null,+ description: null,+ country: "US",+ // Tags from another program must not leak into this document.+ programPartnerTags: [+ { programId: "prog_test", partnerTagId: "ptag_1" },+ { programId: "prog_other", partnerTagId: "ptag_other" },+ ],+ platforms: [{ type: "youtube" as const, identifier: "rafi" }],+ },+ }),+ ]);++ await syncPartnerSearchDocuments({+ enrollmentIds: ["pge_1"],+ searchProvider,+ });++ expect(vi.mocked(searchProvider.upsert).mock.calls[0][0][0]).toMatchObject({+ id: "pge_1",+ country: "US",+ partnerTagIds: ["ptag_1"],+ platformIdentifiers: ["rafi"],+ });+ });+});++describe("findPartnerSearchSyncEnrollmentIds", () => {+ beforeEach(() => {+ mocks.findMany.mockReset();+ });++ it("resolves every enrollment a partner has when no program is given", async () => {+ mocks.findMany.mockResolvedValueOnce([{ id: "pge_1" }, { id: "pge_2" }]);++ const ids = await findPartnerSearchSyncEnrollmentIds({+ partnerIds: ["pn_1"],+ });++ expect(ids).toEqual(["pge_1", "pge_2"]);++ const { where, orderBy } = mocks.findMany.mock.calls[0][0];+ expect(where).toEqual({ partnerId: { in: ["pn_1"] } });+ expect(orderBy).toEqual({ id: "asc" });+ });++ it("narrows to a single program when one is given", async () => {+ mocks.findMany.mockResolvedValueOnce([{ id: "pge_1" }]);++ await findPartnerSearchSyncEnrollmentIds({+ partnerIds: ["pn_1"],+ programId: "prog_test",+ });++ expect(mocks.findMany.mock.calls[0][0].where).toEqual({+ partnerId: { in: ["pn_1"] },+ programId: "prog_test",+ });+ });++ it("pages past the cursor so a large fan-out can resume", async () => {+ mocks.findMany.mockResolvedValueOnce([{ id: "pge_3" }]);++ await findPartnerSearchSyncEnrollmentIds({+ partnerIds: ["pn_1"],+ after: "pge_2",+ take: 2,+ });++ const { where, take } = mocks.findMany.mock.calls[0][0];+ expect(where).toEqual({+ partnerId: { in: ["pn_1"] },+ id: { gt: "pge_2" },+ });+ expect(take).toBe(2);+ });++ it("does not query when there are no partners", async () => {+ const ids = await findPartnerSearchSyncEnrollmentIds({ partnerIds: [] });++ expect(ids).toEqual([]);+ expect(mocks.findMany).not.toHaveBeenCalled();+ });+});