The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Feature/partner-search: read a partner fan-out page in one query (#4486)
Rafi6d83ae7f97e0
diff --git a/apps/web/lib/api/partners/search/sync.ts b/apps/web/lib/api/partners/search/sync.tsindex 2c5cda7102..5d046ce84d 100644--- a/apps/web/lib/api/partners/search/sync.ts+++ b/apps/web/lib/api/partners/search/sync.ts@@ -69,31 +69,38 @@ export async function syncPartnerSearchDocuments({};}-interface FindPartnerSearchSyncEnrollmentIdsOptions {+interface SyncPartnersOptions {partnerIds: string[];programId?: string;after?: string;take?: number;+ searchProvider?: PartnerSearchProvider | null;+}++export interface SyncPartnersResult {+ upserted: number;+ lastEnrollmentId: string | null;}/**- * One page of enrollment IDs for the given partners, for changes that fan out+ * One page of documents for the given partners, for changes that fan out* beyond a single enrollment: a profile or platform edit touches every program- * the partner is in.+ * the partner is in. Paged because that fan-out is unbounded. `programId`+ * narrows it to one enrollment per partner.*- * Paged because that fan-out is unbounded. `programId` narrows it to one- * enrollment per partner.+ * Upserts only. The rows come from the database, so there is nothing to delete.*/-export async function findPartnerSearchSyncEnrollmentIds({+export async function syncPartnerEnrollments({partnerIds,programId,after,take = PARTNER_SEARCH_SYNC_BATCH_SIZE,-}: FindPartnerSearchSyncEnrollmentIdsOptions): Promise<string[]> {+ searchProvider = getPartnerSearchProvider(),+}: SyncPartnersOptions): Promise<SyncPartnersResult> {const ids = unique(partnerIds.filter(Boolean));- if (ids.length === 0) {- return [];+ if (!searchProvider || ids.length === 0) {+ return { upserted: 0, lastEnrollmentId: null };}const enrollments = await prisma.programEnrollment.findMany({@@ -104,14 +111,21 @@ export async function findPartnerSearchSyncEnrollmentIds({...(programId && { programId }),...(after && { id: { gt: after } }),},- select: {- id: true,- },+ select: partnerSearchDocumentSelect,orderBy: {id: "asc",},take,});- return enrollments.map(({ id }) => id);+ if (enrollments.length > 0) {+ await searchProvider.upsert(+ enrollments.map(serializePartnerSearchDocument),+ );+ }++ return {+ upserted: enrollments.length,+ lastEnrollmentId: enrollments.at(-1)?.id ?? null,+ };}diff --git a/apps/web/lib/jobs/handlers/partner-search-sync-job.ts b/apps/web/lib/jobs/handlers/partner-search-sync-job.tsindex 998176bd38..0dbc33648d 100644--- a/apps/web/lib/jobs/handlers/partner-search-sync-job.ts+++ b/apps/web/lib/jobs/handlers/partner-search-sync-job.ts@@ -1,7 +1,7 @@import {- findPartnerSearchSyncEnrollmentIds,getPartnerSearchProvider,PARTNER_SEARCH_SYNC_BATCH_SIZE,+ syncPartnerEnrollments,syncPartnerSearchDocuments,} from "@/lib/api/partners/search";import * as z from "zod/v4";@@ -74,31 +74,27 @@ export const partnerSearchSyncJob = defineJob({return;}- const enrollmentIds = await findPartnerSearchSyncEnrollmentIds({+ const { upserted, lastEnrollmentId } = await syncPartnerEnrollments({partnerIds: input.partnerIds,programId: input.programId,after: input.after,take: PARTNER_SEARCH_SYNC_BATCH_SIZE,+ searchProvider,});- if (enrollmentIds.length === 0) {+ if (upserted === 0) {return;}- const { upserted, deleted } = await syncPartnerSearchDocuments({- enrollmentIds,- searchProvider,- });-console.log(- `[partnerSearchSyncJob] Synced ${upserted} and removed ${deleted} enrollment documents for ${input.partnerIds.length} partner(s).`,+ `[partnerSearchSyncJob] Synced ${upserted} enrollment documents for ${input.partnerIds.length} partner(s).`,);- if (enrollmentIds.length === PARTNER_SEARCH_SYNC_BATCH_SIZE) {+ if (upserted === PARTNER_SEARCH_SYNC_BATCH_SIZE && lastEnrollmentId) {await partnerSearchSyncJob.dispatch({...input,- after: enrollmentIds[enrollmentIds.length - 1],+ after: lastEnrollmentId,},{delay: 1,diff --git a/apps/web/tests/partners/partner-search-sync-job.test.ts b/apps/web/tests/partners/partner-search-sync-job.test.tsindex 89062a1266..8906e5f8b0 100644--- a/apps/web/tests/partners/partner-search-sync-job.test.ts+++ b/apps/web/tests/partners/partner-search-sync-job.test.ts@@ -7,14 +7,14 @@ const BATCH_SIZE = 3;const mocks = vi.hoisted(() => ({getPartnerSearchProvider: vi.fn(),syncPartnerSearchDocuments: vi.fn(),- findPartnerSearchSyncEnrollmentIds: vi.fn(),+ syncPartnerEnrollments: vi.fn(),}));vi.mock("@/lib/api/partners/search", () => ({PARTNER_SEARCH_SYNC_BATCH_SIZE: 3,getPartnerSearchProvider: mocks.getPartnerSearchProvider,syncPartnerSearchDocuments: mocks.syncPartnerSearchDocuments,- findPartnerSearchSyncEnrollmentIds: mocks.findPartnerSearchSyncEnrollmentIds,+ syncPartnerEnrollments: mocks.syncPartnerEnrollments,}));const searchProvider = { name: "turbopuffer" };@@ -26,7 +26,9 @@ describe("partnerSearchSyncJob", () => {mocks.syncPartnerSearchDocuments.mockReset().mockResolvedValue({ upserted: 0, deleted: 0 });- mocks.findPartnerSearchSyncEnrollmentIds.mockReset().mockResolvedValue([]);+ mocks.syncPartnerEnrollments+ .mockReset()+ .mockResolvedValue({ upserted: 0, lastEnrollmentId: null });});it("skips entirely when no provider is configured", async () => {@@ -38,7 +40,7 @@ describe("partnerSearchSyncJob", () => {});expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();- expect(mocks.findPartnerSearchSyncEnrollmentIds).not.toHaveBeenCalled();+ expect(mocks.syncPartnerEnrollments).not.toHaveBeenCalled();});it("syncs the enrollment ids it is given", async () => {@@ -51,30 +53,28 @@ describe("partnerSearchSyncJob", () => {enrollmentIds: ["pge_1", "pge_2"],searchProvider,});- expect(mocks.findPartnerSearchSyncEnrollmentIds).not.toHaveBeenCalled();+ expect(mocks.syncPartnerEnrollments).not.toHaveBeenCalled();});- it("resolves a partner fan-out before syncing", async () => {- mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([- "pge_1",- "pge_2",- ]);+ it("syncs a partner fan-out in one read", async () => {+ mocks.syncPartnerEnrollments.mockResolvedValue({+ upserted: 2,+ lastEnrollmentId: "pge_2",+ });await partnerSearchSyncJob.execute({type: "partners",partnerIds: ["pn_1"],});- expect(mocks.findPartnerSearchSyncEnrollmentIds).toHaveBeenCalledWith({+ expect(mocks.syncPartnerEnrollments).toHaveBeenCalledWith({partnerIds: ["pn_1"],programId: undefined,after: undefined,take: BATCH_SIZE,- });- expect(mocks.syncPartnerSearchDocuments).toHaveBeenCalledWith({- enrollmentIds: ["pge_1", "pge_2"],searchProvider,});+ expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();});it("continues from the last enrollment when a page comes back full", async () => {@@ -82,11 +82,10 @@ describe("partnerSearchSyncJob", () => {.spyOn(partnerSearchSyncJob, "dispatch").mockResolvedValue({ status: "published", messageId: "msg_1" });- mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([- "pge_1",- "pge_2",- "pge_3",- ]);+ mocks.syncPartnerEnrollments.mockResolvedValue({+ upserted: BATCH_SIZE,+ lastEnrollmentId: "pge_3",+ });await partnerSearchSyncJob.execute({type: "partners",@@ -110,10 +109,10 @@ describe("partnerSearchSyncJob", () => {.spyOn(partnerSearchSyncJob, "dispatch").mockResolvedValue({ status: "published", messageId: "msg_1" });- mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([- "pge_1",- "pge_2",- ]);+ mocks.syncPartnerEnrollments.mockResolvedValue({+ upserted: 2,+ lastEnrollmentId: "pge_2",+ });await partnerSearchSyncJob.execute({type: "partners",@@ -128,14 +127,16 @@ describe("partnerSearchSyncJob", () => {.spyOn(partnerSearchSyncJob, "dispatch").mockResolvedValue({ status: "published", messageId: "msg_1" });- mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([]);+ mocks.syncPartnerEnrollments.mockResolvedValue({+ upserted: 0,+ lastEnrollmentId: null,+ });await partnerSearchSyncJob.execute({type: "partners",partnerIds: ["pn_1"],});- expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();expect(dispatch).not.toHaveBeenCalled();});diff --git a/apps/web/tests/partners/partner-search-sync.test.ts b/apps/web/tests/partners/partner-search-sync.test.tsindex ec0d0944ad..c75dd074db 100644--- a/apps/web/tests/partners/partner-search-sync.test.ts+++ b/apps/web/tests/partners/partner-search-sync.test.ts@@ -1,5 +1,6 @@import {- findPartnerSearchSyncEnrollmentIds,+ partnerSearchDocumentSelect,+ syncPartnerEnrollments,syncPartnerSearchDocuments,type PartnerSearchDocumentSource,type PartnerSearchProvider,@@ -169,31 +170,46 @@ describe("syncPartnerSearchDocuments", () => {});});-describe("findPartnerSearchSyncEnrollmentIds", () => {+describe("syncPartnerEnrollments", () => {beforeEach(() => {mocks.findMany.mockReset();});- it("resolves every enrollment a partner has when no program is given", async () => {- mocks.findMany.mockResolvedValueOnce([{ id: "pge_1" }, { id: "pge_2" }]);+ it("reads and upserts one page of documents in a single query", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([+ createSource("pge_1"),+ createSource("pge_2"),+ ]);- const ids = await findPartnerSearchSyncEnrollmentIds({+ const result = await syncPartnerEnrollments({partnerIds: ["pn_1"],+ searchProvider,});- expect(ids).toEqual(["pge_1", "pge_2"]);-- const { where, orderBy } = mocks.findMany.mock.calls[0][0];- expect(where).toEqual({ partnerId: { in: ["pn_1"] } });- expect(orderBy).toEqual({ id: "asc" });+ expect(result).toEqual({ upserted: 2, lastEnrollmentId: "pge_2" });+ expect(mocks.findMany).toHaveBeenCalledTimes(1);+ expect(mocks.findMany.mock.calls[0][0].where).toEqual({+ partnerId: { in: ["pn_1"] },+ });+ expect(mocks.findMany.mock.calls[0][0]).toMatchObject({+ select: partnerSearchDocumentSelect,+ orderBy: { id: "asc" },+ });+ expect(+ vi.mocked(searchProvider.upsert).mock.calls[0][0].map(({ id }) => id),+ ).toEqual(["pge_1", "pge_2"]);+ expect(searchProvider.delete).not.toHaveBeenCalled();});it("narrows to a single program when one is given", async () => {- mocks.findMany.mockResolvedValueOnce([{ id: "pge_1" }]);+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);- await findPartnerSearchSyncEnrollmentIds({+ await syncPartnerEnrollments({partnerIds: ["pn_1"],programId: "prog_test",+ searchProvider,});expect(mocks.findMany.mock.calls[0][0].where).toEqual({@@ -203,12 +219,14 @@ describe("findPartnerSearchSyncEnrollmentIds", () => {});it("pages past the cursor so a large fan-out can resume", async () => {- mocks.findMany.mockResolvedValueOnce([{ id: "pge_3" }]);+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([createSource("pge_3")]);- await findPartnerSearchSyncEnrollmentIds({+ await syncPartnerEnrollments({partnerIds: ["pn_1"],after: "pge_2",take: 2,+ searchProvider,});const { where, take } = mocks.findMany.mock.calls[0][0];@@ -219,10 +237,28 @@ describe("findPartnerSearchSyncEnrollmentIds", () => {expect(take).toBe(2);});+ it("does not write when the page is empty", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValueOnce([]);++ const result = await syncPartnerEnrollments({+ partnerIds: ["pn_1"],+ searchProvider,+ });++ expect(result).toEqual({ upserted: 0, lastEnrollmentId: null });+ expect(searchProvider.upsert).not.toHaveBeenCalled();+ });+it("does not query when there are no partners", async () => {- const ids = await findPartnerSearchSyncEnrollmentIds({ partnerIds: [] });+ const searchProvider = createProvider();++ const result = await syncPartnerEnrollments({+ partnerIds: [],+ searchProvider,+ });- expect(ids).toEqual([]);+ expect(result).toEqual({ upserted: 0, lastEnrollmentId: null });expect(mocks.findMany).not.toHaveBeenCalled();});});