The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Batch partner search document synchronization
Rafid2b522cd4723
diff --git a/apps/web/lib/api/partners/search/backfill.ts b/apps/web/lib/api/partners/search/backfill.tsindex 1f986c2d4b..c4b7d92a14 100644--- a/apps/web/lib/api/partners/search/backfill.ts+++ b/apps/web/lib/api/partners/search/backfill.ts@@ -41,6 +41,7 @@ export async function backfillPartnerSearch({let processed = 0;while (true) {+ // TODO: Add a composite database index on programId and id before production backfillsconst enrollments = await prisma.programEnrollment.findMany({where: {programId,diff --git a/apps/web/lib/api/partners/search/sync-documents.ts b/apps/web/lib/api/partners/search/sync-documents.tsindex a73fd577f3..781c1672a8 100644--- a/apps/web/lib/api/partners/search/sync-documents.ts+++ b/apps/web/lib/api/partners/search/sync-documents.ts@@ -1,10 +1,14 @@import { prisma } from "@/lib/prisma";+import { chunk } from "@dub/utils";+import type { Prisma } from "@prisma/client";import { getPartnerSearchProvider } from "./provider";import {partnerSearchDocumentSelect,serializePartnerSearchDocument,} from "./serialize-document";-import { PartnerSearchProvider } from "./types";+import type { PartnerSearchProvider } from "./types";++const PARTNER_SEARCH_SYNC_BATCH_SIZE = 100;interface PartnerSearchSyncOptions {searchProvider?: PartnerSearchProvider | null;@@ -32,6 +36,40 @@ function uniqueProgramPartners();}+async function upsertPartnerSearchDocuments(+ where: Prisma.ProgramEnrollmentWhereInput,+ searchProvider: PartnerSearchProvider,+) {+ let cursor: string | undefined;++ while (true) {+ const enrollments = await prisma.programEnrollment.findMany({+ where,+ select: partnerSearchDocumentSelect,+ orderBy: { id: "asc" },+ take: PARTNER_SEARCH_SYNC_BATCH_SIZE,+ ...(cursor && {+ cursor: { id: cursor },+ skip: 1,+ }),+ });++ if (enrollments.length === 0) {+ return;+ }++ await searchProvider.upsert(+ enrollments.map(serializePartnerSearchDocument),+ );++ if (enrollments.length < PARTNER_SEARCH_SYNC_BATCH_SIZE) {+ return;+ }++ cursor = enrollments[enrollments.length - 1].id;+ }+}+export async function syncPartnerSearchDocuments(documentIds: string[],{@@ -42,25 +80,29 @@ export async function syncPartnerSearchDocuments(return;}- const uniqueIds = uniqueDocumentIds(documentIds);- const enrollments = await prisma.programEnrollment.findMany({- where: {- id: { in: uniqueIds },- },- select: partnerSearchDocumentSelect,- });-- if (enrollments.length > 0) {- await searchProvider.upsert(- enrollments.map(serializePartnerSearchDocument),- );- }-- const foundIds = new Set(enrollments.map(({ id }) => id));- const missingIds = uniqueIds.filter((id) => !foundIds.has(id));-- if (missingIds.length > 0) {- await searchProvider.delete(missingIds);+ for (const documentIdBatch of chunk(+ uniqueDocumentIds(documentIds),+ PARTNER_SEARCH_SYNC_BATCH_SIZE,+ )) {+ const enrollments = await prisma.programEnrollment.findMany({+ where: {+ id: { in: documentIdBatch },+ },+ select: partnerSearchDocumentSelect,+ });++ if (enrollments.length > 0) {+ await searchProvider.upsert(+ enrollments.map(serializePartnerSearchDocument),+ );+ }++ const foundIds = new Set(enrollments.map(({ id }) => id));+ const missingIds = documentIdBatch.filter((id) => !foundIds.has(id));++ if (missingIds.length > 0) {+ await searchProvider.delete(missingIds);+ }}}@@ -74,16 +116,13 @@ export async function syncPartnerSearchDocumentsByPartnerIds(return;}- const enrollments = await prisma.programEnrollment.findMany({- where: {- partnerId: { in: uniqueDocumentIds(partnerIds) },- },- select: partnerSearchDocumentSelect,- });-- if (enrollments.length > 0) {- await searchProvider.upsert(- enrollments.map(serializePartnerSearchDocument),+ for (const partnerIdBatch of chunk(+ uniqueDocumentIds(partnerIds),+ PARTNER_SEARCH_SYNC_BATCH_SIZE,+ )) {+ await upsertPartnerSearchDocuments(+ { partnerId: { in: partnerIdBatch } },+ searchProvider,);}}@@ -98,16 +137,13 @@ export async function syncPartnerSearchDocumentsByProgramPartners(return;}- const enrollments = await prisma.programEnrollment.findMany({- where: {- OR: uniqueProgramPartners(programPartners),- },- select: partnerSearchDocumentSelect,- });-- if (enrollments.length > 0) {- await searchProvider.upsert(- enrollments.map(serializePartnerSearchDocument),+ for (const programPartnerBatch of chunk(+ uniqueProgramPartners(programPartners),+ PARTNER_SEARCH_SYNC_BATCH_SIZE,+ )) {+ await upsertPartnerSearchDocuments(+ { OR: programPartnerBatch },+ searchProvider,);}}@@ -122,5 +158,10 @@ export async function deletePartnerSearchDocuments(return;}- await searchProvider.delete(uniqueDocumentIds(documentIds));+ for (const documentIdBatch of chunk(+ uniqueDocumentIds(documentIds),+ PARTNER_SEARCH_SYNC_BATCH_SIZE,+ )) {+ await searchProvider.delete(documentIdBatch);+ }}diff --git a/apps/web/tests/partners/partner-search-sync.test.ts b/apps/web/tests/partners/partner-search-sync.test.tsindex c9ed386b68..ffc4c608d1 100644--- a/apps/web/tests/partners/partner-search-sync.test.ts+++ b/apps/web/tests/partners/partner-search-sync.test.ts@@ -1,8 +1,8 @@import {deletePartnerSearchDocuments,partnerSearchDocumentSelect,- PartnerSearchDocumentSource,- PartnerSearchProvider,+ type PartnerSearchDocumentSource,+ type PartnerSearchProvider,syncPartnerSearchDocuments,syncPartnerSearchDocumentsByPartnerIds,syncPartnerSearchDocumentsByProgramPartners,@@ -134,6 +134,8 @@ describe("partner search document sync", () => {partnerId: { in: ["pn_1"] },},select: partnerSearchDocumentSelect,+ orderBy: { id: "asc" },+ take: 100,});expect(searchProvider.upsert).toHaveBeenCalledWith([expect.objectContaining({ id: "pge_1" }),@@ -162,6 +164,8 @@ describe("partner search document sync", () => {],},select: partnerSearchDocumentSelect,+ orderBy: { id: "asc" },+ take: 100,});expect(searchProvider.upsert).toHaveBeenCalledWith([expect.objectContaining({ id: "pge_1" }),@@ -178,4 +182,70 @@ describe("partner search document sync", () => {expect(searchProvider.delete).toHaveBeenCalledWith(["pge_1", "pge_2"]);expect(mocks.findMany).not.toHaveBeenCalled();});++ it("bounds document synchronization batches", async () => {+ const searchProvider = createProvider();+ const upsert = vi.mocked(searchProvider.upsert);+ const documentIds = Array.from(+ { length: 101 },+ (_, index) => `pge_${index + 1}`,+ );+ mocks.findMany.mockImplementation(({ where }) =>+ Promise.resolve(+ (where.id.in as string[]).map((id: string) => createSource(id)),+ ),+ );++ await syncPartnerSearchDocuments(documentIds, { searchProvider });++ expect(mocks.findMany).toHaveBeenCalledTimes(2);+ expect(mocks.findMany.mock.calls[0][0].where.id.in).toHaveLength(100);+ expect(mocks.findMany.mock.calls[1][0].where.id.in).toHaveLength(1);+ expect(searchProvider.upsert).toHaveBeenCalledTimes(2);+ expect(upsert.mock.calls[0][0]).toHaveLength(100);+ expect(upsert.mock.calls[1][0]).toHaveLength(1);+ });++ it("paginates all enrollments matched by a partner batch", async () => {+ const searchProvider = createProvider();+ const upsert = vi.mocked(searchProvider.upsert);+ const firstPage = Array.from({ length: 100 }, (_, index) =>+ createSource(`pge_${index + 1}`),+ );+ mocks.findMany+ .mockResolvedValueOnce(firstPage)+ .mockResolvedValueOnce([createSource("pge_101")]);++ await syncPartnerSearchDocumentsByPartnerIds(["pn_1"], {+ searchProvider,+ });++ expect(mocks.findMany).toHaveBeenNthCalledWith(2, {+ where: { partnerId: { in: ["pn_1"] } },+ select: partnerSearchDocumentSelect,+ orderBy: { id: "asc" },+ take: 100,+ cursor: { id: "pge_100" },+ skip: 1,+ });+ expect(upsert).toHaveBeenCalledTimes(2);+ expect(upsert.mock.calls[0][0]).toHaveLength(100);+ expect(upsert.mock.calls[1][0]).toHaveLength(1);+ });++ it("bounds provider delete batches", async () => {+ const searchProvider = createProvider();+ const deleteDocuments = vi.mocked(searchProvider.delete);+ const documentIds = Array.from(+ { length: 201 },+ (_, index) => `pge_${index + 1}`,+ );++ await deletePartnerSearchDocuments(documentIds, { searchProvider });++ expect(searchProvider.delete).toHaveBeenCalledTimes(3);+ expect(deleteDocuments.mock.calls[0][0]).toHaveLength(100);+ expect(deleteDocuments.mock.calls[1][0]).toHaveLength(100);+ expect(deleteDocuments.mock.calls[2][0]).toHaveLength(1);+ });});