Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Batch partner search document synchronization

Rafid2b522cd4723
diff --git a/apps/web/lib/api/partners/search/backfill.ts b/apps/web/lib/api/partners/search/backfill.ts
index 1f986c2d4b..c4b7d92a14 100644
--- a/apps/web/lib/api/partners/search/backfill.ts
+++ b/apps/web/lib/api/partners/search/backfill.ts
@@ -41,6 +41,7 @@ export async function backfillPartnerSearch({
let processed = 0;
while (true) {
+ // TODO: Add a composite database index on programId and id before production backfills
const enrollments = await prisma.programEnrollment.findMany({
where: {
programId,
diff --git a/apps/web/lib/api/partners/search/sync-documents.ts b/apps/web/lib/api/partners/search/sync-documents.ts
index a73fd577f3..781c1672a8 100644
--- a/apps/web/lib/api/partners/search/sync-documents.ts
+++ b/apps/web/lib/api/partners/search/sync-documents.ts
@@ -1,10 +1,14 @@
import { prisma } from "@/lib/prisma";
+import { chunk } from "@dub/utils";
+import type { Prisma } from "@prisma/client";
import { getPartnerSearchProvider } from "./provider";
import {
partnerSearchDocumentSelect,
serializePartnerSearchDocument,
} from "./serialize-document";
-import { PartnerSearchProvider } from "./types";
+import type { PartnerSearchProvider } from "./types";
+
+const PARTNER_SEARCH_SYNC_BATCH_SIZE = 100;
interface PartnerSearchSyncOptions {
searchProvider?: PartnerSearchProvider | null;
@@ -32,6 +36,40 @@ function uniqueProgramPartners(
);
}
+async function upsertPartnerSearchDocuments(
+ where: Prisma.ProgramEnrollmentWhereInput,
+ searchProvider: PartnerSearchProvider,
+) {
+ let cursor: string | undefined;
+
+ while (true) {
+ const enrollments = await prisma.programEnrollment.findMany({
+ where,
+ select: partnerSearchDocumentSelect,
+ orderBy: { id: "asc" },
+ take: PARTNER_SEARCH_SYNC_BATCH_SIZE,
+ ...(cursor && {
+ cursor: { id: cursor },
+ skip: 1,
+ }),
+ });
+
+ if (enrollments.length === 0) {
+ return;
+ }
+
+ await searchProvider.upsert(
+ enrollments.map(serializePartnerSearchDocument),
+ );
+
+ if (enrollments.length < PARTNER_SEARCH_SYNC_BATCH_SIZE) {
+ return;
+ }
+
+ cursor = enrollments[enrollments.length - 1].id;
+ }
+}
+
export async function syncPartnerSearchDocuments(
documentIds: string[],
{
@@ -42,25 +80,29 @@ export async function syncPartnerSearchDocuments(
return;
}
- const uniqueIds = uniqueDocumentIds(documentIds);
- const enrollments = await prisma.programEnrollment.findMany({
- where: {
- id: { in: uniqueIds },
- },
- select: partnerSearchDocumentSelect,
- });
-
- if (enrollments.length > 0) {
- await searchProvider.upsert(
- enrollments.map(serializePartnerSearchDocument),
- );
- }
-
- const foundIds = new Set(enrollments.map(({ id }) => id));
- const missingIds = uniqueIds.filter((id) => !foundIds.has(id));
-
- if (missingIds.length > 0) {
- await searchProvider.delete(missingIds);
+ for (const documentIdBatch of chunk(
+ uniqueDocumentIds(documentIds),
+ PARTNER_SEARCH_SYNC_BATCH_SIZE,
+ )) {
+ const enrollments = await prisma.programEnrollment.findMany({
+ where: {
+ id: { in: documentIdBatch },
+ },
+ select: partnerSearchDocumentSelect,
+ });
+
+ if (enrollments.length > 0) {
+ await searchProvider.upsert(
+ enrollments.map(serializePartnerSearchDocument),
+ );
+ }
+
+ const foundIds = new Set(enrollments.map(({ id }) => id));
+ const missingIds = documentIdBatch.filter((id) => !foundIds.has(id));
+
+ if (missingIds.length > 0) {
+ await searchProvider.delete(missingIds);
+ }
}
}
@@ -74,16 +116,13 @@ export async function syncPartnerSearchDocumentsByPartnerIds(
return;
}
- const enrollments = await prisma.programEnrollment.findMany({
- where: {
- partnerId: { in: uniqueDocumentIds(partnerIds) },
- },
- select: partnerSearchDocumentSelect,
- });
-
- if (enrollments.length > 0) {
- await searchProvider.upsert(
- enrollments.map(serializePartnerSearchDocument),
+ for (const partnerIdBatch of chunk(
+ uniqueDocumentIds(partnerIds),
+ PARTNER_SEARCH_SYNC_BATCH_SIZE,
+ )) {
+ await upsertPartnerSearchDocuments(
+ { partnerId: { in: partnerIdBatch } },
+ searchProvider,
);
}
}
@@ -98,16 +137,13 @@ export async function syncPartnerSearchDocumentsByProgramPartners(
return;
}
- const enrollments = await prisma.programEnrollment.findMany({
- where: {
- OR: uniqueProgramPartners(programPartners),
- },
- select: partnerSearchDocumentSelect,
- });
-
- if (enrollments.length > 0) {
- await searchProvider.upsert(
- enrollments.map(serializePartnerSearchDocument),
+ for (const programPartnerBatch of chunk(
+ uniqueProgramPartners(programPartners),
+ PARTNER_SEARCH_SYNC_BATCH_SIZE,
+ )) {
+ await upsertPartnerSearchDocuments(
+ { OR: programPartnerBatch },
+ searchProvider,
);
}
}
@@ -122,5 +158,10 @@ export async function deletePartnerSearchDocuments(
return;
}
- await searchProvider.delete(uniqueDocumentIds(documentIds));
+ for (const documentIdBatch of chunk(
+ uniqueDocumentIds(documentIds),
+ PARTNER_SEARCH_SYNC_BATCH_SIZE,
+ )) {
+ await searchProvider.delete(documentIdBatch);
+ }
}
diff --git a/apps/web/tests/partners/partner-search-sync.test.ts b/apps/web/tests/partners/partner-search-sync.test.ts
index c9ed386b68..ffc4c608d1 100644
--- a/apps/web/tests/partners/partner-search-sync.test.ts
+++ b/apps/web/tests/partners/partner-search-sync.test.ts
@@ -1,8 +1,8 @@
import {
deletePartnerSearchDocuments,
partnerSearchDocumentSelect,
- PartnerSearchDocumentSource,
- PartnerSearchProvider,
+ type PartnerSearchDocumentSource,
+ type PartnerSearchProvider,
syncPartnerSearchDocuments,
syncPartnerSearchDocumentsByPartnerIds,
syncPartnerSearchDocumentsByProgramPartners,
@@ -134,6 +134,8 @@ describe("partner search document sync", () => {
partnerId: { in: ["pn_1"] },
},
select: partnerSearchDocumentSelect,
+ orderBy: { id: "asc" },
+ take: 100,
});
expect(searchProvider.upsert).toHaveBeenCalledWith([
expect.objectContaining({ id: "pge_1" }),
@@ -162,6 +164,8 @@ describe("partner search document sync", () => {
],
},
select: partnerSearchDocumentSelect,
+ orderBy: { id: "asc" },
+ take: 100,
});
expect(searchProvider.upsert).toHaveBeenCalledWith([
expect.objectContaining({ id: "pge_1" }),
@@ -178,4 +182,70 @@ describe("partner search document sync", () => {
expect(searchProvider.delete).toHaveBeenCalledWith(["pge_1", "pge_2"]);
expect(mocks.findMany).not.toHaveBeenCalled();
});
+
+ it("bounds document synchronization batches", async () => {
+ const searchProvider = createProvider();
+ const upsert = vi.mocked(searchProvider.upsert);
+ const documentIds = Array.from(
+ { length: 101 },
+ (_, index) => `pge_${index + 1}`,
+ );
+ mocks.findMany.mockImplementation(({ where }) =>
+ Promise.resolve(
+ (where.id.in as string[]).map((id: string) => createSource(id)),
+ ),
+ );
+
+ await syncPartnerSearchDocuments(documentIds, { searchProvider });
+
+ expect(mocks.findMany).toHaveBeenCalledTimes(2);
+ expect(mocks.findMany.mock.calls[0][0].where.id.in).toHaveLength(100);
+ expect(mocks.findMany.mock.calls[1][0].where.id.in).toHaveLength(1);
+ expect(searchProvider.upsert).toHaveBeenCalledTimes(2);
+ expect(upsert.mock.calls[0][0]).toHaveLength(100);
+ expect(upsert.mock.calls[1][0]).toHaveLength(1);
+ });
+
+ it("paginates all enrollments matched by a partner batch", async () => {
+ const searchProvider = createProvider();
+ const upsert = vi.mocked(searchProvider.upsert);
+ const firstPage = Array.from({ length: 100 }, (_, index) =>
+ createSource(`pge_${index + 1}`),
+ );
+ mocks.findMany
+ .mockResolvedValueOnce(firstPage)
+ .mockResolvedValueOnce([createSource("pge_101")]);
+
+ await syncPartnerSearchDocumentsByPartnerIds(["pn_1"], {
+ searchProvider,
+ });
+
+ expect(mocks.findMany).toHaveBeenNthCalledWith(2, {
+ where: { partnerId: { in: ["pn_1"] } },
+ select: partnerSearchDocumentSelect,
+ orderBy: { id: "asc" },
+ take: 100,
+ cursor: { id: "pge_100" },
+ skip: 1,
+ });
+ expect(upsert).toHaveBeenCalledTimes(2);
+ expect(upsert.mock.calls[0][0]).toHaveLength(100);
+ expect(upsert.mock.calls[1][0]).toHaveLength(1);
+ });
+
+ it("bounds provider delete batches", async () => {
+ const searchProvider = createProvider();
+ const deleteDocuments = vi.mocked(searchProvider.delete);
+ const documentIds = Array.from(
+ { length: 201 },
+ (_, index) => `pge_${index + 1}`,
+ );
+
+ await deletePartnerSearchDocuments(documentIds, { searchProvider });
+
+ expect(searchProvider.delete).toHaveBeenCalledTimes(3);
+ expect(deleteDocuments.mock.calls[0][0]).toHaveLength(100);
+ expect(deleteDocuments.mock.calls[1][0]).toHaveLength(100);
+ expect(deleteDocuments.mock.calls[2][0]).toHaveLength(1);
+ });
});