Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Feature/partner-search: read a partner fan-out page in one query (#4486)

Rafi6d83ae7f97e0
diff --git a/apps/web/lib/api/partners/search/sync.ts b/apps/web/lib/api/partners/search/sync.ts
index 2c5cda7102..5d046ce84d 100644
--- a/apps/web/lib/api/partners/search/sync.ts
+++ b/apps/web/lib/api/partners/search/sync.ts
@@ -69,31 +69,38 @@ export async function syncPartnerSearchDocuments({
};
}
-interface FindPartnerSearchSyncEnrollmentIdsOptions {
+interface SyncPartnersOptions {
partnerIds: string[];
programId?: string;
after?: string;
take?: number;
+ searchProvider?: PartnerSearchProvider | null;
+}
+
+export interface SyncPartnersResult {
+ upserted: number;
+ lastEnrollmentId: string | null;
}
/**
- * One page of enrollment IDs for the given partners, for changes that fan out
+ * One page of documents for the given partners, for changes that fan out
* beyond a single enrollment: a profile or platform edit touches every program
- * the partner is in.
+ * the partner is in. Paged because that fan-out is unbounded. `programId`
+ * narrows it to one enrollment per partner.
*
- * Paged because that fan-out is unbounded. `programId` narrows it to one
- * enrollment per partner.
+ * Upserts only. The rows come from the database, so there is nothing to delete.
*/
-export async function findPartnerSearchSyncEnrollmentIds({
+export async function syncPartnerEnrollments({
partnerIds,
programId,
after,
take = PARTNER_SEARCH_SYNC_BATCH_SIZE,
-}: FindPartnerSearchSyncEnrollmentIdsOptions): Promise<string[]> {
+ searchProvider = getPartnerSearchProvider(),
+}: SyncPartnersOptions): Promise<SyncPartnersResult> {
const ids = unique(partnerIds.filter(Boolean));
- if (ids.length === 0) {
- return [];
+ if (!searchProvider || ids.length === 0) {
+ return { upserted: 0, lastEnrollmentId: null };
}
const enrollments = await prisma.programEnrollment.findMany({
@@ -104,14 +111,21 @@ export async function findPartnerSearchSyncEnrollmentIds({
...(programId && { programId }),
...(after && { id: { gt: after } }),
},
- select: {
- id: true,
- },
+ select: partnerSearchDocumentSelect,
orderBy: {
id: "asc",
},
take,
});
- return enrollments.map(({ id }) => id);
+ if (enrollments.length > 0) {
+ await searchProvider.upsert(
+ enrollments.map(serializePartnerSearchDocument),
+ );
+ }
+
+ return {
+ upserted: enrollments.length,
+ lastEnrollmentId: enrollments.at(-1)?.id ?? null,
+ };
}
diff --git a/apps/web/lib/jobs/handlers/partner-search-sync-job.ts b/apps/web/lib/jobs/handlers/partner-search-sync-job.ts
index 998176bd38..0dbc33648d 100644
--- a/apps/web/lib/jobs/handlers/partner-search-sync-job.ts
+++ b/apps/web/lib/jobs/handlers/partner-search-sync-job.ts
@@ -1,7 +1,7 @@
import {
- findPartnerSearchSyncEnrollmentIds,
getPartnerSearchProvider,
PARTNER_SEARCH_SYNC_BATCH_SIZE,
+ syncPartnerEnrollments,
syncPartnerSearchDocuments,
} from "@/lib/api/partners/search";
import * as z from "zod/v4";
@@ -74,31 +74,27 @@ export const partnerSearchSyncJob = defineJob({
return;
}
- const enrollmentIds = await findPartnerSearchSyncEnrollmentIds({
+ const { upserted, lastEnrollmentId } = await syncPartnerEnrollments({
partnerIds: input.partnerIds,
programId: input.programId,
after: input.after,
take: PARTNER_SEARCH_SYNC_BATCH_SIZE,
+ searchProvider,
});
- if (enrollmentIds.length === 0) {
+ if (upserted === 0) {
return;
}
- const { upserted, deleted } = await syncPartnerSearchDocuments({
- enrollmentIds,
- searchProvider,
- });
-
console.log(
- `[partnerSearchSyncJob] Synced ${upserted} and removed ${deleted} enrollment documents for ${input.partnerIds.length} partner(s).`,
+ `[partnerSearchSyncJob] Synced ${upserted} enrollment documents for ${input.partnerIds.length} partner(s).`,
);
- if (enrollmentIds.length === PARTNER_SEARCH_SYNC_BATCH_SIZE) {
+ if (upserted === PARTNER_SEARCH_SYNC_BATCH_SIZE && lastEnrollmentId) {
await partnerSearchSyncJob.dispatch(
{
...input,
- after: enrollmentIds[enrollmentIds.length - 1],
+ after: lastEnrollmentId,
},
{
delay: 1,
diff --git a/apps/web/tests/partners/partner-search-sync-job.test.ts b/apps/web/tests/partners/partner-search-sync-job.test.ts
index 89062a1266..8906e5f8b0 100644
--- a/apps/web/tests/partners/partner-search-sync-job.test.ts
+++ b/apps/web/tests/partners/partner-search-sync-job.test.ts
@@ -7,14 +7,14 @@ const BATCH_SIZE = 3;
const mocks = vi.hoisted(() => ({
getPartnerSearchProvider: vi.fn(),
syncPartnerSearchDocuments: vi.fn(),
- findPartnerSearchSyncEnrollmentIds: vi.fn(),
+ syncPartnerEnrollments: vi.fn(),
}));
vi.mock("@/lib/api/partners/search", () => ({
PARTNER_SEARCH_SYNC_BATCH_SIZE: 3,
getPartnerSearchProvider: mocks.getPartnerSearchProvider,
syncPartnerSearchDocuments: mocks.syncPartnerSearchDocuments,
- findPartnerSearchSyncEnrollmentIds: mocks.findPartnerSearchSyncEnrollmentIds,
+ syncPartnerEnrollments: mocks.syncPartnerEnrollments,
}));
const searchProvider = { name: "turbopuffer" };
@@ -26,7 +26,9 @@ describe("partnerSearchSyncJob", () => {
mocks.syncPartnerSearchDocuments
.mockReset()
.mockResolvedValue({ upserted: 0, deleted: 0 });
- mocks.findPartnerSearchSyncEnrollmentIds.mockReset().mockResolvedValue([]);
+ mocks.syncPartnerEnrollments
+ .mockReset()
+ .mockResolvedValue({ upserted: 0, lastEnrollmentId: null });
});
it("skips entirely when no provider is configured", async () => {
@@ -38,7 +40,7 @@ describe("partnerSearchSyncJob", () => {
});
expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();
- expect(mocks.findPartnerSearchSyncEnrollmentIds).not.toHaveBeenCalled();
+ expect(mocks.syncPartnerEnrollments).not.toHaveBeenCalled();
});
it("syncs the enrollment ids it is given", async () => {
@@ -51,30 +53,28 @@ describe("partnerSearchSyncJob", () => {
enrollmentIds: ["pge_1", "pge_2"],
searchProvider,
});
- expect(mocks.findPartnerSearchSyncEnrollmentIds).not.toHaveBeenCalled();
+ expect(mocks.syncPartnerEnrollments).not.toHaveBeenCalled();
});
- it("resolves a partner fan-out before syncing", async () => {
- mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([
- "pge_1",
- "pge_2",
- ]);
+ it("syncs a partner fan-out in one read", async () => {
+ mocks.syncPartnerEnrollments.mockResolvedValue({
+ upserted: 2,
+ lastEnrollmentId: "pge_2",
+ });
await partnerSearchSyncJob.execute({
type: "partners",
partnerIds: ["pn_1"],
});
- expect(mocks.findPartnerSearchSyncEnrollmentIds).toHaveBeenCalledWith({
+ expect(mocks.syncPartnerEnrollments).toHaveBeenCalledWith({
partnerIds: ["pn_1"],
programId: undefined,
after: undefined,
take: BATCH_SIZE,
- });
- expect(mocks.syncPartnerSearchDocuments).toHaveBeenCalledWith({
- enrollmentIds: ["pge_1", "pge_2"],
searchProvider,
});
+ expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();
});
it("continues from the last enrollment when a page comes back full", async () => {
@@ -82,11 +82,10 @@ describe("partnerSearchSyncJob", () => {
.spyOn(partnerSearchSyncJob, "dispatch")
.mockResolvedValue({ status: "published", messageId: "msg_1" });
- mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([
- "pge_1",
- "pge_2",
- "pge_3",
- ]);
+ mocks.syncPartnerEnrollments.mockResolvedValue({
+ upserted: BATCH_SIZE,
+ lastEnrollmentId: "pge_3",
+ });
await partnerSearchSyncJob.execute({
type: "partners",
@@ -110,10 +109,10 @@ describe("partnerSearchSyncJob", () => {
.spyOn(partnerSearchSyncJob, "dispatch")
.mockResolvedValue({ status: "published", messageId: "msg_1" });
- mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([
- "pge_1",
- "pge_2",
- ]);
+ mocks.syncPartnerEnrollments.mockResolvedValue({
+ upserted: 2,
+ lastEnrollmentId: "pge_2",
+ });
await partnerSearchSyncJob.execute({
type: "partners",
@@ -128,14 +127,16 @@ describe("partnerSearchSyncJob", () => {
.spyOn(partnerSearchSyncJob, "dispatch")
.mockResolvedValue({ status: "published", messageId: "msg_1" });
- mocks.findPartnerSearchSyncEnrollmentIds.mockResolvedValue([]);
+ mocks.syncPartnerEnrollments.mockResolvedValue({
+ upserted: 0,
+ lastEnrollmentId: null,
+ });
await partnerSearchSyncJob.execute({
type: "partners",
partnerIds: ["pn_1"],
});
- expect(mocks.syncPartnerSearchDocuments).not.toHaveBeenCalled();
expect(dispatch).not.toHaveBeenCalled();
});
diff --git a/apps/web/tests/partners/partner-search-sync.test.ts b/apps/web/tests/partners/partner-search-sync.test.ts
index ec0d0944ad..c75dd074db 100644
--- a/apps/web/tests/partners/partner-search-sync.test.ts
+++ b/apps/web/tests/partners/partner-search-sync.test.ts
@@ -1,5 +1,6 @@
import {
- findPartnerSearchSyncEnrollmentIds,
+ partnerSearchDocumentSelect,
+ syncPartnerEnrollments,
syncPartnerSearchDocuments,
type PartnerSearchDocumentSource,
type PartnerSearchProvider,
@@ -169,31 +170,46 @@ describe("syncPartnerSearchDocuments", () => {
});
});
-describe("findPartnerSearchSyncEnrollmentIds", () => {
+describe("syncPartnerEnrollments", () => {
beforeEach(() => {
mocks.findMany.mockReset();
});
- it("resolves every enrollment a partner has when no program is given", async () => {
- mocks.findMany.mockResolvedValueOnce([{ id: "pge_1" }, { id: "pge_2" }]);
+ it("reads and upserts one page of documents in a single query", async () => {
+ const searchProvider = createProvider();
+ mocks.findMany.mockResolvedValueOnce([
+ createSource("pge_1"),
+ createSource("pge_2"),
+ ]);
- const ids = await findPartnerSearchSyncEnrollmentIds({
+ const result = await syncPartnerEnrollments({
partnerIds: ["pn_1"],
+ searchProvider,
});
- expect(ids).toEqual(["pge_1", "pge_2"]);
-
- const { where, orderBy } = mocks.findMany.mock.calls[0][0];
- expect(where).toEqual({ partnerId: { in: ["pn_1"] } });
- expect(orderBy).toEqual({ id: "asc" });
+ expect(result).toEqual({ upserted: 2, lastEnrollmentId: "pge_2" });
+ expect(mocks.findMany).toHaveBeenCalledTimes(1);
+ expect(mocks.findMany.mock.calls[0][0].where).toEqual({
+ partnerId: { in: ["pn_1"] },
+ });
+ expect(mocks.findMany.mock.calls[0][0]).toMatchObject({
+ select: partnerSearchDocumentSelect,
+ orderBy: { id: "asc" },
+ });
+ expect(
+ vi.mocked(searchProvider.upsert).mock.calls[0][0].map(({ id }) => id),
+ ).toEqual(["pge_1", "pge_2"]);
+ expect(searchProvider.delete).not.toHaveBeenCalled();
});
it("narrows to a single program when one is given", async () => {
- mocks.findMany.mockResolvedValueOnce([{ id: "pge_1" }]);
+ const searchProvider = createProvider();
+ mocks.findMany.mockResolvedValueOnce([createSource("pge_1")]);
- await findPartnerSearchSyncEnrollmentIds({
+ await syncPartnerEnrollments({
partnerIds: ["pn_1"],
programId: "prog_test",
+ searchProvider,
});
expect(mocks.findMany.mock.calls[0][0].where).toEqual({
@@ -203,12 +219,14 @@ describe("findPartnerSearchSyncEnrollmentIds", () => {
});
it("pages past the cursor so a large fan-out can resume", async () => {
- mocks.findMany.mockResolvedValueOnce([{ id: "pge_3" }]);
+ const searchProvider = createProvider();
+ mocks.findMany.mockResolvedValueOnce([createSource("pge_3")]);
- await findPartnerSearchSyncEnrollmentIds({
+ await syncPartnerEnrollments({
partnerIds: ["pn_1"],
after: "pge_2",
take: 2,
+ searchProvider,
});
const { where, take } = mocks.findMany.mock.calls[0][0];
@@ -219,10 +237,28 @@ describe("findPartnerSearchSyncEnrollmentIds", () => {
expect(take).toBe(2);
});
+ it("does not write when the page is empty", async () => {
+ const searchProvider = createProvider();
+ mocks.findMany.mockResolvedValueOnce([]);
+
+ const result = await syncPartnerEnrollments({
+ partnerIds: ["pn_1"],
+ searchProvider,
+ });
+
+ expect(result).toEqual({ upserted: 0, lastEnrollmentId: null });
+ expect(searchProvider.upsert).not.toHaveBeenCalled();
+ });
+
it("does not query when there are no partners", async () => {
- const ids = await findPartnerSearchSyncEnrollmentIds({ partnerIds: [] });
+ const searchProvider = createProvider();
+
+ const result = await syncPartnerEnrollments({
+ partnerIds: [],
+ searchProvider,
+ });
- expect(ids).toEqual([]);
+ expect(result).toEqual({ upserted: 0, lastEnrollmentId: null });
expect(mocks.findMany).not.toHaveBeenCalled();
});
});