Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Export partner search backfill utilities

Rafi4f66259aff8a
diff --git a/apps/web/lib/api/partners/search/backfill.ts b/apps/web/lib/api/partners/search/backfill.ts
new file mode 100644
index 0000000000..1f986c2d4b
--- /dev/null
+++ b/apps/web/lib/api/partners/search/backfill.ts
@@ -0,0 +1,83 @@
+import { prisma } from "@/lib/prisma";
+import { getPartnerSearchProvider } from "./provider";
+import {
+ partnerSearchDocumentSelect,
+ serializePartnerSearchDocument,
+} from "./serialize-document";
+import type { PartnerSearchProvider } from "./types";
+
+const DEFAULT_BATCH_SIZE = 500;
+
+export interface PartnerSearchBackfillProgress {
+ batchSize: number;
+ processed: number;
+ lastDocumentId: string;
+}
+
+interface BackfillPartnerSearchOptions {
+ programId: string;
+ batchSize?: number;
+ after?: string;
+ searchProvider?: PartnerSearchProvider | null;
+ onProgress?: (progress: PartnerSearchBackfillProgress) => void;
+}
+
+export async function backfillPartnerSearch({
+ programId,
+ batchSize = DEFAULT_BATCH_SIZE,
+ after,
+ searchProvider = getPartnerSearchProvider(),
+ onProgress,
+}: BackfillPartnerSearchOptions) {
+ if (!searchProvider) {
+ throw new Error("Partner search provider is not configured.");
+ }
+
+ if (!Number.isSafeInteger(batchSize) || batchSize <= 0) {
+ throw new Error("Batch size must be a positive integer.");
+ }
+
+ let lastDocumentId = after;
+ let processed = 0;
+
+ while (true) {
+ const enrollments = await prisma.programEnrollment.findMany({
+ where: {
+ programId,
+ ...(lastDocumentId && {
+ id: { gt: lastDocumentId },
+ }),
+ },
+ select: partnerSearchDocumentSelect,
+ orderBy: {
+ id: "asc",
+ },
+ take: batchSize,
+ });
+
+ if (enrollments.length === 0) {
+ break;
+ }
+
+ await searchProvider.upsert(
+ enrollments.map(serializePartnerSearchDocument),
+ );
+
+ lastDocumentId = enrollments[enrollments.length - 1].id;
+ processed += enrollments.length;
+ onProgress?.({
+ batchSize: enrollments.length,
+ processed,
+ lastDocumentId,
+ });
+
+ if (enrollments.length < batchSize) {
+ break;
+ }
+ }
+
+ return {
+ processed,
+ lastDocumentId: lastDocumentId ?? null,
+ };
+}
diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.ts
index 2981ab2430..38fdd9bc0e 100644
--- a/apps/web/lib/api/partners/search/index.ts
+++ b/apps/web/lib/api/partners/search/index.ts
@@ -1,3 +1,4 @@
+export * from "./backfill";
export * from "./build-search-query";
export * from "./mock-provider";
export * from "./order-search-results";
diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.ts
new file mode 100644
index 0000000000..d127c8ca2b
--- /dev/null
+++ b/apps/web/scripts/partners/backfill-partner-search.ts
@@ -0,0 +1,102 @@
+import {
+ backfillPartnerSearch,
+ type PartnerSearchBackfillProgress,
+} from "@/lib/api/partners/search";
+import { prisma } from "@/lib/prisma";
+import "dotenv-flow/config";
+
+const DEFAULT_BATCH_SIZE = 500;
+const MAX_BATCH_SIZE = 1_000;
+
+interface BackfillArguments {
+ programId: string;
+ batchSize: number;
+ after?: string;
+}
+
+function parsePositiveInteger(value: string | undefined, flag: string) {
+ const parsed = Number(value);
+
+ if (!Number.isSafeInteger(parsed) || parsed <= 0) {
+ throw new Error(`${flag} must be a positive integer.`);
+ }
+
+ return parsed;
+}
+
+function parseArguments(args: string[]): BackfillArguments {
+ let programId: string | undefined;
+ let batchSize = DEFAULT_BATCH_SIZE;
+ let after: string | undefined;
+
+ for (const arg of args) {
+ if (arg.startsWith("--programId=")) {
+ programId = arg.slice("--programId=".length);
+ } else if (arg.startsWith("--batchSize=")) {
+ batchSize = parsePositiveInteger(
+ arg.slice("--batchSize=".length),
+ "--batchSize",
+ );
+ } else if (arg.startsWith("--after=")) {
+ after = arg.slice("--after=".length);
+ } else {
+ throw new Error(`Unknown argument: ${arg}`);
+ }
+ }
+
+ if (!programId) {
+ throw new Error("--programId is required.");
+ }
+
+ if (batchSize > MAX_BATCH_SIZE) {
+ throw new Error(`--batchSize cannot exceed ${MAX_BATCH_SIZE}.`);
+ }
+
+ if (after === "") {
+ throw new Error("--after cannot be empty.");
+ }
+
+ return { programId, batchSize, after };
+}
+
+function reportProgress({
+ batchSize,
+ processed,
+ lastDocumentId,
+}: PartnerSearchBackfillProgress) {
+ console.log(
+ `Indexed ${processed.toLocaleString()} documents (${batchSize.toLocaleString()} in this batch), last document: ${lastDocumentId}`,
+ );
+}
+
+async function main() {
+ const { programId, batchSize, after } = parseArguments(process.argv.slice(2));
+
+ console.log(`Starting partner search backfill for program ${programId}`);
+ console.log(
+ `Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,
+ );
+
+ const result = await backfillPartnerSearch({
+ programId,
+ batchSize,
+ after,
+ onProgress: reportProgress,
+ });
+
+ console.log(
+ `Partner search backfill complete: ${result.processed.toLocaleString()} documents indexed.`,
+ );
+ if (result.lastDocumentId) {
+ console.log(`Last document: ${result.lastDocumentId}`);
+ }
+}
+
+main()
+ .catch((error) => {
+ console.error("Partner search backfill failed:", error);
+ process.exit(1);
+ })
+ .finally(async () => {
+ await prisma.$disconnect();
+ });
diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.ts
new file mode 100644
index 0000000000..0f581309b7
--- /dev/null
+++ b/apps/web/tests/partners/partner-search-backfill.test.ts
@@ -0,0 +1,149 @@
+import {
+ backfillPartnerSearch,
+ partnerSearchDocumentSelect,
+ type PartnerSearchDocumentSource,
+ type PartnerSearchProvider,
+} from "@/lib/api/partners/search";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ findMany: vi.fn(),
+}));
+
+vi.mock("@/lib/prisma", () => ({
+ prisma: {
+ programEnrollment: {
+ findMany: mocks.findMany,
+ },
+ },
+}));
+
+function createSource(id: string): PartnerSearchDocumentSource {
+ const timestamp = new Date("2026-01-01T00:00:00.000Z");
+
+ return {
+ id,
+ programId: "prog_test",
+ partnerId: `pn_${id}`,
+ status: "approved",
+ tenantId: null,
+ groupId: null,
+ totalClicks: 0,
+ totalLeads: 0,
+ totalConversions: 0,
+ totalSaleAmount: BigInt(0),
+ totalCommissions: BigInt(0),
+ netRevenue: BigInt(0),
+ earningsPerClick: 0,
+ averageLifetimeValue: null,
+ clickToLeadRate: null,
+ clickToConversionRate: null,
+ leadToConversionRate: null,
+ returnOnAdSpend: null,
+ createdAt: timestamp,
+ updatedAt: timestamp,
+ partner: {
+ name: "Rafi Hasan",
+ email: "[email protected]",
+ companyName: "Dub Partners",
+ description: "Developer tools educator",
+ country: "CA",
+ updatedAt: timestamp,
+ platforms: [],
+ },
+ links: [],
+ programPartnerTags: [],
+ applicationEvent: null,
+ };
+}
+
+function createProvider(): PartnerSearchProvider {
+ return {
+ search: vi.fn(),
+ count: vi.fn(),
+ groupBy: vi.fn(),
+ upsert: vi.fn(),
+ delete: vi.fn(),
+ };
+}
+
+describe("backfillPartnerSearch", () => {
+ beforeEach(() => {
+ mocks.findMany.mockReset();
+ });
+
+ it("indexes documents in keyset-paginated batches", async () => {
+ const searchProvider = createProvider();
+ const onProgress = vi.fn();
+ mocks.findMany
+ .mockResolvedValueOnce([createSource("pge_1"), createSource("pge_2")])
+ .mockResolvedValueOnce([createSource("pge_3")]);
+
+ const result = await backfillPartnerSearch({
+ programId: "prog_test",
+ batchSize: 2,
+ searchProvider,
+ onProgress,
+ });
+
+ expect(mocks.findMany).toHaveBeenNthCalledWith(1, {
+ where: { programId: "prog_test" },
+ select: partnerSearchDocumentSelect,
+ orderBy: { id: "asc" },
+ take: 2,
+ });
+ expect(mocks.findMany).toHaveBeenNthCalledWith(2, {
+ where: {
+ programId: "prog_test",
+ id: { gt: "pge_2" },
+ },
+ select: partnerSearchDocumentSelect,
+ orderBy: { id: "asc" },
+ take: 2,
+ });
+ expect(searchProvider.upsert).toHaveBeenCalledTimes(2);
+ expect(onProgress).toHaveBeenLastCalledWith({
+ batchSize: 1,
+ processed: 3,
+ lastDocumentId: "pge_3",
+ });
+ expect(result).toEqual({
+ processed: 3,
+ lastDocumentId: "pge_3",
+ });
+ });
+
+ it("resumes after a document ID", async () => {
+ const searchProvider = createProvider();
+ mocks.findMany.mockResolvedValue([]);
+
+ const result = await backfillPartnerSearch({
+ programId: "prog_test",
+ after: "pge_100",
+ searchProvider,
+ });
+
+ expect(mocks.findMany).toHaveBeenCalledWith(
+ expect.objectContaining({
+ where: {
+ programId: "prog_test",
+ id: { gt: "pge_100" },
+ },
+ }),
+ );
+ expect(result).toEqual({
+ processed: 0,
+ lastDocumentId: "pge_100",
+ });
+ });
+
+ it("requires a configured provider", async () => {
+ await expect(
+ backfillPartnerSearch({
+ programId: "prog_test",
+ }),
+ ).rejects.toThrow("Partner search provider is not configured.");
+
+ expect(mocks.findMany).not.toHaveBeenCalled();
+ });
+});