The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Export partner search backfill utilities
Rafi4f66259aff8a
diff --git a/apps/web/lib/api/partners/search/backfill.ts b/apps/web/lib/api/partners/search/backfill.tsnew file mode 100644index 0000000000..1f986c2d4b--- /dev/null+++ b/apps/web/lib/api/partners/search/backfill.ts@@ -0,0 +1,83 @@+import { prisma } from "@/lib/prisma";+import { getPartnerSearchProvider } from "./provider";+import {+ partnerSearchDocumentSelect,+ serializePartnerSearchDocument,+} from "./serialize-document";+import type { PartnerSearchProvider } from "./types";++const DEFAULT_BATCH_SIZE = 500;++export interface PartnerSearchBackfillProgress {+ batchSize: number;+ processed: number;+ lastDocumentId: string;+}++interface BackfillPartnerSearchOptions {+ programId: string;+ batchSize?: number;+ after?: string;+ searchProvider?: PartnerSearchProvider | null;+ onProgress?: (progress: PartnerSearchBackfillProgress) => void;+}++export async function backfillPartnerSearch({+ programId,+ batchSize = DEFAULT_BATCH_SIZE,+ after,+ searchProvider = getPartnerSearchProvider(),+ onProgress,+}: BackfillPartnerSearchOptions) {+ if (!searchProvider) {+ throw new Error("Partner search provider is not configured.");+ }++ if (!Number.isSafeInteger(batchSize) || batchSize <= 0) {+ throw new Error("Batch size must be a positive integer.");+ }++ let lastDocumentId = after;+ let processed = 0;++ while (true) {+ const enrollments = await prisma.programEnrollment.findMany({+ where: {+ programId,+ ...(lastDocumentId && {+ id: { gt: lastDocumentId },+ }),+ },+ select: partnerSearchDocumentSelect,+ orderBy: {+ id: "asc",+ },+ take: batchSize,+ });++ if (enrollments.length === 0) {+ break;+ }++ await searchProvider.upsert(+ enrollments.map(serializePartnerSearchDocument),+ );++ lastDocumentId = enrollments[enrollments.length - 1].id;+ processed += enrollments.length;+ onProgress?.({+ batchSize: enrollments.length,+ processed,+ lastDocumentId,+ });++ if (enrollments.length < batchSize) {+ break;+ }+ }++ return {+ processed,+ lastDocumentId: lastDocumentId ?? null,+ };+}diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.tsindex 2981ab2430..38fdd9bc0e 100644--- a/apps/web/lib/api/partners/search/index.ts+++ b/apps/web/lib/api/partners/search/index.ts@@ -1,3 +1,4 @@+export * from "./backfill";export * from "./build-search-query";export * from "./mock-provider";export * from "./order-search-results";diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.tsnew file mode 100644index 0000000000..d127c8ca2b--- /dev/null+++ b/apps/web/scripts/partners/backfill-partner-search.ts@@ -0,0 +1,102 @@+import {+ backfillPartnerSearch,+ type PartnerSearchBackfillProgress,+} from "@/lib/api/partners/search";+import { prisma } from "@/lib/prisma";+import "dotenv-flow/config";++const DEFAULT_BATCH_SIZE = 500;+const MAX_BATCH_SIZE = 1_000;++interface BackfillArguments {+ programId: string;+ batchSize: number;+ after?: string;+}++function parsePositiveInteger(value: string | undefined, flag: string) {+ const parsed = Number(value);++ if (!Number.isSafeInteger(parsed) || parsed <= 0) {+ throw new Error(`${flag} must be a positive integer.`);+ }++ return parsed;+}++function parseArguments(args: string[]): BackfillArguments {+ let programId: string | undefined;+ let batchSize = DEFAULT_BATCH_SIZE;+ let after: string | undefined;++ for (const arg of args) {+ if (arg.startsWith("--programId=")) {+ programId = arg.slice("--programId=".length);+ } else if (arg.startsWith("--batchSize=")) {+ batchSize = parsePositiveInteger(+ arg.slice("--batchSize=".length),+ "--batchSize",+ );+ } else if (arg.startsWith("--after=")) {+ after = arg.slice("--after=".length);+ } else {+ throw new Error(`Unknown argument: ${arg}`);+ }+ }++ if (!programId) {+ throw new Error("--programId is required.");+ }++ if (batchSize > MAX_BATCH_SIZE) {+ throw new Error(`--batchSize cannot exceed ${MAX_BATCH_SIZE}.`);+ }++ if (after === "") {+ throw new Error("--after cannot be empty.");+ }++ return { programId, batchSize, after };+}++function reportProgress({+ batchSize,+ processed,+ lastDocumentId,+}: PartnerSearchBackfillProgress) {+ console.log(+ `Indexed ${processed.toLocaleString()} documents (${batchSize.toLocaleString()} in this batch), last document: ${lastDocumentId}`,+ );+}++async function main() {+ const { programId, batchSize, after } = parseArguments(process.argv.slice(2));++ console.log(`Starting partner search backfill for program ${programId}`);+ console.log(+ `Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,+ );++ const result = await backfillPartnerSearch({+ programId,+ batchSize,+ after,+ onProgress: reportProgress,+ });++ console.log(+ `Partner search backfill complete: ${result.processed.toLocaleString()} documents indexed.`,+ );+ if (result.lastDocumentId) {+ console.log(`Last document: ${result.lastDocumentId}`);+ }+}++main()+ .catch((error) => {+ console.error("Partner search backfill failed:", error);+ process.exit(1);+ })+ .finally(async () => {+ await prisma.$disconnect();+ });diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.tsnew file mode 100644index 0000000000..0f581309b7--- /dev/null+++ b/apps/web/tests/partners/partner-search-backfill.test.ts@@ -0,0 +1,149 @@+import {+ backfillPartnerSearch,+ partnerSearchDocumentSelect,+ type PartnerSearchDocumentSource,+ type PartnerSearchProvider,+} from "@/lib/api/partners/search";+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ findMany: vi.fn(),+}));++vi.mock("@/lib/prisma", () => ({+ prisma: {+ programEnrollment: {+ findMany: mocks.findMany,+ },+ },+}));++function createSource(id: string): PartnerSearchDocumentSource {+ const timestamp = new Date("2026-01-01T00:00:00.000Z");++ return {+ id,+ programId: "prog_test",+ partnerId: `pn_${id}`,+ status: "approved",+ tenantId: null,+ groupId: null,+ totalClicks: 0,+ totalLeads: 0,+ totalConversions: 0,+ totalSaleAmount: BigInt(0),+ totalCommissions: BigInt(0),+ netRevenue: BigInt(0),+ earningsPerClick: 0,+ averageLifetimeValue: null,+ clickToLeadRate: null,+ clickToConversionRate: null,+ leadToConversionRate: null,+ returnOnAdSpend: null,+ createdAt: timestamp,+ updatedAt: timestamp,+ partner: {+ name: "Rafi Hasan",+ email: "[email protected]",+ companyName: "Dub Partners",+ description: "Developer tools educator",+ country: "CA",+ updatedAt: timestamp,+ platforms: [],+ },+ links: [],+ programPartnerTags: [],+ applicationEvent: null,+ };+}++function createProvider(): PartnerSearchProvider {+ return {+ search: vi.fn(),+ count: vi.fn(),+ groupBy: vi.fn(),+ upsert: vi.fn(),+ delete: vi.fn(),+ };+}++describe("backfillPartnerSearch", () => {+ beforeEach(() => {+ mocks.findMany.mockReset();+ });++ it("indexes documents in keyset-paginated batches", async () => {+ const searchProvider = createProvider();+ const onProgress = vi.fn();+ mocks.findMany+ .mockResolvedValueOnce([createSource("pge_1"), createSource("pge_2")])+ .mockResolvedValueOnce([createSource("pge_3")]);++ const result = await backfillPartnerSearch({+ programId: "prog_test",+ batchSize: 2,+ searchProvider,+ onProgress,+ });++ expect(mocks.findMany).toHaveBeenNthCalledWith(1, {+ where: { programId: "prog_test" },+ select: partnerSearchDocumentSelect,+ orderBy: { id: "asc" },+ take: 2,+ });+ expect(mocks.findMany).toHaveBeenNthCalledWith(2, {+ where: {+ programId: "prog_test",+ id: { gt: "pge_2" },+ },+ select: partnerSearchDocumentSelect,+ orderBy: { id: "asc" },+ take: 2,+ });+ expect(searchProvider.upsert).toHaveBeenCalledTimes(2);+ expect(onProgress).toHaveBeenLastCalledWith({+ batchSize: 1,+ processed: 3,+ lastDocumentId: "pge_3",+ });+ expect(result).toEqual({+ processed: 3,+ lastDocumentId: "pge_3",+ });+ });++ it("resumes after a document ID", async () => {+ const searchProvider = createProvider();+ mocks.findMany.mockResolvedValue([]);++ const result = await backfillPartnerSearch({+ programId: "prog_test",+ after: "pge_100",+ searchProvider,+ });++ expect(mocks.findMany).toHaveBeenCalledWith(+ expect.objectContaining({+ where: {+ programId: "prog_test",+ id: { gt: "pge_100" },+ },+ }),+ );+ expect(result).toEqual({+ processed: 0,+ lastDocumentId: "pge_100",+ });+ });++ it("requires a configured provider", async () => {+ await expect(+ backfillPartnerSearch({+ programId: "prog_test",+ }),+ ).rejects.toThrow("Partner search provider is not configured.");++ expect(mocks.findMany).not.toHaveBeenCalled();+ });+});