The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Support provider-aware partner search index management
Rafi1e117565ff22
diff --git a/apps/web/lib/api/partners/search/provider.ts b/apps/web/lib/api/partners/search/provider.tsindex eef9233cb0..47a2bd01f1 100644--- a/apps/web/lib/api/partners/search/provider.ts+++ b/apps/web/lib/api/partners/search/provider.ts@@ -5,13 +5,29 @@ import type { PartnerSearchProvider } from "./types";let upstashRedisProvider: PartnerSearchProvider | null = null;let upstashSearchProvider: PartnerSearchProvider | null = null;-export function getPartnerSearchProvider(): PartnerSearchProvider | null {+export type PartnerSearchProviderName = "upstash-redis" | "upstash-search";++export function getPartnerSearchProviderName(): PartnerSearchProviderName | null {const provider = process.env.PARTNER_SEARCH_PROVIDER?.trim();if (!provider) {return null;}+ if (provider === "upstash-redis" || provider === "upstash-search") {+ return provider;+ }++ throw new Error(`Unsupported partner search provider: ${provider}`);+}++export function getPartnerSearchProvider(): PartnerSearchProvider | null {+ const provider = getPartnerSearchProviderName();++ if (!provider) {+ return null;+ }+if (provider === "upstash-redis") {if (!upstashRedisProvider) {upstashRedisProvider = createUpstashRedisPartnerSearchProvider();diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.tsindex c73ab1b49f..5fd79f70d1 100644--- a/apps/web/lib/api/partners/search/providers/upstash-search.ts+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts@@ -66,6 +66,7 @@ interface UpstashSearchIndexClient {): Promise<string>;delete(documentIds: string[]): Promise<{ deleted: number }>;info(): Promise<{ pendingDocumentCount: number; documentCount: number }>;+ reset(): Promise<string>;}interface CreateUpstashSearchPartnerSearchProviderOptions {@@ -274,6 +275,22 @@ function logPartnerSearchDebug(console.log(`[Partner Search Debug] Upstash Search ${operation}`, details);}+export async function resetUpstashSearchPartnerSearchIndex({+ searchIndex,+ indexName,+}: CreateUpstashSearchPartnerSearchProviderOptions = {}) {+ const resolvedIndexName = getIndexName(indexName);+ const index = searchIndex ?? createSearchIndex(resolvedIndexName);+ const { documentCount } = await withTransientRetry(() => index.info());++ // Upstash Search reset clears the whole index in one operation. It is the+ // appropriate way to remove documents from an obsolete index shape before+ // a full backfill, instead of issuing one delete request per document.+ await withTransientRetry(() => index.reset());++ return { indexName: resolvedIndexName, documentCount };+}+export function createUpstashSearchPartnerSearchProvider({searchIndex,indexName,diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.tsindex a75652dbd4..d893e8b29e 100644--- a/apps/web/scripts/dev/benchmark-partner-search.ts+++ b/apps/web/scripts/dev/benchmark-partner-search.ts@@ -1,6 +1,8 @@import { getPartners } from "@/lib/api/partners/get-partners";import {getPartnerSearchProvider,+ getPartnerSearchProviderName,+ PARTNER_SEARCH_CANDIDATE_LIMIT,partnerSearchDocumentSelect,serializePartnerSearchDocument,type PartnerSearchDocument,@@ -89,8 +91,10 @@ function parseArguments(args: string[]): BenchmarkArguments {if (concurrency > requests) {throw new Error("--concurrency cannot exceed --requests.");}- if (pageSize > 100) {- throw new Error("--pageSize cannot exceed 100.");+ if (pageSize > PARTNER_SEARCH_CANDIDATE_LIMIT) {+ throw new Error(+ `--pageSize cannot exceed ${PARTNER_SEARCH_CANDIDATE_LIMIT}.`,+ );}return {@@ -235,6 +239,7 @@ async function runWithConcurrency<T>(async function main() {const options = parseArguments(process.argv.slice(2));const searchProvider = getPartnerSearchProvider();+ const providerName = getPartnerSearchProviderName();if (!searchProvider) {throw new Error(@@ -287,7 +292,8 @@ async function main() {};console.log(`Partner search benchmark for program ${options.programId}`);- console.log(`${partnerCount.toLocaleString()} indexed partners`);+ console.log(`Provider: ${providerName}`);+ console.log(`${partnerCount.toLocaleString()} program partners`);console.log(`${options.requests.toLocaleString()} measured requests, ${options.warmupRequests.toLocaleString()} warm-up requests, concurrency ${options.concurrency}`,);diff --git a/apps/web/scripts/dev/debug-partner-search.ts b/apps/web/scripts/dev/debug-partner-search.tsindex de6d4ca948..11dde7ee8a 100644--- a/apps/web/scripts/dev/debug-partner-search.ts+++ b/apps/web/scripts/dev/debug-partner-search.ts@@ -1,13 +1,13 @@import {getPartnerSearchableValues,getPartnerSearchProvider,+ getPartnerSearchProviderName,normalizePartnerSearchQuery,PARTNER_SEARCH_CANDIDATE_LIMIT,partnerSearchDocumentSelect,serializePartnerSearchDocument,type PartnerSearchDocument,type PartnerSearchHit,- type PartnerSearchSortField,} from "@/lib/api/partners/search";import { prisma } from "@/lib/prisma";import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";@@ -15,28 +15,12 @@ import { ProgramEnrollmentStatus } from "@prisma/client";import "dotenv-flow/config";const DEFAULT_LIMIT = 10;-const MAX_LIMIT = 100;-const SORT_FIELDS: PartnerSearchSortField[] = [- "createdAt",- "totalClicks",- "totalLeads",- "totalConversions",- "totalSaleAmount",- "totalCommissions",- "netRevenue",- "earningsPerClick",- "averageLifetimeValue",- "clickToLeadRate",- "clickToConversionRate",- "leadToConversionRate",- "returnOnAdSpend",-];+const MAX_LIMIT = PARTNER_SEARCH_CANDIDATE_LIMIT;interface DebugArguments {programId: string;query: string;limit: number;- sortBy: PartnerSearchSortField;status?: ProgramEnrollmentStatus;}@@ -44,7 +28,6 @@ function parseArguments(args: string[]): DebugArguments {let programId: string | undefined;let query: string | undefined;let limit = DEFAULT_LIMIT;- let sortBy: PartnerSearchSortField = "totalSaleAmount";let status: ProgramEnrollmentStatus | undefined;for (const arg of args) {@@ -54,12 +37,6 @@ function parseArguments(args: string[]): DebugArguments {query = arg.slice("--query=".length);} else if (arg.startsWith("--limit=")) {limit = parsePositiveInteger(arg.slice("--limit=".length), "--limit");- } else if (arg.startsWith("--sortBy=")) {- const value = arg.slice("--sortBy=".length) as PartnerSearchSortField;- if (!SORT_FIELDS.includes(value)) {- throw new Error(`--sortBy must be one of: ${SORT_FIELDS.join(", ")}.`);- }- sortBy = value;} else if (arg.startsWith("--status=")) {const value = arg.slice("--status=".length) as ProgramEnrollmentStatus;if (!Object.values(ProgramEnrollmentStatus).includes(value)) {@@ -83,7 +60,7 @@ function parseArguments(args: string[]): DebugArguments {throw new Error(`--limit cannot exceed ${MAX_LIMIT}.`);}- return { programId, query, limit, sortBy, status };+ return { programId, query, limit, status };}async function getDatabaseDocuments(@@ -122,13 +99,11 @@ function reportResults({label,hits,databaseDocuments,- sortBy,normalizedQuery,}: {label: string;hits: PartnerSearchHit[];databaseDocuments: Awaited<ReturnType<typeof getDatabaseDocuments>>;- sortBy: PartnerSearchSortField;normalizedQuery: string;}) {console.log(`\n${label}`);@@ -139,7 +114,6 @@ function reportResults({return {rank: index + 1,providerScore: hit.score,- databaseSortValue: databaseDocument?.[sortBy] ?? null,containsLiteralQuery: containsLiteralQuery(databaseDocument,normalizedQuery,@@ -155,7 +129,7 @@ function reportResults({}async function main() {- const { programId, query, limit, sortBy, status } = parseArguments(+ const { programId, query, limit, status } = parseArguments(process.argv.slice(2),);const searchProvider = getPartnerSearchProvider();@@ -163,7 +137,7 @@ async function main() {throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");}- const providerName = process.env.PARTNER_SEARCH_PROVIDER?.trim();+ const providerName = getPartnerSearchProviderName();const startedAt = performance.now();// Step 1: Fetch the same relevance candidates used by the website@@ -202,7 +176,6 @@ async function main() {label: "Provider relevance order",hits: filteredHits,databaseDocuments,- sortBy,normalizedQuery: normalizePartnerSearchQuery(query),});}diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.tsindex b320067885..83a7d29be3 100644--- a/apps/web/scripts/partners/backfill-partner-search.ts+++ b/apps/web/scripts/partners/backfill-partner-search.ts@@ -1,5 +1,7 @@import {backfillPartnerSearch,+ createUpstashRedisPartnerSearchIndex,+ getPartnerSearchProviderName,type PartnerSearchBackfillProgress,} from "@/lib/api/partners/search";import { prisma } from "@/lib/prisma";@@ -62,8 +64,19 @@ function reportProgress({async function main() {const { programId, batchSize, after } = parseArguments(process.argv.slice(2));+ const providerName = getPartnerSearchProviderName();+ if (!providerName) {+ throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");+ }++ // Redis Search indexes have an explicit schema and must exist before the+ // backfill starts. Upstash Search creates its index on the first upsert.+ if (providerName === "upstash-redis") {+ await createUpstashRedisPartnerSearchIndex();+ }console.log(`Starting partner search backfill for program ${programId}`);+ console.log(`Provider: ${providerName}`);console.log(`Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,);diff --git a/apps/web/scripts/partners/create-partner-search-index.ts b/apps/web/scripts/partners/create-partner-search-index.tsindex 2ac281af5a..a98a8931b4 100644--- a/apps/web/scripts/partners/create-partner-search-index.ts+++ b/apps/web/scripts/partners/create-partner-search-index.ts@@ -1,7 +1,22 @@-import { createUpstashRedisPartnerSearchIndex } from "@/lib/api/partners/search";+import {+ createUpstashRedisPartnerSearchIndex,+ getPartnerSearchProviderName,+} from "@/lib/api/partners/search";import "dotenv-flow/config";async function main() {+ const providerName = getPartnerSearchProviderName();+ if (!providerName) {+ throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");+ }++ if (providerName === "upstash-search") {+ console.log(+ "Upstash Search creates the partner search index on the first backfill upsert.",+ );+ return;+ }+const index = await createUpstashRedisPartnerSearchIndex();const description = await index.describe();diff --git a/apps/web/scripts/partners/delete-partner-search-index.ts b/apps/web/scripts/partners/delete-partner-search-index.tsindex 57f4390fef..a0446d2813 100644--- a/apps/web/scripts/partners/delete-partner-search-index.ts+++ b/apps/web/scripts/partners/delete-partner-search-index.ts@@ -1,3 +1,7 @@+import {+ getPartnerSearchProviderName,+ resetUpstashSearchPartnerSearchIndex,+} from "@/lib/api/partners/search";import { chunk } from "@dub/utils";import { Redis } from "@upstash/redis";import "dotenv-flow/config";@@ -96,19 +100,35 @@ async function deleteDocuments(redis: Redis, indexName: string) {async function main() {const { indexName } = parseArguments(process.argv.slice(2));+ const providerName = getPartnerSearchProviderName();+ if (!providerName) {+ throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");+ }++ if (providerName === "upstash-search") {+ const { documentCount } = await resetUpstashSearchPartnerSearchIndex({+ indexName,+ });+ console.log(+ `Partner search cleanup complete: ${documentCount.toLocaleString()} documents removed from ${indexName}.`,+ );+ return;+ }+const redis = createRedisClient();const index = redis.search.index({ name: indexName });const description = await index.describe();- const expectedPrefix = `${indexName}:`;+ const expectedPrefixes = [`${indexName}:`, `${indexName}:partner:`];- // Step 1: Verify that an existing index owns only the requested key prefix+ // Step 1: Accept the earlier broad prefix and the current lean prefix. Both+ // are contained by the index namespace deleted below.if (description &&(description.prefixes.length !== 1 ||- description.prefixes[0] !== expectedPrefix)+ !expectedPrefixes.includes(description.prefixes[0]))) {throw new Error(- `Index ${indexName} does not use the expected ${expectedPrefix} prefix.`,+ `Index ${indexName} does not use an expected partner search prefix.`,);}@@ -120,7 +140,8 @@ async function main() {: `Partner search index ${indexName} did not exist`,);- // Step 3: Delete partner and tag documents stored under this index namespace+ // Step 3: Delete current partner documents and any obsolete documents left+ // under the same index namespace by an earlier index shapeconst deleted = await deleteDocuments(redis, indexName);console.log(`Partner search cleanup complete: ${deleted.toLocaleString()} documents removed.`,diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.tsindex 7dd39c58b5..172d951912 100644--- a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts@@ -1,5 +1,6 @@import {createUpstashSearchPartnerSearchProvider,+ resetUpstashSearchPartnerSearchIndex,type PartnerSearchDocument,} from "@/lib/api/partners/search";import { beforeEach, describe, expect, it, vi } from "vitest";@@ -43,6 +44,7 @@ const document: PartnerSearchDocument = {const mocks = vi.hoisted(() => ({delete: vi.fn(),info: vi.fn(),+ reset: vi.fn(),search: vi.fn(),upsert: vi.fn(),}));@@ -51,6 +53,7 @@ function createSearchIndexMock() {return {delete: mocks.delete,info: mocks.info,+ reset: mocks.reset,search: mocks.search,upsert: mocks.upsert,};@@ -91,6 +94,7 @@ describe("Upstash Search partner search provider", () => {documentCount: 0,pendingDocumentCount: 0,});+ mocks.reset.mockResolvedValue("Success");mocks.search.mockResolvedValue([]);mocks.upsert.mockResolvedValue("Success");});@@ -272,4 +276,22 @@ describe("Upstash Search partner search provider", () => {vi.useRealTimers();}});++ it("resets every document in the index in one operation", async () => {+ mocks.info.mockResolvedValue({+ documentCount: 100_005,+ pendingDocumentCount: 0,+ });++ await expect(+ resetUpstashSearchPartnerSearchIndex({+ searchIndex: createSearchIndexMock(),+ indexName: "test-index",+ }),+ ).resolves.toEqual({+ indexName: "test-index",+ documentCount: 100_005,+ });+ expect(mocks.reset).toHaveBeenCalledOnce();+ });});