The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Choose turbopuffer as the primary search provider and remove Upstash Redis
Rafi9616d1d840e8
diff --git a/apps/web/.env.example b/apps/web/.env.exampleindex 3a19674bd6..3d94400d97 100644--- a/apps/web/.env.example+++ b/apps/web/.env.example@@ -23,12 +23,7 @@ PLANETSCALE_DATABASE_URL="http://root:unused@localhost:3900/planetscale"UPSTASH_REDIS_REST_URL=UPSTASH_REDIS_REST_TOKEN=-# Full-text partner search-# Supported values: upstash-redis, turbopuffer-PARTNER_SEARCH_PROVIDER=-PARTNER_SEARCH_INDEX_NAME=partner-search-v1--# For partner search+# Full-text partner search. Unset falls back to the database search path.TURBOPUFFER_API_KEY=# Upstash QStash – required for queues and background jobsdiff --git a/apps/web/lib/api/partners/search/backfill.ts b/apps/web/lib/api/partners/search/backfill.tsindex d9d49e87a8..1f986c2d4b 100644--- a/apps/web/lib/api/partners/search/backfill.ts+++ b/apps/web/lib/api/partners/search/backfill.ts@@ -76,8 +76,6 @@ export async function backfillPartnerSearch({}}- await searchProvider.waitForIndexing();-return {processed,lastDocumentId: lastDocumentId ?? null,diff --git a/apps/web/lib/api/partners/search/provider.ts b/apps/web/lib/api/partners/search/provider.tsindex f1659f6049..0d1c6d692d 100644--- a/apps/web/lib/api/partners/search/provider.ts+++ b/apps/web/lib/api/partners/search/provider.ts@@ -1,54 +1,21 @@import { createTurbopufferPartnerSearchProvider } from "./providers/turbopuffer";-import { createUpstashRedisPartnerSearchProvider } from "./providers/upstash-redis";import type { PartnerSearchProvider } from "./types";-const PARTNER_SEARCH_PROVIDER_NAMES = ["upstash-redis", "turbopuffer"] as const;--export type PartnerSearchProviderName =- (typeof PARTNER_SEARCH_PROVIDER_NAMES)[number];--const providerFactories: Record<- PartnerSearchProviderName,- () => PartnerSearchProvider-> = {- "upstash-redis": createUpstashRedisPartnerSearchProvider,- turbopuffer: createTurbopufferPartnerSearchProvider,-};--const providerCache = new Map<- PartnerSearchProviderName,- PartnerSearchProvider->();--export function getPartnerSearchProviderName(): PartnerSearchProviderName | null {- const provider = process.env.PARTNER_SEARCH_PROVIDER?.trim();-- if (!provider) {- return null;- }-- if ((PARTNER_SEARCH_PROVIDER_NAMES as readonly string[]).includes(provider)) {- return provider as PartnerSearchProviderName;- }-- throw new Error(- `Unsupported partner search provider: ${provider}. Expected one of ${PARTNER_SEARCH_PROVIDER_NAMES.join(", ")}.`,- );-}+// Built once per process. A fresh client per call would create a new HTTP+// connection pool each time, paying a TLS handshake on every search.+let cachedSearchProvider: PartnerSearchProvider | null = null;+/**+ * Null until turbopuffer is configured, which is what keeps this dark: every+ * caller falls back to the database search path when there is no provider, so+ * an unset key costs the wider field coverage rather than the partner list.+ */export function getPartnerSearchProvider(): PartnerSearchProvider | null {- const provider = getPartnerSearchProviderName();-- if (!provider) {+ if (!process.env.TURBOPUFFER_API_KEY?.trim()) {return null;}- let instance = providerCache.get(provider);-- if (!instance) {- instance = providerFactories[provider]();- providerCache.set(provider, instance);- }+ cachedSearchProvider ??= createTurbopufferPartnerSearchProvider();- return instance;+ return cachedSearchProvider;}diff --git a/apps/web/lib/api/partners/search/providers/shared.ts b/apps/web/lib/api/partners/search/providers/shared.tsindex 1bf3ed866f..88bb72f592 100644--- a/apps/web/lib/api/partners/search/providers/shared.ts+++ b/apps/web/lib/api/partners/search/providers/shared.ts@@ -44,15 +44,3 @@ export function getQueryNgrams(query: string): string[] {return toNgrams(query);}--/**- * First non-blank wins. `||` rather than `??` because a blank value has to fall- * through to the default, otherwise an empty env var silently targets an index- * named "".- */-export function resolveIndexName(- candidates: (string | undefined)[],- fallback: string,-): string {- return candidates.map((value) => value?.trim()).find(Boolean) || fallback;-}diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.tsindex 50f923fca5..c23ed6e72b 100644--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts@@ -8,9 +8,15 @@ import {import type { PartnerSearchDocument, PartnerSearchProvider } from "../types";import { validatePartnerSearchCandidateLimit } from "../types";import { withQueryDeadline, withTransientRetry } from "./resilience";-import { getEmailNgrams, getQueryNgrams, resolveIndexName } from "./shared";+import { getEmailNgrams, getQueryNgrams } from "./shared";-const DEFAULT_NAMESPACE = "partner-search-v2";+/**+ * Bumped whenever the document shape changes, because turbopuffer keeps the+ * schema a namespace was created with — writing a new one does not migrate an+ * existing namespace. A new version is backfilled alongside the old one and+ * swapped in, rather than rebuilt in place.+ */+const NAMESPACE = "partner-search-v2";const WRITE_BATCH_SIZE = 500;const QUERY_OPERATION_TIMEOUT_MS = 1_000;@@ -84,10 +90,7 @@ export interface TurbopufferNamespace {}function getNamespaceName(namespaceName?: string): string {- return resolveIndexName(- [namespaceName, process.env.PARTNER_SEARCH_INDEX_NAME],- DEFAULT_NAMESPACE,- );+ return namespaceName?.trim() || NAMESPACE;}function createNamespace(namespaceName: string): TurbopufferNamespace {@@ -259,11 +262,6 @@ export function createTurbopufferPartnerSearchProvider({return { hits };},- async waitForIndexing() {- // Turbopuffer writes are read-after-write consistent — a committed write- // is immediately queryable, so there is no indexing lag to wait on.- },-async upsert(documents) {for (const documentBatch of chunk(documents, WRITE_BATCH_SIZE)) {await withTransientRetry(() =>diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.tsdeleted file mode 100644index ee0c85259e..0000000000--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts+++ /dev/null@@ -1,333 +0,0 @@-import { chunk } from "@dub/utils";-import {- Redis,- s,- type InferFilterFromSchema,- type SearchIndex,-} from "@upstash/redis";-import {- getPartnerSearchableValues,- normalizePartnerSearchQuery,-} from "../searchable-values";-import type {- PartnerSearchCandidateQuery,- PartnerSearchDocument,- PartnerSearchProvider,-} from "../types";-import { validatePartnerSearchCandidateLimit } from "../types";-import {- withDeadline,- withQueryDeadline,- withTransientRetry,-} from "./resilience";-import { getEmailNgrams, getQueryNgrams, resolveIndexName } from "./shared";--const DEFAULT_INDEX_NAME = "partner-search-v1";-// Keep the full query operation within the one-second latency target while-// leaving a small window to retry failures that return quickly-const QUERY_REQUEST_TIMEOUT_MS = 900;-const QUERY_OPERATION_TIMEOUT_MS = 1_000;-const WRITE_REQUEST_TIMEOUT_MS = 10_000;-const WRITE_BATCH_SIZE = 100;-const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;--export const upstashPartnerSearchSchema = s.object({- programId: s.keyword(),- searchText: s.string().noStem(),- emailNgrams: s.string().noStem(),-});--type UpstashPartnerSearchSchema = typeof upstashPartnerSearchSchema;-type UpstashPartnerSearchFilter =- InferFilterFromSchema<UpstashPartnerSearchSchema>;-type UpstashPartnerSearchIndex = SearchIndex<UpstashPartnerSearchSchema>;--interface UpstashPartnerSearchDocument extends Record<string, unknown> {- programId: string;- searchText: string;- emailNgrams: string;-}--interface CreateUpstashRedisPartnerSearchProviderOptions {- redisClient?: Redis;- queryRedisClient?: Redis;- indexName?: string;-}--function getIndexName(indexName?: string): string {- return resolveIndexName(- [indexName, process.env.PARTNER_SEARCH_INDEX_NAME],- DEFAULT_INDEX_NAME,- );-}--function getDocumentPrefix(indexName: string): string {- return `${indexName}:partner:`;-}--function getDocumentKey(indexName: string, documentId: string): string {- return `${getDocumentPrefix(indexName)}${documentId}`;-}--function createRedisClient(requestTimeoutMs: number): Redis {- const url = process.env.UPSTASH_REDIS_REST_URL;- const token = process.env.UPSTASH_REDIS_REST_TOKEN;-- if (!url || !token) {- throw new Error(- "UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN are required for partner search.",- );- }-- return new Redis({- url,- token,- // The provider wrapper owns the retry budget- retry: { retries: 0 },- signal: () => AbortSignal.timeout(requestTimeoutMs),- });-}--function serializeUpstashDocument(- document: PartnerSearchDocument,-): UpstashPartnerSearchDocument {- return {- programId: document.programId,- searchText: getPartnerSearchableValues(document)- .map(normalizePartnerSearchQuery)- .join(" "),- emailNgrams: getEmailNgrams(document.email),- };-}--function buildTextAlternatives(query: string): UpstashPartnerSearchFilter[][] {- // $smart handles word-boundary and prefix matching via the inverted index.- // Email n-grams handle infix/substring matching (e.g. "examp" in "[email protected]").- // A $regex path was intentionally omitted because regex queries scan the- // inverted index linearly, which degrades p99 latency at 100K+ documents.- const alternatives: UpstashPartnerSearchFilter[][] = [- [{ searchText: { $smart: query } }],- ];-- const emailNgrams = getQueryNgrams(query);- if (emailNgrams.length > 0) {- // Require exact trigrams so smart matching cannot fuzzy-match unrelated emails- alternatives.push(- emailNgrams.map((ngram) => ({- emailNgrams: { $eq: ngram },- })),- );- }-- return alternatives;-}--function buildUpstashFilter({- programId,- query,-}: Pick<- PartnerSearchCandidateQuery,- "programId" | "query"->): UpstashPartnerSearchFilter {- // Each branch includes the program scope because a root-level $must would- // turn the sibling $should clauses into optional score boosts. Keeping both- // requirements in every branch prevents unrelated program documents from- // being returned with low scores.- return {- $should: buildTextAlternatives(normalizePartnerSearchQuery(query)).map(- (textConditions) => ({- $must: [{ programId: { $eq: programId } }, ...textConditions],- }),- ),- } as UpstashPartnerSearchFilter;-}--function getSchemaSignature(schema: Record<string, Record<string, unknown>>) {- return JSON.stringify(- Object.fromEntries(- Object.entries(schema)- .sort(([left], [right]) => left.localeCompare(right))- .map(([field, options]) => [- field,- Object.fromEntries(- Object.entries(options).sort(([left], [right]) =>- left.localeCompare(right),- ),- ),- ]),- ),- );-}--function validateIndexDescription(- description: NonNullable<- Awaited<ReturnType<UpstashPartnerSearchIndex["describe"]>>- >,- indexName: string,-) {- const expectedPrefix = getDocumentPrefix(indexName);- const hasExpectedPrefix =- description.prefixes.length === 1 &&- description.prefixes[0] === expectedPrefix;- const hasExpectedSchema =- getSchemaSignature(- description.schema as Record<string, Record<string, unknown>>,- ) ===- getSchemaSignature(- upstashPartnerSearchSchema as Record<string, Record<string, unknown>>,- );-- if (- description.dataType !== "json" ||- !hasExpectedPrefix ||- !hasExpectedSchema- ) {- throw new Error(- `Partner search index ${indexName} does not match the configured schema. Delete and recreate it before backfilling.`,- );- }-}--function getUpsertEntries(- indexName: string,- documents: PartnerSearchDocument[],-) {- return documents.map((document) => ({- key: getDocumentKey(indexName, document.id),- path: "$",- value: serializeUpstashDocument(document),- }));-}--async function upsertDocumentBatch(- redisClient: Redis,- indexName: string,- documents: PartnerSearchDocument[],-) {- const upsertEntries = getUpsertEntries(indexName, documents);-- await withTransientRetry(() => redisClient.json.mset(...upsertEntries));-}--async function deleteDocumentBatch(- redisClient: Redis,- indexName: string,- documentIds: string[],-) {- await withTransientRetry(() =>- redisClient.del(- ...documentIds.map((documentId) => getDocumentKey(indexName, documentId)),- ),- );-}--export async function createUpstashRedisPartnerSearchIndex({- redisClient,- indexName,-}: CreateUpstashRedisPartnerSearchProviderOptions = {}) {- const resolvedRedisClient =- redisClient ?? createRedisClient(WRITE_REQUEST_TIMEOUT_MS);- const resolvedIndexName = getIndexName(indexName);-- const index = await resolvedRedisClient.search.createIndex({- name: resolvedIndexName,- dataType: "json",- prefix: getDocumentPrefix(resolvedIndexName),- schema: upstashPartnerSearchSchema,- skipInitialScan: true,- existsOk: true,- });- const description = await index.describe();-- if (!description) {- throw new Error(- `Partner search index ${resolvedIndexName} was not created.`,- );- }-- validateIndexDescription(description, resolvedIndexName);- // Hand back the description so callers can report on the index without- // paying for a second describe() round trip.- return { index, description };-}--export function createUpstashRedisPartnerSearchProvider({- redisClient,- queryRedisClient,- indexName,-}: CreateUpstashRedisPartnerSearchProviderOptions = {}): PartnerSearchProvider {- const resolvedRedisClient =- redisClient ?? createRedisClient(WRITE_REQUEST_TIMEOUT_MS);- const resolvedQueryRedisClient =- queryRedisClient ??- redisClient ??- createRedisClient(QUERY_REQUEST_TIMEOUT_MS);- const resolvedIndexName = getIndexName(indexName);- const queryIndex: UpstashPartnerSearchIndex =- resolvedQueryRedisClient.search.index({- name: resolvedIndexName,- schema: upstashPartnerSearchSchema,- });- const writeIndex: UpstashPartnerSearchIndex =- resolvedRedisClient.search.index({- name: resolvedIndexName,- schema: upstashPartnerSearchSchema,- });-- async function findCandidates({- programId,- query,- limit,- }: PartnerSearchCandidateQuery) {- validatePartnerSearchCandidateLimit(limit);- const filter = buildUpstashFilter({ programId, query });- const results = await withQueryDeadline(- () => queryIndex.query({ filter, limit, select: {} }),- QUERY_OPERATION_TIMEOUT_MS,- );- const hits = results.map(({ key, score }) => ({- id: key.slice(getDocumentPrefix(resolvedIndexName).length),- score,- }));-- return { hits };- }-- return {- searchCandidates: findCandidates,-- async waitForIndexing() {- const result = await withDeadline(- () => withTransientRetry(() => writeIndex.waitIndexing()),- WAIT_FOR_INDEXING_TIMEOUT_MS,- `Partner search waitForIndexing timed out after ${WAIT_FOR_INDEXING_TIMEOUT_MS}ms.`,- );-- if (result === 0) {- throw new Error(- `Partner search index ${resolvedIndexName} was not found.`,- );- }- },-- async upsert(documents) {- for (const documentBatch of chunk(documents, WRITE_BATCH_SIZE)) {- await upsertDocumentBatch(- resolvedRedisClient,- resolvedIndexName,- documentBatch,- );- }- },-- async delete(documentIds) {- for (const documentIdBatch of chunk(documentIds, WRITE_BATCH_SIZE)) {- await deleteDocumentBatch(- resolvedRedisClient,- resolvedIndexName,- documentIdBatch,- );- }- },- };-}diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.tsindex 98ce65ef94..2a1f4b3f3f 100644--- a/apps/web/lib/api/partners/search/types.ts+++ b/apps/web/lib/api/partners/search/types.ts@@ -77,7 +77,6 @@ export interface PartnerSearchProvider {searchCandidates(query: PartnerSearchCandidateQuery,): Promise<PartnerSearchResult>;- waitForIndexing(): Promise<void>;upsert(documents: PartnerSearchDocument[]): Promise<void>;delete(documentIds: string[]): Promise<void>;}diff --git a/apps/web/package.json b/apps/web/package.jsonindex 308e676570..d6892612b2 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -60,7 +60,7 @@"@types/buffer-crc32": "0.2.0","@upstash/qstash": "^2.11.0","@upstash/ratelimit": "^2.0.6",- "@upstash/redis": "^1.38.2",+ "@upstash/redis": "^1.35.3","@upstash/vector": "^1.2.2","@upstash/workflow": "^1.3.0","@vercel/edge-config": "^0.4.1",diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.tsindex 0ea053318c..b2b2e75bb8 100644--- a/apps/web/scripts/dev/benchmark-partner-search.ts+++ b/apps/web/scripts/dev/benchmark-partner-search.ts@@ -21,14 +21,13 @@* pnpm run script dev/benchmark-partner-search --programId=prog_123 [--requests=1000]* [--sampleSize=100] [--concurrency=10] [--warmup=50] [--thresholdMs=1000] [--searchOnly]*- * Requires PARTNER_SEARCH_PROVIDER and an index backfilled for the program:+ * Requires TURBOPUFFER_API_KEY and an index backfilled for the program:* pnpm run script partners/backfill-partner-search --programId=prog_123*/import { getPartners } from "@/lib/api/partners/get-partners";import {getPartnerSearchProvider,- getPartnerSearchProviderName,PARTNER_SEARCH_CANDIDATE_LIMIT,partnerSearchDocumentSelect,serializePartnerSearchDocument,@@ -370,7 +369,6 @@ async function runWithConcurrency<T>(async function main() {const options = parseArguments(process.argv.slice(2));const searchProvider = getPartnerSearchProvider();- const providerName = getPartnerSearchProviderName();if (!searchProvider) {throw new Error(@@ -451,7 +449,6 @@ async function main() {};console.log(`Partner search benchmark for program ${options.programId}`);- console.log(`Provider: ${providerName}`);console.log(`${partnerCount.toLocaleString()} program partners`);console.log(`${options.requests.toLocaleString()} measured requests, ${options.warmupRequests.toLocaleString()} warm-up requests, concurrency ${options.concurrency}`,diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.tsindex caca3938fd..42bf99ea25 100644--- a/apps/web/scripts/partners/backfill-partner-search.ts+++ b/apps/web/scripts/partners/backfill-partner-search.ts@@ -13,15 +13,14 @@* pnpm run script partners/backfill-partner-search --programId=prog_123* [--batchSize=500] [--after=pge_123]*- * Requires PARTNER_SEARCH_PROVIDER to be configured.+ * Requires TURBOPUFFER_API_KEY to be configured.*/import {backfillPartnerSearch,- getPartnerSearchProviderName,+ getPartnerSearchProvider,type PartnerSearchBackfillProgress,} from "@/lib/api/partners/search";-import { createUpstashRedisPartnerSearchIndex } from "@/lib/api/partners/search/providers/upstash-redis";import { prisma } from "@/lib/prisma";import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";import "dotenv-flow/config";@@ -102,15 +101,8 @@ function createProgressReporter(totalDocuments: number, batchSize: number) {async function main() {const { programId, batchSize, after } = parseArguments(process.argv.slice(2));resumeAfter = after;- const providerName = getPartnerSearchProviderName();- if (!providerName) {- throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");- }-- // Redis Search indexes have an explicit schema and must exist before the- // backfill starts. Turbopuffer creates its namespace on the first upsert.- if (providerName === "upstash-redis") {- await createUpstashRedisPartnerSearchIndex();+ if (!getPartnerSearchProvider()) {+ throw new Error("TURBOPUFFER_API_KEY is not configured.");}// Same where-clause the backfill pages with, so chunk totals line up.@@ -122,7 +114,6 @@ async function main() {});console.log(`Starting partner search backfill for program ${programId}`);- console.log(`Provider: ${providerName}`);console.log(`Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,);diff --git a/apps/web/scripts/partners/delete-partner-search-index.ts b/apps/web/scripts/partners/delete-partner-search-index.tsindex e74c1b0b4f..e811f1f9f7 100644--- a/apps/web/scripts/partners/delete-partner-search-index.ts+++ b/apps/web/scripts/partners/delete-partner-search-index.ts@@ -1,13 +1,20 @@-import { getPartnerSearchProviderName } from "@/lib/api/partners/search";+/**+ * Empties a partner search namespace.+ *+ * Takes the namespace explicitly rather than reading the one the code writes to,+ * so retiring an old version after a migration is possible — and requires it+ * twice, because this is not recoverable without a backfill.+ *+ * cd apps/web+ * pnpm run script partners/delete-partner-search-index+ * --indexName=partner-search-v2 --confirm=partner-search-v2+ *+ * Requires TURBOPUFFER_API_KEY.+ */+import { deleteTurbopufferPartnerSearchNamespace } from "@/lib/api/partners/search/providers/turbopuffer";-import { chunk } from "@dub/utils";-import { Redis } from "@upstash/redis";import "dotenv-flow/config";-const DELETE_BATCH_SIZE = 1_000;-const SCAN_COUNT = 10_000;-const MAX_DELETE_PASSES = 3;-interface DeletePartnerSearchIndexArguments {indexName: string;}@@ -39,107 +46,16 @@ function parseArguments(args: string[]): DeletePartnerSearchIndexArguments {return { indexName };}-function createRedisClient() {- const url = process.env.UPSTASH_REDIS_REST_URL;- const token = process.env.UPSTASH_REDIS_REST_TOKEN;-- if (!url || !token) {- throw new Error(- "UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN are required.",- );- }-- return new Redis({ url, token });-}--async function deleteDocumentPass(redis: Redis, documentPattern: string) {- let cursor = "0";- let deleted = 0;-- do {- const [nextCursor, keys] = await redis.scan(cursor, {- match: documentPattern,- count: SCAN_COUNT,- });-- for (const keyBatch of chunk(keys, DELETE_BATCH_SIZE)) {- deleted += await redis.del(...keyBatch);- }-- cursor = nextCursor;- } while (cursor !== "0");-- return deleted;-}--async function deleteDocuments(redis: Redis, indexName: string) {- const documentPattern = `${indexName}:partner:*`;- let totalDeleted = 0;-- // Repeat the scan to verify that no matching keys remain after deletion- for (let pass = 1; pass <= MAX_DELETE_PASSES; pass++) {- const deleted = await deleteDocumentPass(redis, documentPattern);- totalDeleted += deleted;-- console.log(- `Deletion pass ${pass}: ${deleted.toLocaleString()} documents removed`,- );-- if (deleted === 0) {- return totalDeleted;- }- }-- throw new Error(- `Documents matching ${documentPattern} are still being created. Stop application processes using this index and run the script again.`,- );-}-async function main() {const { indexName } = parseArguments(process.argv.slice(2));- const providerName = getPartnerSearchProviderName();- if (!providerName) {- throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");- }-- if (providerName === "turbopuffer") {- await deleteTurbopufferPartnerSearchNamespace({ namespaceName: indexName });- console.log(`Partner search cleanup complete: emptied ${indexName}.`);- return;- }- const redis = createRedisClient();- const index = redis.search.index({ name: indexName });- const description = await index.describe();- const expectedPrefix = `${indexName}:partner:`;-- // Verify that the index contains only partner search documents- if (!description) {- console.warn(- `Index ${indexName} does not exist, so its prefix cannot be verified. Removing any documents left under ${expectedPrefix} only.`,- );- } else if (- description.prefixes.length !== 1 ||- description.prefixes[0] !== expectedPrefix- ) {- throw new Error(- `Index ${indexName} does not use an expected partner search prefix.`,- );+ if (!process.env.TURBOPUFFER_API_KEY?.trim()) {+ throw new Error("TURBOPUFFER_API_KEY is not configured.");}- // Drop the search index without affecting unrelated Redis data- const dropped = await index.drop();- console.log(- dropped === 1- ? `Dropped partner search index ${indexName}`- : `Partner search index ${indexName} did not exist`,- );+ await deleteTurbopufferPartnerSearchNamespace({ namespaceName: indexName });- // Step 3: Delete the partner documents stored for this index- const deleted = await deleteDocuments(redis, indexName);- console.log(- `Partner search cleanup complete: ${deleted.toLocaleString()} documents removed.`,- );+ console.log(`Partner search cleanup complete: emptied ${indexName}.`);}main().catch((error) => {diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.tsindex 4b6e78b99d..14e2487928 100644--- a/apps/web/tests/partners/get-partners-count-search.test.ts+++ b/apps/web/tests/partners/get-partners-count-search.test.ts@@ -28,7 +28,6 @@ function createSearchProvider(): PartnerSearchProvider {searchCandidates: vi.fn().mockResolvedValue({hits: [{ id: "pge_2" }, { id: "pge_1" }],}),- waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),};diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.tsindex dc164e0a51..f63cd0f733 100644--- a/apps/web/tests/partners/get-partners-search.test.ts+++ b/apps/web/tests/partners/get-partners-search.test.ts@@ -37,7 +37,6 @@ function createSearchProvider(): PartnerSearchProvider {return {searchCandidates: vi.fn().mockResolvedValue({ hits }),- waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),};diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.tsindex 732030193c..ecd599901d 100644--- a/apps/web/tests/partners/partner-search-backfill.test.ts+++ b/apps/web/tests/partners/partner-search-backfill.test.ts@@ -37,7 +37,6 @@ function createSource(id: string): PartnerSearchDocumentSource {function createProvider(): PartnerSearchProvider {return {searchCandidates: vi.fn(),- waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),};@@ -78,7 +77,6 @@ describe("backfillPartnerSearch", () => {take: 2,});expect(searchProvider.upsert).toHaveBeenCalledTimes(2);- expect(searchProvider.waitForIndexing).toHaveBeenCalledOnce();expect(onProgress).toHaveBeenLastCalledWith({batchSize: 1,processed: 3,diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.tsindex aa363b38c6..1a4915604f 100644--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts@@ -349,11 +349,6 @@ describe("Turbopuffer partner search provider", () => {});});- it("does not wait for indexing, which turbopuffer does not need", async () => {- await expect(createProvider().waitForIndexing()).resolves.toBeUndefined();- expect(mocks.write).not.toHaveBeenCalled();- });-it("empties the namespace", async () => {mocks.deleteAll.mockResolvedValue({});@@ -366,8 +361,7 @@ describe("Turbopuffer partner search provider", () => {expect(result).toEqual({ namespaceName: "test-namespace" });});- it("falls back to the default namespace when the env var is blank", async () => {- vi.stubEnv("PARTNER_SEARCH_INDEX_NAME", "");+ it("uses the pinned namespace when none is passed", async () => {mocks.deleteAll.mockResolvedValue({});const { namespaceName } = await deleteTurbopufferPartnerSearchNamespace({diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.tsdeleted file mode 100644index 5ba751c7e1..0000000000--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts+++ /dev/null@@ -1,426 +0,0 @@-import {- PARTNER_SEARCH_CANDIDATE_LIMIT,- type PartnerSearchDocument,-} from "@/lib/api/partners/search";-import {- createUpstashRedisPartnerSearchIndex,- createUpstashRedisPartnerSearchProvider,- upstashPartnerSearchSchema,-} from "@/lib/api/partners/search/providers/upstash-redis";-import type { Redis } from "@upstash/redis";-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";--const document: PartnerSearchDocument = {- id: "pge_test",- programId: "prog_test",- partnerId: "pn_test",- name: "Rafi Hasan",- email: "[email protected]",- companyName: "Dub Partners",- description: "Developer tools educator",- platformTypes: ["website", "youtube", "twitter"],- platformIdentifiers: ["rafi.dev", "@rafi-youtube", "@rafi-on-x"],- linkDomains: ["dub.sh"],- linkKeys: ["rafi"],- shortLinks: ["https://dub.sh/rafi"],- destinationUrls: ["https://example.com/referrals/rafi"],-};--const mocks = vi.hoisted(() => ({- createIndex: vi.fn(),- del: vi.fn(),- describe: vi.fn(),- index: vi.fn(),- jsonMset: vi.fn(),- query: vi.fn(),- waitIndexing: vi.fn(),-}));--function createRedisMock(): Redis {- const searchIndex = {- describe: mocks.describe,- query: mocks.query,- waitIndexing: mocks.waitIndexing,- };-- mocks.index.mockReturnValue(searchIndex);- mocks.createIndex.mockReturnValue(searchIndex);-- const redisMock = {- del: mocks.del,- json: {- mset: mocks.jsonMset,- },- search: {- createIndex: mocks.createIndex,- index: mocks.index,- },- } as unknown as Redis;-- return redisMock;-}--describe("Upstash Redis partner search provider", () => {- beforeEach(() => {- vi.clearAllMocks();- mocks.del.mockResolvedValue(0);- mocks.describe.mockResolvedValue({- name: "test-index",- dataType: "json",- prefixes: ["test-index:partner:"],- schema: upstashPartnerSearchSchema,- });- mocks.jsonMset.mockResolvedValue("OK");- mocks.query.mockResolvedValue([]);- mocks.waitIndexing.mockResolvedValue(1);- });-- // In afterEach rather than the test body, so a failed assertion cannot leak- // a stubbed env var into later tests.- afterEach(() => {- vi.unstubAllEnvs();- });-- it("retrieves bounded relevance candidates without business filters", async () => {- mocks.query.mockResolvedValue([- {- key: "test-index:partner:pge_test",- score: 4,- data: {},- },- ]);- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await expect(- provider.searchCandidates({- programId: document.programId,- query: "rafi",- limit: PARTNER_SEARCH_CANDIDATE_LIMIT,- }),- ).resolves.toEqual({- hits: [{ id: document.id, score: 4 }],- });-- expect(mocks.query).toHaveBeenCalledWith(- expect.objectContaining({- limit: PARTNER_SEARCH_CANDIDATE_LIMIT,- select: {},- }),- );- const request = mocks.query.mock.calls[0]![0];- expect(request.offset).toBeUndefined();- expect(request.orderBy).toBeUndefined();- expect(JSON.stringify(request.filter)).toContain(- '"programId":{"$eq":"prog_test"}',- );- });-- it("requires program-scoped text or partial email matches", async () => {- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await provider.searchCandidates({- programId: document.programId,- query: "examp",- limit: 25,- });-- const filter = mocks.query.mock.calls[0]![0].filter;- expect(filter.$must).toBeUndefined();- expect(filter.$should).toHaveLength(2);- for (const branch of filter.$should) {- expect(branch.$must).toEqual(- expect.arrayContaining([{ programId: { $eq: "prog_test" } }]),- );- }- expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"exa"}');- expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"xam"}');- expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"amp"}');- });-- it("retries a transient provider error", async () => {- mocks.query- .mockRejectedValueOnce(new Error("Upstash returned 503"))- .mockResolvedValueOnce([]);- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await expect(- provider.searchCandidates({- programId: document.programId,- query: "rafi",- limit: 10,- }),- ).resolves.toEqual({ hits: [] });- expect(mocks.query).toHaveBeenCalledTimes(2);- });-- it("does not retry a permanent provider error", async () => {- mocks.query.mockRejectedValue(new Error("Invalid search filter"));- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await expect(- provider.searchCandidates({- programId: document.programId,- query: "rafi",- limit: 10,- }),- ).rejects.toThrow("Invalid search filter");- expect(mocks.query).toHaveBeenCalledTimes(1);- });-- it("retries a network TypeError", async () => {- // undici surfaces network failures as `TypeError: fetch failed`.- mocks.query- .mockRejectedValueOnce(new TypeError("fetch failed"))- .mockResolvedValueOnce([]);- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await expect(- provider.searchCandidates({- programId: document.programId,- query: "rafi",- limit: 10,- }),- ).resolves.toEqual({ hits: [] });- expect(mocks.query).toHaveBeenCalledTimes(2);- });-- it("does not retry a programming TypeError", async () => {- mocks.query.mockRejectedValue(- new TypeError("Cannot read properties of undefined (reading 'query')"),- );- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await expect(- provider.searchCandidates({- programId: document.programId,- query: "rafi",- limit: 10,- }),- ).rejects.toThrow("Cannot read properties of undefined");- expect(mocks.query).toHaveBeenCalledTimes(1);- });-- it("does not retry a request timeout after it consumes the query budget", async () => {- mocks.query.mockRejectedValue(- new DOMException(- "The operation was aborted due to timeout",- "TimeoutError",- ),- );- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await expect(- provider.searchCandidates({- programId: document.programId,- query: "rafi",- limit: 10,- }),- ).rejects.toThrow("The operation was aborted due to timeout");- expect(mocks.query).toHaveBeenCalledTimes(1);- });-- it("allows query latency within the one-second deadline", async () => {- vi.useFakeTimers();- mocks.query.mockImplementation(- () => new Promise((resolve) => setTimeout(() => resolve([]), 500)),- );- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- try {- const result = provider.searchCandidates({- programId: document.programId,- query: "rafi",- limit: 10,- });- await vi.advanceTimersByTimeAsync(500);-- await expect(result).resolves.toEqual({ hits: [] });- expect(vi.getTimerCount()).toBe(0);- } finally {- vi.useRealTimers();- }- });-- it("bounds the total query operation to one second", async () => {- vi.useFakeTimers();- mocks.query.mockImplementation(() => new Promise(() => {}));- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- try {- const result = expect(- provider.searchCandidates({- programId: document.programId,- query: "rafi",- limit: 10,- }),- ).rejects.toThrow("Partner search query timed out after 1000ms");- await vi.advanceTimersByTimeAsync(1_000);-- await result;- expect(vi.getTimerCount()).toBe(0);- } finally {- vi.useRealTimers();- }- });-- it("writes one lean search document per enrollment", async () => {- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await provider.upsert([document]);-- const entries = mocks.jsonMset.mock.calls[0]!;- expect(entries).toHaveLength(1);- expect(entries[0]).toEqual({- key: "test-index:partner:pge_test",- path: "$",- value: {- programId: "prog_test",- searchText: expect.any(String),- emailNgrams: expect.any(String),- },- });- expect(entries[0].value.searchText).toContain("[email protected]");- expect(entries[0].value.searchText).toContain("dub partners");- expect(entries[0].value.searchText).toContain("rafi-on-x");- expect(entries[0].value.searchText).toContain("referrals/rafi");- expect(entries[0].value.emailNgrams).toContain("exa");- expect(entries[0].value).not.toHaveProperty("partnerId");- expect(entries[0].value).not.toHaveProperty("status");- expect(entries[0].value).not.toHaveProperty("partnerTagIds");- expect(entries[0].value).not.toHaveProperty("totalSaleAmount");- });-- it("bounds Upstash write batches", async () => {- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });- const documents = Array.from({ length: 101 }, (_, index) => ({- ...document,- id: `pge_${index}`,- }));-- await provider.upsert(documents);-- expect(mocks.jsonMset).toHaveBeenCalledTimes(2);- expect(mocks.jsonMset.mock.calls[0]).toHaveLength(100);- expect(mocks.jsonMset.mock.calls[1]).toHaveLength(1);- });-- it("deletes the enrollment document directly", async () => {- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await provider.delete([document.id]);-- expect(mocks.del).toHaveBeenCalledWith("test-index:partner:pge_test");- });-- it("waits for pending index updates", async () => {- vi.useFakeTimers();- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- try {- await provider.waitForIndexing();-- expect(mocks.waitIndexing).toHaveBeenCalledOnce();- expect(vi.getTimerCount()).toBe(0);- } finally {- vi.useRealTimers();- }- });-- it("creates the index explicitly with the partner-search prefix", async () => {- const redisClient = createRedisMock();-- await createUpstashRedisPartnerSearchIndex({- redisClient,- indexName: "test-index",- });-- expect(mocks.createIndex).toHaveBeenCalledWith(- expect.objectContaining({- name: "test-index",- dataType: "json",- prefix: "test-index:partner:",- existsOk: true,- skipInitialScan: true,- }),- );- expect(Object.keys(upstashPartnerSearchSchema).sort()).toEqual([- "emailNgrams",- "programId",- "searchText",- ]);- });-- it("rejects an existing index with a stale schema", async () => {- mocks.describe.mockResolvedValue({- name: "test-index",- dataType: "json",- prefixes: ["test-index:partner:"],- schema: { id: { type: "KEYWORD" } },- });-- await expect(- createUpstashRedisPartnerSearchIndex({- redisClient: createRedisMock(),- indexName: "test-index",- }),- ).rejects.toThrow("Delete and recreate it before backfilling");- });-- it.each([- ["blank", ""],- ["whitespace", " "],- ])(- "falls back to the default index when the env var is %s",- async (_l, value) => {- vi.stubEnv("PARTNER_SEARCH_INDEX_NAME", value);- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- });-- await provider.upsert([document]);-- expect(mocks.jsonMset).toHaveBeenCalledWith(- expect.objectContaining({ key: "partner-search-v1:partner:pge_test" }),- );- },- );-});