Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Choose turbopuffer as the primary search provider and remove Upstash Redis

Rafi9616d1d840e8
diff --git a/apps/web/.env.example b/apps/web/.env.example
index 3a19674bd6..3d94400d97 100644
--- a/apps/web/.env.example
+++ b/apps/web/.env.example
@@ -23,12 +23,7 @@ PLANETSCALE_DATABASE_URL="http://root:unused@localhost:3900/planetscale"
UPSTASH_REDIS_REST_URL=
UPSTASH_REDIS_REST_TOKEN=
-# Full-text partner search
-# Supported values: upstash-redis, turbopuffer
-PARTNER_SEARCH_PROVIDER=
-PARTNER_SEARCH_INDEX_NAME=partner-search-v1
-
-# For partner search
+# Full-text partner search. Unset falls back to the database search path.
TURBOPUFFER_API_KEY=
# Upstash QStash – required for queues and background jobs
diff --git a/apps/web/lib/api/partners/search/backfill.ts b/apps/web/lib/api/partners/search/backfill.ts
index d9d49e87a8..1f986c2d4b 100644
--- a/apps/web/lib/api/partners/search/backfill.ts
+++ b/apps/web/lib/api/partners/search/backfill.ts
@@ -76,8 +76,6 @@ export async function backfillPartnerSearch({
}
}
- await searchProvider.waitForIndexing();
-
return {
processed,
lastDocumentId: lastDocumentId ?? null,
diff --git a/apps/web/lib/api/partners/search/provider.ts b/apps/web/lib/api/partners/search/provider.ts
index f1659f6049..0d1c6d692d 100644
--- a/apps/web/lib/api/partners/search/provider.ts
+++ b/apps/web/lib/api/partners/search/provider.ts
@@ -1,54 +1,21 @@
import { createTurbopufferPartnerSearchProvider } from "./providers/turbopuffer";
-import { createUpstashRedisPartnerSearchProvider } from "./providers/upstash-redis";
import type { PartnerSearchProvider } from "./types";
-const PARTNER_SEARCH_PROVIDER_NAMES = ["upstash-redis", "turbopuffer"] as const;
-
-export type PartnerSearchProviderName =
- (typeof PARTNER_SEARCH_PROVIDER_NAMES)[number];
-
-const providerFactories: Record<
- PartnerSearchProviderName,
- () => PartnerSearchProvider
-> = {
- "upstash-redis": createUpstashRedisPartnerSearchProvider,
- turbopuffer: createTurbopufferPartnerSearchProvider,
-};
-
-const providerCache = new Map<
- PartnerSearchProviderName,
- PartnerSearchProvider
->();
-
-export function getPartnerSearchProviderName(): PartnerSearchProviderName | null {
- const provider = process.env.PARTNER_SEARCH_PROVIDER?.trim();
-
- if (!provider) {
- return null;
- }
-
- if ((PARTNER_SEARCH_PROVIDER_NAMES as readonly string[]).includes(provider)) {
- return provider as PartnerSearchProviderName;
- }
-
- throw new Error(
- `Unsupported partner search provider: ${provider}. Expected one of ${PARTNER_SEARCH_PROVIDER_NAMES.join(", ")}.`,
- );
-}
+// Built once per process. A fresh client per call would create a new HTTP
+// connection pool each time, paying a TLS handshake on every search.
+let cachedSearchProvider: PartnerSearchProvider | null = null;
+/**
+ * Null until turbopuffer is configured, which is what keeps this dark: every
+ * caller falls back to the database search path when there is no provider, so
+ * an unset key costs the wider field coverage rather than the partner list.
+ */
export function getPartnerSearchProvider(): PartnerSearchProvider | null {
- const provider = getPartnerSearchProviderName();
-
- if (!provider) {
+ if (!process.env.TURBOPUFFER_API_KEY?.trim()) {
return null;
}
- let instance = providerCache.get(provider);
-
- if (!instance) {
- instance = providerFactories[provider]();
- providerCache.set(provider, instance);
- }
+ cachedSearchProvider ??= createTurbopufferPartnerSearchProvider();
- return instance;
+ return cachedSearchProvider;
}
diff --git a/apps/web/lib/api/partners/search/providers/shared.ts b/apps/web/lib/api/partners/search/providers/shared.ts
index 1bf3ed866f..88bb72f592 100644
--- a/apps/web/lib/api/partners/search/providers/shared.ts
+++ b/apps/web/lib/api/partners/search/providers/shared.ts
@@ -44,15 +44,3 @@ export function getQueryNgrams(query: string): string[] {
return toNgrams(query);
}
-
-/**
- * First non-blank wins. `||` rather than `??` because a blank value has to fall
- * through to the default, otherwise an empty env var silently targets an index
- * named "".
- */
-export function resolveIndexName(
- candidates: (string | undefined)[],
- fallback: string,
-): string {
- return candidates.map((value) => value?.trim()).find(Boolean) || fallback;
-}
diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
index 50f923fca5..c23ed6e72b 100644
--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts
+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
@@ -8,9 +8,15 @@ import {
import type { PartnerSearchDocument, PartnerSearchProvider } from "../types";
import { validatePartnerSearchCandidateLimit } from "../types";
import { withQueryDeadline, withTransientRetry } from "./resilience";
-import { getEmailNgrams, getQueryNgrams, resolveIndexName } from "./shared";
+import { getEmailNgrams, getQueryNgrams } from "./shared";
-const DEFAULT_NAMESPACE = "partner-search-v2";
+/**
+ * Bumped whenever the document shape changes, because turbopuffer keeps the
+ * schema a namespace was created with — writing a new one does not migrate an
+ * existing namespace. A new version is backfilled alongside the old one and
+ * swapped in, rather than rebuilt in place.
+ */
+const NAMESPACE = "partner-search-v2";
const WRITE_BATCH_SIZE = 500;
const QUERY_OPERATION_TIMEOUT_MS = 1_000;
@@ -84,10 +90,7 @@ export interface TurbopufferNamespace {
}
function getNamespaceName(namespaceName?: string): string {
- return resolveIndexName(
- [namespaceName, process.env.PARTNER_SEARCH_INDEX_NAME],
- DEFAULT_NAMESPACE,
- );
+ return namespaceName?.trim() || NAMESPACE;
}
function createNamespace(namespaceName: string): TurbopufferNamespace {
@@ -259,11 +262,6 @@ export function createTurbopufferPartnerSearchProvider({
return { hits };
},
- async waitForIndexing() {
- // Turbopuffer writes are read-after-write consistent — a committed write
- // is immediately queryable, so there is no indexing lag to wait on.
- },
-
async upsert(documents) {
for (const documentBatch of chunk(documents, WRITE_BATCH_SIZE)) {
await withTransientRetry(() =>
diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
deleted file mode 100644
index ee0c85259e..0000000000
--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts
+++ /dev/null
@@ -1,333 +0,0 @@
-import { chunk } from "@dub/utils";
-import {
- Redis,
- s,
- type InferFilterFromSchema,
- type SearchIndex,
-} from "@upstash/redis";
-import {
- getPartnerSearchableValues,
- normalizePartnerSearchQuery,
-} from "../searchable-values";
-import type {
- PartnerSearchCandidateQuery,
- PartnerSearchDocument,
- PartnerSearchProvider,
-} from "../types";
-import { validatePartnerSearchCandidateLimit } from "../types";
-import {
- withDeadline,
- withQueryDeadline,
- withTransientRetry,
-} from "./resilience";
-import { getEmailNgrams, getQueryNgrams, resolveIndexName } from "./shared";
-
-const DEFAULT_INDEX_NAME = "partner-search-v1";
-// Keep the full query operation within the one-second latency target while
-// leaving a small window to retry failures that return quickly
-const QUERY_REQUEST_TIMEOUT_MS = 900;
-const QUERY_OPERATION_TIMEOUT_MS = 1_000;
-const WRITE_REQUEST_TIMEOUT_MS = 10_000;
-const WRITE_BATCH_SIZE = 100;
-const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;
-
-export const upstashPartnerSearchSchema = s.object({
- programId: s.keyword(),
- searchText: s.string().noStem(),
- emailNgrams: s.string().noStem(),
-});
-
-type UpstashPartnerSearchSchema = typeof upstashPartnerSearchSchema;
-type UpstashPartnerSearchFilter =
- InferFilterFromSchema<UpstashPartnerSearchSchema>;
-type UpstashPartnerSearchIndex = SearchIndex<UpstashPartnerSearchSchema>;
-
-interface UpstashPartnerSearchDocument extends Record<string, unknown> {
- programId: string;
- searchText: string;
- emailNgrams: string;
-}
-
-interface CreateUpstashRedisPartnerSearchProviderOptions {
- redisClient?: Redis;
- queryRedisClient?: Redis;
- indexName?: string;
-}
-
-function getIndexName(indexName?: string): string {
- return resolveIndexName(
- [indexName, process.env.PARTNER_SEARCH_INDEX_NAME],
- DEFAULT_INDEX_NAME,
- );
-}
-
-function getDocumentPrefix(indexName: string): string {
- return `${indexName}:partner:`;
-}
-
-function getDocumentKey(indexName: string, documentId: string): string {
- return `${getDocumentPrefix(indexName)}${documentId}`;
-}
-
-function createRedisClient(requestTimeoutMs: number): Redis {
- const url = process.env.UPSTASH_REDIS_REST_URL;
- const token = process.env.UPSTASH_REDIS_REST_TOKEN;
-
- if (!url || !token) {
- throw new Error(
- "UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN are required for partner search.",
- );
- }
-
- return new Redis({
- url,
- token,
- // The provider wrapper owns the retry budget
- retry: { retries: 0 },
- signal: () => AbortSignal.timeout(requestTimeoutMs),
- });
-}
-
-function serializeUpstashDocument(
- document: PartnerSearchDocument,
-): UpstashPartnerSearchDocument {
- return {
- programId: document.programId,
- searchText: getPartnerSearchableValues(document)
- .map(normalizePartnerSearchQuery)
- .join(" "),
- emailNgrams: getEmailNgrams(document.email),
- };
-}
-
-function buildTextAlternatives(query: string): UpstashPartnerSearchFilter[][] {
- // $smart handles word-boundary and prefix matching via the inverted index.
- // Email n-grams handle infix/substring matching (e.g. "examp" in "[email protected]").
- // A $regex path was intentionally omitted because regex queries scan the
- // inverted index linearly, which degrades p99 latency at 100K+ documents.
- const alternatives: UpstashPartnerSearchFilter[][] = [
- [{ searchText: { $smart: query } }],
- ];
-
- const emailNgrams = getQueryNgrams(query);
- if (emailNgrams.length > 0) {
- // Require exact trigrams so smart matching cannot fuzzy-match unrelated emails
- alternatives.push(
- emailNgrams.map((ngram) => ({
- emailNgrams: { $eq: ngram },
- })),
- );
- }
-
- return alternatives;
-}
-
-function buildUpstashFilter({
- programId,
- query,
-}: Pick<
- PartnerSearchCandidateQuery,
- "programId" | "query"
->): UpstashPartnerSearchFilter {
- // Each branch includes the program scope because a root-level $must would
- // turn the sibling $should clauses into optional score boosts. Keeping both
- // requirements in every branch prevents unrelated program documents from
- // being returned with low scores.
- return {
- $should: buildTextAlternatives(normalizePartnerSearchQuery(query)).map(
- (textConditions) => ({
- $must: [{ programId: { $eq: programId } }, ...textConditions],
- }),
- ),
- } as UpstashPartnerSearchFilter;
-}
-
-function getSchemaSignature(schema: Record<string, Record<string, unknown>>) {
- return JSON.stringify(
- Object.fromEntries(
- Object.entries(schema)
- .sort(([left], [right]) => left.localeCompare(right))
- .map(([field, options]) => [
- field,
- Object.fromEntries(
- Object.entries(options).sort(([left], [right]) =>
- left.localeCompare(right),
- ),
- ),
- ]),
- ),
- );
-}
-
-function validateIndexDescription(
- description: NonNullable<
- Awaited<ReturnType<UpstashPartnerSearchIndex["describe"]>>
- >,
- indexName: string,
-) {
- const expectedPrefix = getDocumentPrefix(indexName);
- const hasExpectedPrefix =
- description.prefixes.length === 1 &&
- description.prefixes[0] === expectedPrefix;
- const hasExpectedSchema =
- getSchemaSignature(
- description.schema as Record<string, Record<string, unknown>>,
- ) ===
- getSchemaSignature(
- upstashPartnerSearchSchema as Record<string, Record<string, unknown>>,
- );
-
- if (
- description.dataType !== "json" ||
- !hasExpectedPrefix ||
- !hasExpectedSchema
- ) {
- throw new Error(
- `Partner search index ${indexName} does not match the configured schema. Delete and recreate it before backfilling.`,
- );
- }
-}
-
-function getUpsertEntries(
- indexName: string,
- documents: PartnerSearchDocument[],
-) {
- return documents.map((document) => ({
- key: getDocumentKey(indexName, document.id),
- path: "$",
- value: serializeUpstashDocument(document),
- }));
-}
-
-async function upsertDocumentBatch(
- redisClient: Redis,
- indexName: string,
- documents: PartnerSearchDocument[],
-) {
- const upsertEntries = getUpsertEntries(indexName, documents);
-
- await withTransientRetry(() => redisClient.json.mset(...upsertEntries));
-}
-
-async function deleteDocumentBatch(
- redisClient: Redis,
- indexName: string,
- documentIds: string[],
-) {
- await withTransientRetry(() =>
- redisClient.del(
- ...documentIds.map((documentId) => getDocumentKey(indexName, documentId)),
- ),
- );
-}
-
-export async function createUpstashRedisPartnerSearchIndex({
- redisClient,
- indexName,
-}: CreateUpstashRedisPartnerSearchProviderOptions = {}) {
- const resolvedRedisClient =
- redisClient ?? createRedisClient(WRITE_REQUEST_TIMEOUT_MS);
- const resolvedIndexName = getIndexName(indexName);
-
- const index = await resolvedRedisClient.search.createIndex({
- name: resolvedIndexName,
- dataType: "json",
- prefix: getDocumentPrefix(resolvedIndexName),
- schema: upstashPartnerSearchSchema,
- skipInitialScan: true,
- existsOk: true,
- });
- const description = await index.describe();
-
- if (!description) {
- throw new Error(
- `Partner search index ${resolvedIndexName} was not created.`,
- );
- }
-
- validateIndexDescription(description, resolvedIndexName);
- // Hand back the description so callers can report on the index without
- // paying for a second describe() round trip.
- return { index, description };
-}
-
-export function createUpstashRedisPartnerSearchProvider({
- redisClient,
- queryRedisClient,
- indexName,
-}: CreateUpstashRedisPartnerSearchProviderOptions = {}): PartnerSearchProvider {
- const resolvedRedisClient =
- redisClient ?? createRedisClient(WRITE_REQUEST_TIMEOUT_MS);
- const resolvedQueryRedisClient =
- queryRedisClient ??
- redisClient ??
- createRedisClient(QUERY_REQUEST_TIMEOUT_MS);
- const resolvedIndexName = getIndexName(indexName);
- const queryIndex: UpstashPartnerSearchIndex =
- resolvedQueryRedisClient.search.index({
- name: resolvedIndexName,
- schema: upstashPartnerSearchSchema,
- });
- const writeIndex: UpstashPartnerSearchIndex =
- resolvedRedisClient.search.index({
- name: resolvedIndexName,
- schema: upstashPartnerSearchSchema,
- });
-
- async function findCandidates({
- programId,
- query,
- limit,
- }: PartnerSearchCandidateQuery) {
- validatePartnerSearchCandidateLimit(limit);
- const filter = buildUpstashFilter({ programId, query });
- const results = await withQueryDeadline(
- () => queryIndex.query({ filter, limit, select: {} }),
- QUERY_OPERATION_TIMEOUT_MS,
- );
- const hits = results.map(({ key, score }) => ({
- id: key.slice(getDocumentPrefix(resolvedIndexName).length),
- score,
- }));
-
- return { hits };
- }
-
- return {
- searchCandidates: findCandidates,
-
- async waitForIndexing() {
- const result = await withDeadline(
- () => withTransientRetry(() => writeIndex.waitIndexing()),
- WAIT_FOR_INDEXING_TIMEOUT_MS,
- `Partner search waitForIndexing timed out after ${WAIT_FOR_INDEXING_TIMEOUT_MS}ms.`,
- );
-
- if (result === 0) {
- throw new Error(
- `Partner search index ${resolvedIndexName} was not found.`,
- );
- }
- },
-
- async upsert(documents) {
- for (const documentBatch of chunk(documents, WRITE_BATCH_SIZE)) {
- await upsertDocumentBatch(
- resolvedRedisClient,
- resolvedIndexName,
- documentBatch,
- );
- }
- },
-
- async delete(documentIds) {
- for (const documentIdBatch of chunk(documentIds, WRITE_BATCH_SIZE)) {
- await deleteDocumentBatch(
- resolvedRedisClient,
- resolvedIndexName,
- documentIdBatch,
- );
- }
- },
- };
-}
diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.ts
index 98ce65ef94..2a1f4b3f3f 100644
--- a/apps/web/lib/api/partners/search/types.ts
+++ b/apps/web/lib/api/partners/search/types.ts
@@ -77,7 +77,6 @@ export interface PartnerSearchProvider {
searchCandidates(
query: PartnerSearchCandidateQuery,
): Promise<PartnerSearchResult>;
- waitForIndexing(): Promise<void>;
upsert(documents: PartnerSearchDocument[]): Promise<void>;
delete(documentIds: string[]): Promise<void>;
}
diff --git a/apps/web/package.json b/apps/web/package.json
index 308e676570..d6892612b2 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -60,7 +60,7 @@
"@types/buffer-crc32": "0.2.0",
"@upstash/qstash": "^2.11.0",
"@upstash/ratelimit": "^2.0.6",
- "@upstash/redis": "^1.38.2",
+ "@upstash/redis": "^1.35.3",
"@upstash/vector": "^1.2.2",
"@upstash/workflow": "^1.3.0",
"@vercel/edge-config": "^0.4.1",
diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.ts
index 0ea053318c..b2b2e75bb8 100644
--- a/apps/web/scripts/dev/benchmark-partner-search.ts
+++ b/apps/web/scripts/dev/benchmark-partner-search.ts
@@ -21,14 +21,13 @@
* pnpm run script dev/benchmark-partner-search --programId=prog_123 [--requests=1000]
* [--sampleSize=100] [--concurrency=10] [--warmup=50] [--thresholdMs=1000] [--searchOnly]
*
- * Requires PARTNER_SEARCH_PROVIDER and an index backfilled for the program:
+ * Requires TURBOPUFFER_API_KEY and an index backfilled for the program:
* pnpm run script partners/backfill-partner-search --programId=prog_123
*/
import { getPartners } from "@/lib/api/partners/get-partners";
import {
getPartnerSearchProvider,
- getPartnerSearchProviderName,
PARTNER_SEARCH_CANDIDATE_LIMIT,
partnerSearchDocumentSelect,
serializePartnerSearchDocument,
@@ -370,7 +369,6 @@ async function runWithConcurrency<T>(
async function main() {
const options = parseArguments(process.argv.slice(2));
const searchProvider = getPartnerSearchProvider();
- const providerName = getPartnerSearchProviderName();
if (!searchProvider) {
throw new Error(
@@ -451,7 +449,6 @@ async function main() {
};
console.log(`Partner search benchmark for program ${options.programId}`);
- console.log(`Provider: ${providerName}`);
console.log(`${partnerCount.toLocaleString()} program partners`);
console.log(
`${options.requests.toLocaleString()} measured requests, ${options.warmupRequests.toLocaleString()} warm-up requests, concurrency ${options.concurrency}`,
diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.ts
index caca3938fd..42bf99ea25 100644
--- a/apps/web/scripts/partners/backfill-partner-search.ts
+++ b/apps/web/scripts/partners/backfill-partner-search.ts
@@ -13,15 +13,14 @@
* pnpm run script partners/backfill-partner-search --programId=prog_123
* [--batchSize=500] [--after=pge_123]
*
- * Requires PARTNER_SEARCH_PROVIDER to be configured.
+ * Requires TURBOPUFFER_API_KEY to be configured.
*/
import {
backfillPartnerSearch,
- getPartnerSearchProviderName,
+ getPartnerSearchProvider,
type PartnerSearchBackfillProgress,
} from "@/lib/api/partners/search";
-import { createUpstashRedisPartnerSearchIndex } from "@/lib/api/partners/search/providers/upstash-redis";
import { prisma } from "@/lib/prisma";
import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";
import "dotenv-flow/config";
@@ -102,15 +101,8 @@ function createProgressReporter(totalDocuments: number, batchSize: number) {
async function main() {
const { programId, batchSize, after } = parseArguments(process.argv.slice(2));
resumeAfter = after;
- const providerName = getPartnerSearchProviderName();
- if (!providerName) {
- throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");
- }
-
- // Redis Search indexes have an explicit schema and must exist before the
- // backfill starts. Turbopuffer creates its namespace on the first upsert.
- if (providerName === "upstash-redis") {
- await createUpstashRedisPartnerSearchIndex();
+ if (!getPartnerSearchProvider()) {
+ throw new Error("TURBOPUFFER_API_KEY is not configured.");
}
// Same where-clause the backfill pages with, so chunk totals line up.
@@ -122,7 +114,6 @@ async function main() {
});
console.log(`Starting partner search backfill for program ${programId}`);
- console.log(`Provider: ${providerName}`);
console.log(
`Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,
);
diff --git a/apps/web/scripts/partners/delete-partner-search-index.ts b/apps/web/scripts/partners/delete-partner-search-index.ts
index e74c1b0b4f..e811f1f9f7 100644
--- a/apps/web/scripts/partners/delete-partner-search-index.ts
+++ b/apps/web/scripts/partners/delete-partner-search-index.ts
@@ -1,13 +1,20 @@
-import { getPartnerSearchProviderName } from "@/lib/api/partners/search";
+/**
+ * Empties a partner search namespace.
+ *
+ * Takes the namespace explicitly rather than reading the one the code writes to,
+ * so retiring an old version after a migration is possible — and requires it
+ * twice, because this is not recoverable without a backfill.
+ *
+ * cd apps/web
+ * pnpm run script partners/delete-partner-search-index
+ * --indexName=partner-search-v2 --confirm=partner-search-v2
+ *
+ * Requires TURBOPUFFER_API_KEY.
+ */
+
import { deleteTurbopufferPartnerSearchNamespace } from "@/lib/api/partners/search/providers/turbopuffer";
-import { chunk } from "@dub/utils";
-import { Redis } from "@upstash/redis";
import "dotenv-flow/config";
-const DELETE_BATCH_SIZE = 1_000;
-const SCAN_COUNT = 10_000;
-const MAX_DELETE_PASSES = 3;
-
interface DeletePartnerSearchIndexArguments {
indexName: string;
}
@@ -39,107 +46,16 @@ function parseArguments(args: string[]): DeletePartnerSearchIndexArguments {
return { indexName };
}
-function createRedisClient() {
- const url = process.env.UPSTASH_REDIS_REST_URL;
- const token = process.env.UPSTASH_REDIS_REST_TOKEN;
-
- if (!url || !token) {
- throw new Error(
- "UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN are required.",
- );
- }
-
- return new Redis({ url, token });
-}
-
-async function deleteDocumentPass(redis: Redis, documentPattern: string) {
- let cursor = "0";
- let deleted = 0;
-
- do {
- const [nextCursor, keys] = await redis.scan(cursor, {
- match: documentPattern,
- count: SCAN_COUNT,
- });
-
- for (const keyBatch of chunk(keys, DELETE_BATCH_SIZE)) {
- deleted += await redis.del(...keyBatch);
- }
-
- cursor = nextCursor;
- } while (cursor !== "0");
-
- return deleted;
-}
-
-async function deleteDocuments(redis: Redis, indexName: string) {
- const documentPattern = `${indexName}:partner:*`;
- let totalDeleted = 0;
-
- // Repeat the scan to verify that no matching keys remain after deletion
- for (let pass = 1; pass <= MAX_DELETE_PASSES; pass++) {
- const deleted = await deleteDocumentPass(redis, documentPattern);
- totalDeleted += deleted;
-
- console.log(
- `Deletion pass ${pass}: ${deleted.toLocaleString()} documents removed`,
- );
-
- if (deleted === 0) {
- return totalDeleted;
- }
- }
-
- throw new Error(
- `Documents matching ${documentPattern} are still being created. Stop application processes using this index and run the script again.`,
- );
-}
-
async function main() {
const { indexName } = parseArguments(process.argv.slice(2));
- const providerName = getPartnerSearchProviderName();
- if (!providerName) {
- throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");
- }
-
- if (providerName === "turbopuffer") {
- await deleteTurbopufferPartnerSearchNamespace({ namespaceName: indexName });
- console.log(`Partner search cleanup complete: emptied ${indexName}.`);
- return;
- }
- const redis = createRedisClient();
- const index = redis.search.index({ name: indexName });
- const description = await index.describe();
- const expectedPrefix = `${indexName}:partner:`;
-
- // Verify that the index contains only partner search documents
- if (!description) {
- console.warn(
- `Index ${indexName} does not exist, so its prefix cannot be verified. Removing any documents left under ${expectedPrefix} only.`,
- );
- } else if (
- description.prefixes.length !== 1 ||
- description.prefixes[0] !== expectedPrefix
- ) {
- throw new Error(
- `Index ${indexName} does not use an expected partner search prefix.`,
- );
+ if (!process.env.TURBOPUFFER_API_KEY?.trim()) {
+ throw new Error("TURBOPUFFER_API_KEY is not configured.");
}
- // Drop the search index without affecting unrelated Redis data
- const dropped = await index.drop();
- console.log(
- dropped === 1
- ? `Dropped partner search index ${indexName}`
- : `Partner search index ${indexName} did not exist`,
- );
+ await deleteTurbopufferPartnerSearchNamespace({ namespaceName: indexName });
- // Step 3: Delete the partner documents stored for this index
- const deleted = await deleteDocuments(redis, indexName);
- console.log(
- `Partner search cleanup complete: ${deleted.toLocaleString()} documents removed.`,
- );
+ console.log(`Partner search cleanup complete: emptied ${indexName}.`);
}
main().catch((error) => {
diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.ts
index 4b6e78b99d..14e2487928 100644
--- a/apps/web/tests/partners/get-partners-count-search.test.ts
+++ b/apps/web/tests/partners/get-partners-count-search.test.ts
@@ -28,7 +28,6 @@ function createSearchProvider(): PartnerSearchProvider {
searchCandidates: vi.fn().mockResolvedValue({
hits: [{ id: "pge_2" }, { id: "pge_1" }],
}),
- waitForIndexing: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
};
diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.ts
index dc164e0a51..f63cd0f733 100644
--- a/apps/web/tests/partners/get-partners-search.test.ts
+++ b/apps/web/tests/partners/get-partners-search.test.ts
@@ -37,7 +37,6 @@ function createSearchProvider(
): PartnerSearchProvider {
return {
searchCandidates: vi.fn().mockResolvedValue({ hits }),
- waitForIndexing: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
};
diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.ts
index 732030193c..ecd599901d 100644
--- a/apps/web/tests/partners/partner-search-backfill.test.ts
+++ b/apps/web/tests/partners/partner-search-backfill.test.ts
@@ -37,7 +37,6 @@ function createSource(id: string): PartnerSearchDocumentSource {
function createProvider(): PartnerSearchProvider {
return {
searchCandidates: vi.fn(),
- waitForIndexing: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
};
@@ -78,7 +77,6 @@ describe("backfillPartnerSearch", () => {
take: 2,
});
expect(searchProvider.upsert).toHaveBeenCalledTimes(2);
- expect(searchProvider.waitForIndexing).toHaveBeenCalledOnce();
expect(onProgress).toHaveBeenLastCalledWith({
batchSize: 1,
processed: 3,
diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
index aa363b38c6..1a4915604f 100644
--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
@@ -349,11 +349,6 @@ describe("Turbopuffer partner search provider", () => {
});
});
- it("does not wait for indexing, which turbopuffer does not need", async () => {
- await expect(createProvider().waitForIndexing()).resolves.toBeUndefined();
- expect(mocks.write).not.toHaveBeenCalled();
- });
-
it("empties the namespace", async () => {
mocks.deleteAll.mockResolvedValue({});
@@ -366,8 +361,7 @@ describe("Turbopuffer partner search provider", () => {
expect(result).toEqual({ namespaceName: "test-namespace" });
});
- it("falls back to the default namespace when the env var is blank", async () => {
- vi.stubEnv("PARTNER_SEARCH_INDEX_NAME", "");
+ it("uses the pinned namespace when none is passed", async () => {
mocks.deleteAll.mockResolvedValue({});
const { namespaceName } = await deleteTurbopufferPartnerSearchNamespace({
diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
deleted file mode 100644
index 5ba751c7e1..0000000000
--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
+++ /dev/null
@@ -1,426 +0,0 @@
-import {
- PARTNER_SEARCH_CANDIDATE_LIMIT,
- type PartnerSearchDocument,
-} from "@/lib/api/partners/search";
-import {
- createUpstashRedisPartnerSearchIndex,
- createUpstashRedisPartnerSearchProvider,
- upstashPartnerSearchSchema,
-} from "@/lib/api/partners/search/providers/upstash-redis";
-import type { Redis } from "@upstash/redis";
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-
-const document: PartnerSearchDocument = {
- id: "pge_test",
- programId: "prog_test",
- partnerId: "pn_test",
- name: "Rafi Hasan",
- email: "[email protected]",
- companyName: "Dub Partners",
- description: "Developer tools educator",
- platformTypes: ["website", "youtube", "twitter"],
- platformIdentifiers: ["rafi.dev", "@rafi-youtube", "@rafi-on-x"],
- linkDomains: ["dub.sh"],
- linkKeys: ["rafi"],
- shortLinks: ["https://dub.sh/rafi"],
- destinationUrls: ["https://example.com/referrals/rafi"],
-};
-
-const mocks = vi.hoisted(() => ({
- createIndex: vi.fn(),
- del: vi.fn(),
- describe: vi.fn(),
- index: vi.fn(),
- jsonMset: vi.fn(),
- query: vi.fn(),
- waitIndexing: vi.fn(),
-}));
-
-function createRedisMock(): Redis {
- const searchIndex = {
- describe: mocks.describe,
- query: mocks.query,
- waitIndexing: mocks.waitIndexing,
- };
-
- mocks.index.mockReturnValue(searchIndex);
- mocks.createIndex.mockReturnValue(searchIndex);
-
- const redisMock = {
- del: mocks.del,
- json: {
- mset: mocks.jsonMset,
- },
- search: {
- createIndex: mocks.createIndex,
- index: mocks.index,
- },
- } as unknown as Redis;
-
- return redisMock;
-}
-
-describe("Upstash Redis partner search provider", () => {
- beforeEach(() => {
- vi.clearAllMocks();
- mocks.del.mockResolvedValue(0);
- mocks.describe.mockResolvedValue({
- name: "test-index",
- dataType: "json",
- prefixes: ["test-index:partner:"],
- schema: upstashPartnerSearchSchema,
- });
- mocks.jsonMset.mockResolvedValue("OK");
- mocks.query.mockResolvedValue([]);
- mocks.waitIndexing.mockResolvedValue(1);
- });
-
- // In afterEach rather than the test body, so a failed assertion cannot leak
- // a stubbed env var into later tests.
- afterEach(() => {
- vi.unstubAllEnvs();
- });
-
- it("retrieves bounded relevance candidates without business filters", async () => {
- mocks.query.mockResolvedValue([
- {
- key: "test-index:partner:pge_test",
- score: 4,
- data: {},
- },
- ]);
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await expect(
- provider.searchCandidates({
- programId: document.programId,
- query: "rafi",
- limit: PARTNER_SEARCH_CANDIDATE_LIMIT,
- }),
- ).resolves.toEqual({
- hits: [{ id: document.id, score: 4 }],
- });
-
- expect(mocks.query).toHaveBeenCalledWith(
- expect.objectContaining({
- limit: PARTNER_SEARCH_CANDIDATE_LIMIT,
- select: {},
- }),
- );
- const request = mocks.query.mock.calls[0]![0];
- expect(request.offset).toBeUndefined();
- expect(request.orderBy).toBeUndefined();
- expect(JSON.stringify(request.filter)).toContain(
- '"programId":{"$eq":"prog_test"}',
- );
- });
-
- it("requires program-scoped text or partial email matches", async () => {
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await provider.searchCandidates({
- programId: document.programId,
- query: "examp",
- limit: 25,
- });
-
- const filter = mocks.query.mock.calls[0]![0].filter;
- expect(filter.$must).toBeUndefined();
- expect(filter.$should).toHaveLength(2);
- for (const branch of filter.$should) {
- expect(branch.$must).toEqual(
- expect.arrayContaining([{ programId: { $eq: "prog_test" } }]),
- );
- }
- expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"exa"}');
- expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"xam"}');
- expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"amp"}');
- });
-
- it("retries a transient provider error", async () => {
- mocks.query
- .mockRejectedValueOnce(new Error("Upstash returned 503"))
- .mockResolvedValueOnce([]);
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await expect(
- provider.searchCandidates({
- programId: document.programId,
- query: "rafi",
- limit: 10,
- }),
- ).resolves.toEqual({ hits: [] });
- expect(mocks.query).toHaveBeenCalledTimes(2);
- });
-
- it("does not retry a permanent provider error", async () => {
- mocks.query.mockRejectedValue(new Error("Invalid search filter"));
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await expect(
- provider.searchCandidates({
- programId: document.programId,
- query: "rafi",
- limit: 10,
- }),
- ).rejects.toThrow("Invalid search filter");
- expect(mocks.query).toHaveBeenCalledTimes(1);
- });
-
- it("retries a network TypeError", async () => {
- // undici surfaces network failures as `TypeError: fetch failed`.
- mocks.query
- .mockRejectedValueOnce(new TypeError("fetch failed"))
- .mockResolvedValueOnce([]);
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await expect(
- provider.searchCandidates({
- programId: document.programId,
- query: "rafi",
- limit: 10,
- }),
- ).resolves.toEqual({ hits: [] });
- expect(mocks.query).toHaveBeenCalledTimes(2);
- });
-
- it("does not retry a programming TypeError", async () => {
- mocks.query.mockRejectedValue(
- new TypeError("Cannot read properties of undefined (reading 'query')"),
- );
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await expect(
- provider.searchCandidates({
- programId: document.programId,
- query: "rafi",
- limit: 10,
- }),
- ).rejects.toThrow("Cannot read properties of undefined");
- expect(mocks.query).toHaveBeenCalledTimes(1);
- });
-
- it("does not retry a request timeout after it consumes the query budget", async () => {
- mocks.query.mockRejectedValue(
- new DOMException(
- "The operation was aborted due to timeout",
- "TimeoutError",
- ),
- );
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await expect(
- provider.searchCandidates({
- programId: document.programId,
- query: "rafi",
- limit: 10,
- }),
- ).rejects.toThrow("The operation was aborted due to timeout");
- expect(mocks.query).toHaveBeenCalledTimes(1);
- });
-
- it("allows query latency within the one-second deadline", async () => {
- vi.useFakeTimers();
- mocks.query.mockImplementation(
- () => new Promise((resolve) => setTimeout(() => resolve([]), 500)),
- );
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- try {
- const result = provider.searchCandidates({
- programId: document.programId,
- query: "rafi",
- limit: 10,
- });
- await vi.advanceTimersByTimeAsync(500);
-
- await expect(result).resolves.toEqual({ hits: [] });
- expect(vi.getTimerCount()).toBe(0);
- } finally {
- vi.useRealTimers();
- }
- });
-
- it("bounds the total query operation to one second", async () => {
- vi.useFakeTimers();
- mocks.query.mockImplementation(() => new Promise(() => {}));
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- try {
- const result = expect(
- provider.searchCandidates({
- programId: document.programId,
- query: "rafi",
- limit: 10,
- }),
- ).rejects.toThrow("Partner search query timed out after 1000ms");
- await vi.advanceTimersByTimeAsync(1_000);
-
- await result;
- expect(vi.getTimerCount()).toBe(0);
- } finally {
- vi.useRealTimers();
- }
- });
-
- it("writes one lean search document per enrollment", async () => {
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await provider.upsert([document]);
-
- const entries = mocks.jsonMset.mock.calls[0]!;
- expect(entries).toHaveLength(1);
- expect(entries[0]).toEqual({
- key: "test-index:partner:pge_test",
- path: "$",
- value: {
- programId: "prog_test",
- searchText: expect.any(String),
- emailNgrams: expect.any(String),
- },
- });
- expect(entries[0].value.searchText).toContain("[email protected]");
- expect(entries[0].value.searchText).toContain("dub partners");
- expect(entries[0].value.searchText).toContain("rafi-on-x");
- expect(entries[0].value.searchText).toContain("referrals/rafi");
- expect(entries[0].value.emailNgrams).toContain("exa");
- expect(entries[0].value).not.toHaveProperty("partnerId");
- expect(entries[0].value).not.toHaveProperty("status");
- expect(entries[0].value).not.toHaveProperty("partnerTagIds");
- expect(entries[0].value).not.toHaveProperty("totalSaleAmount");
- });
-
- it("bounds Upstash write batches", async () => {
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
- const documents = Array.from({ length: 101 }, (_, index) => ({
- ...document,
- id: `pge_${index}`,
- }));
-
- await provider.upsert(documents);
-
- expect(mocks.jsonMset).toHaveBeenCalledTimes(2);
- expect(mocks.jsonMset.mock.calls[0]).toHaveLength(100);
- expect(mocks.jsonMset.mock.calls[1]).toHaveLength(1);
- });
-
- it("deletes the enrollment document directly", async () => {
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await provider.delete([document.id]);
-
- expect(mocks.del).toHaveBeenCalledWith("test-index:partner:pge_test");
- });
-
- it("waits for pending index updates", async () => {
- vi.useFakeTimers();
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- try {
- await provider.waitForIndexing();
-
- expect(mocks.waitIndexing).toHaveBeenCalledOnce();
- expect(vi.getTimerCount()).toBe(0);
- } finally {
- vi.useRealTimers();
- }
- });
-
- it("creates the index explicitly with the partner-search prefix", async () => {
- const redisClient = createRedisMock();
-
- await createUpstashRedisPartnerSearchIndex({
- redisClient,
- indexName: "test-index",
- });
-
- expect(mocks.createIndex).toHaveBeenCalledWith(
- expect.objectContaining({
- name: "test-index",
- dataType: "json",
- prefix: "test-index:partner:",
- existsOk: true,
- skipInitialScan: true,
- }),
- );
- expect(Object.keys(upstashPartnerSearchSchema).sort()).toEqual([
- "emailNgrams",
- "programId",
- "searchText",
- ]);
- });
-
- it("rejects an existing index with a stale schema", async () => {
- mocks.describe.mockResolvedValue({
- name: "test-index",
- dataType: "json",
- prefixes: ["test-index:partner:"],
- schema: { id: { type: "KEYWORD" } },
- });
-
- await expect(
- createUpstashRedisPartnerSearchIndex({
- redisClient: createRedisMock(),
- indexName: "test-index",
- }),
- ).rejects.toThrow("Delete and recreate it before backfilling");
- });
-
- it.each([
- ["blank", ""],
- ["whitespace", " "],
- ])(
- "falls back to the default index when the env var is %s",
- async (_l, value) => {
- vi.stubEnv("PARTNER_SEARCH_INDEX_NAME", value);
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- });
-
- await provider.upsert([document]);
-
- expect(mocks.jsonMset).toHaveBeenCalledWith(
- expect.objectContaining({ key: "partner-search-v1:partner:pge_test" }),
- );
- },
- );
-});