Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Add Upstash Search partner search provider

Rafi775399f8d11c
diff --git a/apps/web/.env.example b/apps/web/.env.example
index 0d09c879ea..486c67880a 100644
--- a/apps/web/.env.example
+++ b/apps/web/.env.example
@@ -24,10 +24,16 @@ UPSTASH_REDIS_REST_URL=
UPSTASH_REDIS_REST_TOKEN=
# Full-text partner search
+# Supported values: upstash-redis, upstash-search
PARTNER_SEARCH_PROVIDER=
PARTNER_SEARCH_INDEX_NAME=partner-search-v1
+PARTNER_UPSTASH_SEARCH_INDEX_NAME=partner-search-v1
PARTNER_SEARCH_DEBUG=false
+# Required when PARTNER_SEARCH_PROVIDER=upstash-search
+UPSTASH_SEARCH_REST_URL=
+UPSTASH_SEARCH_REST_TOKEN=
+
# Upstash QStash – required for queues and background jobs
# Get your QStash Token here: https://upstash.com/docs/qstash/overall/getstarted
QSTASH_URL="https://qstash-us-east-1.upstash.io"
diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.ts
index 9d83ad4291..64ae4ddc11 100644
--- a/apps/web/lib/api/partners/search/index.ts
+++ b/apps/web/lib/api/partners/search/index.ts
@@ -4,6 +4,7 @@ export * from "./order-search-results";
export * from "./provider";
export * from "./providers/mock";
export * from "./providers/upstash-redis";
+export * from "./providers/upstash-search";
export * from "./searchable-values";
export * from "./serialize-document";
export * from "./sync-documents";
diff --git a/apps/web/lib/api/partners/search/provider.ts b/apps/web/lib/api/partners/search/provider.ts
index c40aeb63be..eef9233cb0 100644
--- a/apps/web/lib/api/partners/search/provider.ts
+++ b/apps/web/lib/api/partners/search/provider.ts
@@ -1,7 +1,9 @@
-import type { PartnerSearchProvider } from "./types";
import { createUpstashRedisPartnerSearchProvider } from "./providers/upstash-redis";
+import { createUpstashSearchPartnerSearchProvider } from "./providers/upstash-search";
+import type { PartnerSearchProvider } from "./types";
let upstashRedisProvider: PartnerSearchProvider | null = null;
+let upstashSearchProvider: PartnerSearchProvider | null = null;
export function getPartnerSearchProvider(): PartnerSearchProvider | null {
const provider = process.env.PARTNER_SEARCH_PROVIDER?.trim();
@@ -10,13 +12,19 @@ export function getPartnerSearchProvider(): PartnerSearchProvider | null {
return null;
}
- if (provider !== "upstash-redis") {
- throw new Error(`Unsupported partner search provider: ${provider}`);
+ if (provider === "upstash-redis") {
+ if (!upstashRedisProvider) {
+ upstashRedisProvider = createUpstashRedisPartnerSearchProvider();
+ }
+ return upstashRedisProvider;
}
- if (!upstashRedisProvider) {
- upstashRedisProvider = createUpstashRedisPartnerSearchProvider();
+ if (provider === "upstash-search") {
+ if (!upstashSearchProvider) {
+ upstashSearchProvider = createUpstashSearchPartnerSearchProvider();
+ }
+ return upstashSearchProvider;
}
- return upstashRedisProvider;
+ throw new Error(`Unsupported partner search provider: ${provider}`);
}
diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.ts
new file mode 100644
index 0000000000..760e782f28
--- /dev/null
+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts
@@ -0,0 +1,671 @@
+import { chunk } from "@dub/utils";
+import { Search } from "@upstash/search";
+import { normalizePartnerSearchQuery } from "../searchable-values";
+import type {
+ PartnerSearchCountQuery,
+ PartnerSearchDocument,
+ PartnerSearchGroup,
+ PartnerSearchGroupField,
+ PartnerSearchListFilter,
+ PartnerSearchProvider,
+ PartnerSearchQuery,
+} from "../types";
+
+const DEFAULT_INDEX_NAME = "partner-search-v1";
+const NULL_VALUE = "\0__null:f47ac10b-58cc-4372-a567-0e02b2c3d479__";
+const MAX_SEARCH_RESULTS = 100;
+const WRITE_BATCH_SIZE = 100;
+const DELETE_BATCH_SIZE = 1_000;
+const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;
+const WAIT_FOR_INDEXING_POLL_MS = 100;
+const CANDIDATE_CACHE_TTL_MS = 1_000;
+const MAX_CANDIDATE_CACHE_ENTRIES = 100;
+const TRANSIENT_RETRY_ATTEMPTS = 2;
+const QUERY_OPERATION_TIMEOUT_MS = 1_000;
+
+interface UpstashSearchContent extends Record<string, unknown> {
+ partnerId: string;
+ name: string;
+ email: string;
+ companyName: string;
+ description: string;
+ platforms: string;
+ links: string;
+ emailNgrams: string;
+}
+
+interface UpstashSearchMetadata extends Record<string, unknown> {
+ programId: string;
+ partnerId: string;
+ status: string;
+ tenantId: string;
+ groupId: string;
+ country: string;
+ partnerTagIds: string[];
+ referredByPartnerId: string;
+ totalClicks: number;
+ totalLeads: number;
+ totalConversions: number;
+ totalSaleAmount: number;
+ totalCommissions: number;
+ netRevenue: number;
+ earningsPerClick: number;
+ averageLifetimeValue?: number;
+ clickToLeadRate?: number;
+ clickToConversionRate?: number;
+ leadToConversionRate?: number;
+ returnOnAdSpend?: number;
+ createdAt: number;
+}
+
+interface UpstashSearchResult {
+ id: string;
+ content: UpstashSearchContent;
+ metadata?: UpstashSearchMetadata;
+ score: number;
+}
+
+interface UpstashSearchIndexClient {
+ search(params: {
+ query: string;
+ limit: number;
+ filter: string;
+ reranking: boolean;
+ semanticWeight: number;
+ inputEnrichment: boolean;
+ }): Promise<UpstashSearchResult[]>;
+ upsert(
+ documents:
+ | {
+ id: string;
+ content: UpstashSearchContent;
+ metadata: UpstashSearchMetadata;
+ }
+ | {
+ id: string;
+ content: UpstashSearchContent;
+ metadata: UpstashSearchMetadata;
+ }[],
+ ): Promise<string>;
+ delete(documentIds: string[]): Promise<{ deleted: number }>;
+ info(): Promise<{ pendingDocumentCount: number; documentCount: number }>;
+}
+
+interface CreateUpstashSearchPartnerSearchProviderOptions {
+ searchIndex?: UpstashSearchIndexClient;
+ indexName?: string;
+}
+
+interface CandidateCacheEntry {
+ expiresAt: number;
+ promise: Promise<UpstashSearchResult[]>;
+}
+
+function getIndexName(indexName?: string): string {
+ return (
+ indexName ??
+ process.env.PARTNER_UPSTASH_SEARCH_INDEX_NAME?.trim() ??
+ process.env.PARTNER_SEARCH_INDEX_NAME?.trim() ??
+ DEFAULT_INDEX_NAME
+ );
+}
+
+function createSearchIndex(indexName: string): UpstashSearchIndexClient {
+ const url = process.env.UPSTASH_SEARCH_REST_URL;
+ const token = process.env.UPSTASH_SEARCH_REST_TOKEN;
+
+ if (!url || !token) {
+ throw new Error(
+ "UPSTASH_SEARCH_REST_URL and UPSTASH_SEARCH_REST_TOKEN are required for partner search.",
+ );
+ }
+
+ const client = new Search({ url, token, retry: false });
+ return client.index<UpstashSearchContent, UpstashSearchMetadata>(indexName);
+}
+
+function truncate(value: string | null, maxLength: number): string {
+ return value?.slice(0, maxLength) ?? "";
+}
+
+function joinValues(values: string[], maxLength: number): string {
+ return truncate(values.join(" "), maxLength);
+}
+
+function getEmailNgrams(email: string | null): string {
+ if (!email) {
+ return "";
+ }
+
+ const normalized = normalizePartnerSearchQuery(email);
+ if (normalized.length < 3) {
+ return normalized;
+ }
+
+ return Array.from(
+ new Set(
+ Array.from({ length: normalized.length - 2 }, (_, index) =>
+ normalized.slice(index, index + 3),
+ ),
+ ),
+ ).join(" ");
+}
+
+function getQueryNgrams(query: string): string | null {
+ if (query.length < 3 || /\s/u.test(query)) {
+ return null;
+ }
+
+ return Array.from(
+ new Set(
+ Array.from({ length: query.length - 2 }, (_, index) =>
+ query.slice(index, index + 3),
+ ),
+ ),
+ ).join(" ");
+}
+
+function serializeUpstashSearchDocument(document: PartnerSearchDocument) {
+ // Upstash Search limits searchable content to 4,096 characters. Give each
+ // assignment field its own budget so a long description cannot crowd out
+ // platforms, links, or the email n-grams used for partial email matching.
+ const content: UpstashSearchContent = {
+ partnerId: truncate(document.partnerId, 50),
+ name: truncate(document.name, 200),
+ email: truncate(normalizePartnerSearchQuery(document.email ?? ""), 320),
+ companyName: truncate(document.companyName, 200),
+ description: truncate(document.description, 700),
+ platforms: joinValues(
+ [...document.platformTypes, ...document.platformIdentifiers],
+ 500,
+ ),
+ links: joinValues(
+ [
+ ...document.linkDomains,
+ ...document.linkKeys,
+ ...document.shortLinks,
+ ...document.destinationUrls,
+ ],
+ 850,
+ ),
+ emailNgrams: truncate(getEmailNgrams(document.email), 800),
+ };
+
+ const metadata: UpstashSearchMetadata = {
+ programId: document.programId,
+ partnerId: document.partnerId,
+ status: document.status,
+ tenantId: document.tenantId ?? NULL_VALUE,
+ groupId: document.groupId ?? NULL_VALUE,
+ country: document.country ?? NULL_VALUE,
+ partnerTagIds: document.partnerTagIds,
+ referredByPartnerId: document.referredByPartnerId ?? NULL_VALUE,
+ totalClicks: document.totalClicks,
+ totalLeads: document.totalLeads,
+ totalConversions: document.totalConversions,
+ totalSaleAmount: document.totalSaleAmount,
+ totalCommissions: document.totalCommissions,
+ netRevenue: document.netRevenue,
+ earningsPerClick: document.earningsPerClick,
+ ...(document.averageLifetimeValue !== null && {
+ averageLifetimeValue: document.averageLifetimeValue,
+ }),
+ ...(document.clickToLeadRate !== null && {
+ clickToLeadRate: document.clickToLeadRate,
+ }),
+ ...(document.clickToConversionRate !== null && {
+ clickToConversionRate: document.clickToConversionRate,
+ }),
+ ...(document.leadToConversionRate !== null && {
+ leadToConversionRate: document.leadToConversionRate,
+ }),
+ ...(document.returnOnAdSpend !== null && {
+ returnOnAdSpend: document.returnOnAdSpend,
+ }),
+ createdAt: new Date(document.createdAt).getTime(),
+ };
+
+ return { id: document.id, content, metadata };
+}
+
+function quoteFilterValue(value: string): string {
+ return JSON.stringify(value);
+}
+
+function buildScalarListFilter(
+ field: string,
+ filter: PartnerSearchListFilter,
+): string {
+ const values = filter.values.map(quoteFilterValue).join(", ");
+ return `@metadata.${field} ${filter.operator === "IN" ? "IN" : "NOT IN"} (${values})`;
+}
+
+function buildArrayListFilter(
+ field: string,
+ filter: PartnerSearchListFilter,
+): string {
+ const operator = filter.operator === "IN" ? "CONTAINS" : "NOT CONTAINS";
+ const booleanOperator = filter.operator === "IN" ? " OR " : " AND ";
+ return `(${filter.values
+ .map((value) => `@metadata.${field} ${operator} ${quoteFilterValue(value)}`)
+ .join(booleanOperator)})`;
+}
+
+function buildUpstashSearchFilter({
+ programId,
+ filters,
+}: PartnerSearchCountQuery): string {
+ const conditions = [`@metadata.programId = ${quoteFilterValue(programId)}`];
+
+ if (filters?.status) {
+ conditions.push(`@metadata.status = ${quoteFilterValue(filters.status)}`);
+ }
+ if (filters?.tenantId) {
+ conditions.push(
+ `@metadata.tenantId = ${quoteFilterValue(filters.tenantId)}`,
+ );
+ }
+ if (filters?.partnerIds?.length) {
+ conditions.push(
+ buildScalarListFilter("partnerId", {
+ values: filters.partnerIds,
+ operator: "IN",
+ }),
+ );
+ }
+
+ const listFilters: [
+ "groupId" | "country",
+ PartnerSearchListFilter | undefined,
+ ][] = [
+ ["groupId", filters?.groupIds],
+ ["country", filters?.countries],
+ ];
+
+ for (const [field, listFilter] of listFilters) {
+ if (listFilter?.values.length) {
+ conditions.push(buildScalarListFilter(field, listFilter));
+ }
+ }
+
+ if (filters?.partnerTagIds?.values.length) {
+ conditions.push(
+ buildArrayListFilter("partnerTagIds", filters.partnerTagIds),
+ );
+ }
+ if (filters?.referredByPartnerId) {
+ conditions.push(
+ `@metadata.referredByPartnerId = ${quoteFilterValue(filters.referredByPartnerId)}`,
+ );
+ }
+
+ for (const [field, range] of Object.entries(filters?.metrics ?? {})) {
+ if (range.min !== undefined) {
+ conditions.push(`@metadata.${field} >= ${range.min}`);
+ }
+ if (range.max !== undefined) {
+ conditions.push(`@metadata.${field} <= ${range.max}`);
+ }
+ }
+
+ return conditions.join(" AND ");
+}
+
+function assertCompleteResults(results: UpstashSearchResult[], query: string) {
+ // Upstash Search has no count, aggregation, offset, or search cursor API.
+ // We can preserve the provider contract only while the complete match set
+ // fits below its top-1,000 response limit. Reject capped results so the UI
+ // never presents an incomplete count, facet, sorted page, or later page.
+ if (results.length >= MAX_SEARCH_RESULTS) {
+ throw new Error(
+ `Upstash Search returned its ${MAX_SEARCH_RESULTS.toLocaleString()}-result limit for query ${JSON.stringify(query)}. Exact pagination, counts, and facets are unavailable for this query.`,
+ );
+ }
+}
+
+function isTransientError(error: unknown): boolean {
+ if (error instanceof TypeError) {
+ return true;
+ }
+
+ const message = error instanceof Error ? error.message : String(error);
+ return /\b(429|500|502|503|504)\b|rate.?limit|timeout|timed out|fetch failed|network|ECONNRESET|ETIMEDOUT/i.test(
+ message,
+ );
+}
+
+function isTimeoutError(error: unknown): boolean {
+ const message = error instanceof Error ? error.message : String(error);
+ return /timeout|timed out|ETIMEDOUT/i.test(message);
+}
+
+async function withTransientRetry<T>(
+ operation: () => Promise<T>,
+ { retryTimeouts = true }: { retryTimeouts?: boolean } = {},
+): Promise<T> {
+ for (let attempt = 1; attempt <= TRANSIENT_RETRY_ATTEMPTS; attempt++) {
+ try {
+ return await operation();
+ } catch (error) {
+ if (
+ attempt === TRANSIENT_RETRY_ATTEMPTS ||
+ !isTransientError(error) ||
+ (!retryTimeouts && isTimeoutError(error))
+ ) {
+ throw error;
+ }
+
+ await new Promise((resolve) =>
+ setTimeout(resolve, 50 * attempt + Math.random() * 25),
+ );
+ }
+ }
+
+ throw new Error("Partner search operation failed.");
+}
+
+async function withQueryDeadline<T>(operation: () => Promise<T>): Promise<T> {
+ let timeoutId: ReturnType<typeof setTimeout> | undefined;
+ const timeout = new Promise<never>((_, reject) => {
+ timeoutId = setTimeout(
+ () =>
+ reject(
+ new Error(
+ `Partner search query timed out after ${QUERY_OPERATION_TIMEOUT_MS}ms.`,
+ ),
+ ),
+ QUERY_OPERATION_TIMEOUT_MS,
+ );
+ });
+
+ try {
+ return await Promise.race([
+ withTransientRetry(operation, { retryTimeouts: false }),
+ timeout,
+ ]);
+ } finally {
+ clearTimeout(timeoutId);
+ }
+}
+
+function mergeSearchResults(
+ standardResults: UpstashSearchResult[],
+ emailResults: UpstashSearchResult[],
+): UpstashSearchResult[] {
+ const resultsById = new Map<string, UpstashSearchResult>();
+
+ for (const result of [...standardResults, ...emailResults]) {
+ const existing = resultsById.get(result.id);
+ if (!existing || result.score > existing.score) {
+ resultsById.set(result.id, result);
+ }
+ }
+
+ return Array.from(resultsById.values()).sort(
+ (left, right) =>
+ right.score - left.score || left.id.localeCompare(right.id),
+ );
+}
+
+function compareResults(
+ left: UpstashSearchResult,
+ right: UpstashSearchResult,
+ sort: NonNullable<PartnerSearchQuery["sort"]>,
+): number {
+ const leftValue = left.metadata?.[sort.field] as number | undefined;
+ const rightValue = right.metadata?.[sort.field] as number | undefined;
+
+ if (leftValue === rightValue) {
+ return left.id.localeCompare(right.id);
+ }
+ if (leftValue === undefined) {
+ return 1;
+ }
+ if (rightValue === undefined) {
+ return -1;
+ }
+
+ const comparison = leftValue < rightValue ? -1 : 1;
+ return sort.order === "asc" ? comparison : -comparison;
+}
+
+function getGroupValues(
+ result: UpstashSearchResult,
+ field: PartnerSearchGroupField,
+): string[] {
+ const metadata = result.metadata;
+ if (!metadata) {
+ return [];
+ }
+ if (field === "partnerTagId") {
+ return metadata.partnerTagIds;
+ }
+
+ const value = metadata[field];
+ if (typeof value !== "string") {
+ return [];
+ }
+ if (field === "referredByPartnerId" && value === NULL_VALUE) {
+ return [];
+ }
+ return [value];
+}
+
+function mapGroupValue(value: string): string | null {
+ return value === NULL_VALUE ? null : value;
+}
+
+function logPartnerSearchDebug(
+ operation: "search" | "count" | "groupBy",
+ details: Record<string, unknown>,
+) {
+ if (process.env.PARTNER_SEARCH_DEBUG !== "true") {
+ return;
+ }
+
+ console.log(`[Partner Search Debug] Upstash Search ${operation}`, details);
+}
+
+export function createUpstashSearchPartnerSearchProvider({
+ searchIndex,
+ indexName,
+}: CreateUpstashSearchPartnerSearchProviderOptions = {}): PartnerSearchProvider {
+ const resolvedIndexName = getIndexName(indexName);
+ const index = searchIndex ?? createSearchIndex(resolvedIndexName);
+ const candidateCache = new Map<string, CandidateCacheEntry>();
+
+ async function searchCandidates(
+ query: PartnerSearchCountQuery,
+ { limit, requireComplete }: { limit: number; requireComplete: boolean },
+ ): Promise<UpstashSearchResult[]> {
+ const normalizedQuery = normalizePartnerSearchQuery(query.query);
+ const filter = buildUpstashSearchFilter(query);
+ const searchParams = {
+ limit,
+ filter,
+ reranking: false,
+ // This table needs literal full-text matching rather than related results
+ // that happen to be semantically similar to a name, email, or URL.
+ semanticWeight: 0,
+ inputEnrichment: false,
+ };
+ const queryNgrams = getQueryNgrams(normalizedQuery);
+
+ const [standardResults, rawEmailResults] = await withQueryDeadline(() =>
+ Promise.all([
+ index.search({ ...searchParams, query: normalizedQuery }),
+ queryNgrams
+ ? index.search({ ...searchParams, query: queryNgrams })
+ : Promise.resolve([]),
+ ]),
+ );
+
+ if (requireComplete) {
+ assertCompleteResults(standardResults, normalizedQuery);
+ assertCompleteResults(rawEmailResults, normalizedQuery);
+ }
+
+ const emailResults = rawEmailResults.filter(({ content }) =>
+ content.email.includes(normalizedQuery),
+ );
+ return mergeSearchResults(standardResults, emailResults);
+ }
+
+ function getCandidates(
+ query: PartnerSearchCountQuery,
+ ): Promise<UpstashSearchResult[]> {
+ const now = Date.now();
+ const cacheKey = JSON.stringify({
+ programId: query.programId,
+ query: query.query,
+ filters: query.filters,
+ });
+ const cached = candidateCache.get(cacheKey);
+ if (cached && cached.expiresAt > now) {
+ return cached.promise;
+ }
+
+ if (candidateCache.size >= MAX_CANDIDATE_CACHE_ENTRIES) {
+ candidateCache.delete(candidateCache.keys().next().value!);
+ }
+
+ const promise = searchCandidates(query, {
+ limit: MAX_SEARCH_RESULTS,
+ requireComplete: true,
+ }).catch((error) => {
+ candidateCache.delete(cacheKey);
+ throw error;
+ });
+ candidateCache.set(cacheKey, {
+ expiresAt: now + CANDIDATE_CACHE_TTL_MS,
+ promise,
+ });
+ return promise;
+ }
+
+ return {
+ mode: "relevance-only",
+
+ async search(query) {
+ const { sort } = query;
+ if (sort) {
+ const candidates = [...(await getCandidates(query))];
+ candidates.sort((left, right) => compareResults(left, right, sort));
+
+ const offset = (query.page - 1) * query.pageSize;
+ const hits = candidates
+ .slice(offset, offset + query.pageSize)
+ .flatMap(({ id, metadata, score }) =>
+ metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],
+ );
+
+ logPartnerSearchDebug("search", {
+ indexName: resolvedIndexName,
+ query,
+ candidateCount: candidates.length,
+ resultCount: hits.length,
+ hits,
+ });
+
+ return { hits };
+ }
+
+ const requestedResults = query.page * query.pageSize;
+ if (requestedResults > MAX_SEARCH_RESULTS) {
+ throw new Error(
+ `Upstash Search relevance pagination is limited to the first ${MAX_SEARCH_RESULTS.toLocaleString()} results.`,
+ );
+ }
+
+ // Upstash Search natively returns relevance-ranked results. Fetch enough
+ // results to derive the requested page without requiring a complete set.
+ const candidates = await searchCandidates(query, {
+ limit: requestedResults,
+ requireComplete: false,
+ });
+ const offset = (query.page - 1) * query.pageSize;
+ const hits = candidates
+ .slice(offset, offset + query.pageSize)
+ .flatMap(({ id, metadata, score }) =>
+ metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],
+ );
+
+ logPartnerSearchDebug("search", {
+ indexName: resolvedIndexName,
+ query,
+ candidateCount: candidates.length,
+ resultCount: hits.length,
+ hits,
+ });
+
+ return { hits };
+ },
+
+ async count(query) {
+ const count = (await getCandidates(query)).length;
+ logPartnerSearchDebug("count", {
+ indexName: resolvedIndexName,
+ query,
+ count,
+ });
+ return count;
+ },
+
+ async groupBy(query, field): Promise<PartnerSearchGroup[]> {
+ const groups = new Map<string, number>();
+ for (const candidate of await getCandidates(query)) {
+ for (const value of getGroupValues(candidate, field)) {
+ groups.set(value, (groups.get(value) ?? 0) + 1);
+ }
+ }
+
+ const result = Array.from(groups, ([value, count]) => ({
+ value: mapGroupValue(value),
+ count,
+ }));
+ logPartnerSearchDebug("groupBy", {
+ indexName: resolvedIndexName,
+ query,
+ field,
+ groupCount: result.length,
+ groups: result,
+ });
+ return result;
+ },
+
+ async waitForIndexing() {
+ const deadline = Date.now() + WAIT_FOR_INDEXING_TIMEOUT_MS;
+ while (Date.now() < deadline) {
+ const { pendingDocumentCount } = await withTransientRetry(() =>
+ index.info(),
+ );
+ if (pendingDocumentCount === 0) {
+ return;
+ }
+ await new Promise((resolve) =>
+ setTimeout(resolve, WAIT_FOR_INDEXING_POLL_MS),
+ );
+ }
+
+ throw new Error(
+ `Partner search waitForIndexing timed out after ${WAIT_FOR_INDEXING_TIMEOUT_MS}ms.`,
+ );
+ },
+
+ async upsert(documents) {
+ for (const documentBatch of chunk(documents, WRITE_BATCH_SIZE)) {
+ await withTransientRetry(() =>
+ index.upsert(documentBatch.map(serializeUpstashSearchDocument)),
+ );
+ }
+ candidateCache.clear();
+ },
+
+ async delete(documentIds) {
+ for (const documentIdBatch of chunk(documentIds, DELETE_BATCH_SIZE)) {
+ await withTransientRetry(() => index.delete(documentIdBatch));
+ }
+ candidateCache.clear();
+ },
+ };
+}
diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.ts
index dea2f68d35..8283d2266f 100644
--- a/apps/web/lib/api/partners/search/types.ts
+++ b/apps/web/lib/api/partners/search/types.ts
@@ -129,6 +129,7 @@ export interface PartnerSearchResult {
}
export interface PartnerSearchProvider {
+ mode?: "full" | "relevance-only";
search(query: PartnerSearchQuery): Promise<PartnerSearchResult>;
count(query: PartnerSearchCountQuery): Promise<number>;
groupBy(
diff --git a/apps/web/package.json b/apps/web/package.json
index 6bf535220f..f6028eacd7 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -60,6 +60,7 @@
"@upstash/qstash": "^2.11.0",
"@upstash/ratelimit": "^2.0.6",
"@upstash/redis": "^1.38.2",
+ "@upstash/search": "^0.1.7",
"@upstash/vector": "^1.2.2",
"@upstash/workflow": "^1.3.0",
"@vercel/edge-config": "^0.4.1",
diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.ts
index d37b2240c3..ca112d0269 100644
--- a/apps/web/scripts/dev/benchmark-partner-search.ts
+++ b/apps/web/scripts/dev/benchmark-partner-search.ts
@@ -251,6 +251,7 @@ async function main() {
}
const searchCases = await loadSearchCases(options.programId);
+ const relevanceOnly = searchProvider.mode === "relevance-only";
const runSearch = async (index: number): Promise<BenchmarkResult> => {
const searchCase = searchCases[index % searchCases.length];
@@ -259,7 +260,9 @@ async function main() {
search: searchCase.query,
page: 1,
pageSize: options.pageSize,
- sortBy: "totalSaleAmount" as const,
+ sortBy: relevanceOnly
+ ? ("relevance" as const)
+ : ("totalSaleAmount" as const),
sortOrder: "desc" as const,
};
const startedAt = performance.now();
@@ -267,7 +270,9 @@ async function main() {
try {
const [partners, count] = await Promise.all([
getPartners(filters, { searchProvider }),
- getPartnersCount<number>(filters, { searchProvider }),
+ relevanceOnly
+ ? Promise.resolve(null)
+ : getPartnersCount<number>(filters, { searchProvider }),
]);
if (partners.length === 0 || count === 0) {
@@ -296,7 +301,9 @@ async function main() {
`${options.requests.toLocaleString()} measured requests, ${options.warmupRequests.toLocaleString()} warm-up requests, concurrency ${options.concurrency}`,
);
console.log(
- `Each request runs the partner list and count paths in parallel across ${searchCases.length} search cases.`,
+ relevanceOnly
+ ? `Each request runs the relevance-ranked partner list path across ${searchCases.length} search cases.`
+ : `Each request runs the partner list and count paths in parallel across ${searchCases.length} search cases.`,
);
const warmupResults = await runWithConcurrency(
diff --git a/apps/web/scripts/dev/debug-partner-search.ts b/apps/web/scripts/dev/debug-partner-search.ts
index b50c665462..219aba7ab6 100644
--- a/apps/web/scripts/dev/debug-partner-search.ts
+++ b/apps/web/scripts/dev/debug-partner-search.ts
@@ -167,25 +167,28 @@ async function main() {
const providerName = process.env.PARTNER_SEARCH_PROVIDER?.trim();
const filters = status ? { status } : undefined;
const baseQuery = { programId, query, filters };
+ const relevanceOnly = searchProvider.mode === "relevance-only";
const startedAt = performance.now();
// Step 1: Compare relevance ranking with the website's field-sorted query
- const [matchingDocuments, relevanceResult, sortedResult] = await Promise.all([
- searchProvider.count(baseQuery),
- searchProvider.search({
- ...baseQuery,
- page: 1,
- pageSize: limit,
- }),
- searchProvider.search({
- ...baseQuery,
- page: 1,
- pageSize: limit,
- sort: { field: sortBy, order: sortOrder },
- }),
- ]);
-
- const allHits = [...relevanceResult.hits, ...sortedResult.hits];
+ const relevanceResult = await searchProvider.search({
+ ...baseQuery,
+ page: 1,
+ pageSize: limit,
+ });
+ const [matchingDocuments, sortedResult] = relevanceOnly
+ ? ([null, null] as const)
+ : await Promise.all([
+ searchProvider.count(baseQuery),
+ searchProvider.search({
+ ...baseQuery,
+ page: 1,
+ pageSize: limit,
+ sort: { field: sortBy, order: sortOrder },
+ }),
+ ]);
+
+ const allHits = [...relevanceResult.hits, ...(sortedResult?.hits ?? [])];
// Step 2: Build the canonical search documents from the database for comparison
const databaseDocuments = await getDatabaseDocuments(allHits);
@@ -197,12 +200,17 @@ async function main() {
query,
normalizedQuery: normalizePartnerSearchQuery(query),
status: status ?? "all",
- matchingDocuments,
+ matchingDocuments: matchingDocuments ?? "not supported",
elapsedMs: (performance.now() - startedAt).toFixed(1),
});
console.log(
"Provider scores are provider-defined. Compare result order and database values across providers.",
);
+ if (relevanceOnly) {
+ console.log(
+ "This provider supports relevance results only, so exact count and explicit field-order comparisons were skipped.",
+ );
+ }
reportResults({
label: "Provider relevance order",
@@ -211,13 +219,15 @@ async function main() {
sortBy,
normalizedQuery: normalizePartnerSearchQuery(query),
});
- reportResults({
- label: `${sortBy} ${sortOrder} — explicit field order`,
- hits: sortedResult.hits,
- databaseDocuments,
- sortBy,
- normalizedQuery: normalizePartnerSearchQuery(query),
- });
+ if (sortedResult) {
+ reportResults({
+ label: `${sortBy} ${sortOrder} — explicit field order`,
+ hits: sortedResult.hits,
+ databaseDocuments,
+ sortBy,
+ normalizedQuery: normalizePartnerSearchQuery(query),
+ });
+ }
}
main()
diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
new file mode 100644
index 0000000000..6e3deecd87
--- /dev/null
+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
@@ -0,0 +1,313 @@
+import {
+ createUpstashSearchPartnerSearchProvider,
+ type PartnerSearchDocument,
+} from "@/lib/api/partners/search";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const document: PartnerSearchDocument = {
+ id: "pge_test",
+ programId: "prog_test",
+ partnerId: "pn_test",
+ name: "Rafi Hasan",
+ email: "[email protected]",
+ companyName: "Dub Partners",
+ description: "Developer tools educator",
+ platformTypes: ["website", "youtube", "twitter"],
+ platformIdentifiers: ["rafi.dev", "@rafi-youtube", "@rafi-on-x"],
+ linkDomains: ["dub.sh"],
+ linkKeys: ["rafi"],
+ shortLinks: ["https://dub.sh/rafi"],
+ destinationUrls: ["https://example.com/referrals/rafi"],
+ status: "approved",
+ tenantId: null,
+ groupId: null,
+ country: "CA",
+ partnerTagIds: ["ptag_test"],
+ referredByPartnerId: null,
+ totalClicks: 100,
+ totalLeads: 20,
+ totalConversions: 10,
+ totalSaleAmount: 50_000,
+ totalCommissions: 10_000,
+ netRevenue: 40_000,
+ earningsPerClick: 5,
+ averageLifetimeValue: 5_000,
+ clickToLeadRate: 0.2,
+ clickToConversionRate: 0.1,
+ leadToConversionRate: 0.5,
+ returnOnAdSpend: 5,
+ createdAt: "2026-08-08T00:00:00.000Z",
+ updatedAt: "2026-08-08T00:00:00.000Z",
+};
+
+const mocks = vi.hoisted(() => ({
+ delete: vi.fn(),
+ info: vi.fn(),
+ search: vi.fn(),
+ upsert: vi.fn(),
+}));
+
+function createSearchIndexMock() {
+ return {
+ delete: mocks.delete,
+ info: mocks.info,
+ search: mocks.search,
+ upsert: mocks.upsert,
+ };
+}
+
+function getSearchResult(
+ overrides: {
+ id?: string;
+ partnerId?: string;
+ name?: string;
+ email?: string;
+ score?: number;
+ totalSaleAmount?: number;
+ } = {},
+) {
+ return {
+ id: overrides.id ?? document.id,
+ content: {
+ partnerId: overrides.partnerId ?? document.partnerId,
+ name: overrides.name ?? document.name,
+ email: overrides.email ?? document.email!,
+ companyName: document.companyName!,
+ description: document.description!,
+ platforms: document.platformIdentifiers.join(" "),
+ links: document.destinationUrls.join(" "),
+ emailNgrams: "exa xam amp",
+ },
+ metadata: {
+ programId: document.programId,
+ partnerId: overrides.partnerId ?? document.partnerId,
+ status: document.status,
+ tenantId: "null",
+ groupId: "null",
+ country: document.country!,
+ partnerTagIds: document.partnerTagIds,
+ referredByPartnerId: "null",
+ totalClicks: document.totalClicks,
+ totalLeads: document.totalLeads,
+ totalConversions: document.totalConversions,
+ totalSaleAmount: overrides.totalSaleAmount ?? document.totalSaleAmount,
+ totalCommissions: document.totalCommissions,
+ netRevenue: document.netRevenue,
+ earningsPerClick: document.earningsPerClick,
+ averageLifetimeValue: document.averageLifetimeValue!,
+ clickToLeadRate: document.clickToLeadRate!,
+ clickToConversionRate: document.clickToConversionRate!,
+ leadToConversionRate: document.leadToConversionRate!,
+ returnOnAdSpend: document.returnOnAdSpend!,
+ createdAt: new Date(document.createdAt).getTime(),
+ },
+ score: overrides.score ?? 0.9,
+ };
+}
+
+describe("Upstash Search partner search provider", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.delete.mockResolvedValue({ deleted: 0 });
+ mocks.info.mockResolvedValue({
+ documentCount: 0,
+ pendingDocumentCount: 0,
+ });
+ mocks.search.mockResolvedValue([]);
+ mocks.upsert.mockResolvedValue("Success");
+ });
+
+ it("stores every searchable assignment field within the content limit", async () => {
+ const provider = createUpstashSearchPartnerSearchProvider({
+ searchIndex: createSearchIndexMock(),
+ indexName: "test-index",
+ });
+
+ await provider.upsert([
+ {
+ ...document,
+ description: "description ".repeat(1_000),
+ destinationUrls: [`https://example.com/${"path/".repeat(1_000)}`],
+ },
+ ]);
+
+ const indexedDocument = mocks.upsert.mock.calls[0]![0][0];
+ expect(JSON.stringify(indexedDocument.content).length).toBeLessThan(4_096);
+ expect(indexedDocument.content).toEqual(
+ expect.objectContaining({
+ name: document.name,
+ email: document.email,
+ companyName: document.companyName,
+ platforms: expect.stringContaining("rafi-on-x"),
+ links: expect.stringContaining("dub.sh"),
+ emailNgrams: expect.stringContaining("exa"),
+ }),
+ );
+ expect(indexedDocument.metadata).toEqual(
+ expect.objectContaining({
+ programId: document.programId,
+ partnerId: document.partnerId,
+ totalSaleAmount: document.totalSaleAmount,
+ }),
+ );
+ });
+
+ it("adds an exact email n-gram search for partial email matches", async () => {
+ const emailResult = getSearchResult();
+ mocks.search.mockImplementation(async ({ query }: { query: string }) =>
+ query === "exa xam amp" ? [emailResult] : [],
+ );
+ const provider = createUpstashSearchPartnerSearchProvider({
+ searchIndex: createSearchIndexMock(),
+ indexName: "test-index",
+ });
+
+ await expect(
+ provider.search({
+ programId: document.programId,
+ query: "examp",
+ page: 1,
+ pageSize: 10,
+ }),
+ ).resolves.toEqual({
+ hits: [
+ {
+ id: document.id,
+ partnerId: document.partnerId,
+ score: emailResult.score,
+ },
+ ],
+ });
+
+ expect(mocks.search).toHaveBeenCalledTimes(2);
+ expect(mocks.search).toHaveBeenCalledWith(
+ expect.objectContaining({
+ query: "examp",
+ inputEnrichment: false,
+ reranking: false,
+ semanticWeight: 0,
+ }),
+ );
+ expect(mocks.search).toHaveBeenCalledWith(
+ expect.objectContaining({ query: "exa xam amp" }),
+ );
+ expect(mocks.search.mock.calls[0]![0].filter).toContain(
+ '@metadata.programId = "prog_test"',
+ );
+ });
+
+ it("returns relevance results without requiring a complete match set", async () => {
+ mocks.search.mockResolvedValue(
+ Array.from({ length: 100 }, (_, index) =>
+ getSearchResult({
+ id: `pge_${index}`,
+ partnerId: `pn_${index}`,
+ score: 1 - index / 100,
+ }),
+ ),
+ );
+ const provider = createUpstashSearchPartnerSearchProvider({
+ searchIndex: createSearchIndexMock(),
+ indexName: "test-index",
+ });
+
+ const result = await provider.search({
+ programId: document.programId,
+ query: "drew moore",
+ page: 1,
+ pageSize: 10,
+ });
+
+ expect(provider.mode).toBe("relevance-only");
+ expect(result.hits).toHaveLength(10);
+ expect(result.hits[0]).toEqual(
+ expect.objectContaining({ id: "pge_0", partnerId: "pn_0" }),
+ );
+ expect(mocks.search).toHaveBeenCalledWith(
+ expect.objectContaining({ query: "drew moore", limit: 10 }),
+ );
+ });
+
+ it("shares candidates across search, count, and grouping", async () => {
+ const lower = getSearchResult({
+ id: "pge_lower",
+ partnerId: "pn_lower",
+ score: 0.8,
+ totalSaleAmount: 100,
+ });
+ const higher = getSearchResult({
+ id: "pge_higher",
+ partnerId: "pn_higher",
+ score: 0.7,
+ totalSaleAmount: 1_000,
+ });
+ mocks.search.mockImplementation(async ({ query }: { query: string }) =>
+ query === "rafi" ? [lower, higher] : [],
+ );
+ const provider = createUpstashSearchPartnerSearchProvider({
+ searchIndex: createSearchIndexMock(),
+ indexName: "test-index",
+ });
+ const query = { programId: document.programId, query: "rafi" };
+
+ const [searchResult, count, groups] = await Promise.all([
+ provider.search({
+ ...query,
+ page: 1,
+ pageSize: 10,
+ sort: { field: "totalSaleAmount", order: "desc" },
+ }),
+ provider.count(query),
+ provider.groupBy(query, "country"),
+ ]);
+
+ expect(searchResult.hits.map(({ id }) => id)).toEqual([
+ "pge_higher",
+ "pge_lower",
+ ]);
+ expect(count).toBe(2);
+ expect(groups).toEqual([{ value: "CA", count: 2 }]);
+ expect(mocks.search).toHaveBeenCalledTimes(2);
+ });
+
+ it("rejects capped result sets instead of returning incomplete counts", async () => {
+ mocks.search.mockImplementation(async ({ query }: { query: string }) =>
+ query === "rafi"
+ ? Array.from({ length: 1_000 }, (_, index) =>
+ getSearchResult({
+ id: `pge_${index}`,
+ partnerId: `pn_${index}`,
+ }),
+ )
+ : [],
+ );
+ const provider = createUpstashSearchPartnerSearchProvider({
+ searchIndex: createSearchIndexMock(),
+ indexName: "test-index",
+ });
+
+ await expect(
+ provider.count({ programId: document.programId, query: "rafi" }),
+ ).rejects.toThrow("Exact pagination, counts, and facets are unavailable");
+ });
+
+ it("waits until pending documents finish indexing", async () => {
+ vi.useFakeTimers();
+ mocks.info
+ .mockResolvedValueOnce({ documentCount: 1, pendingDocumentCount: 1 })
+ .mockResolvedValueOnce({ documentCount: 2, pendingDocumentCount: 0 });
+ const provider = createUpstashSearchPartnerSearchProvider({
+ searchIndex: createSearchIndexMock(),
+ indexName: "test-index",
+ });
+
+ try {
+ const result = provider.waitForIndexing();
+ await vi.advanceTimersByTimeAsync(100);
+ await expect(result).resolves.toBeUndefined();
+ expect(mocks.info).toHaveBeenCalledTimes(2);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index a30e511162..d808ee13eb 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -158,6 +158,9 @@ importers:
'@upstash/redis':
specifier: ^1.38.2
version: 1.38.2
+ '@upstash/search':
+ specifier: ^0.1.7
+ version: 0.1.7
'@upstash/vector':
specifier: ^1.2.2
version: 1.2.2
@@ -6039,6 +6042,9 @@ packages:
'@upstash/[email protected]':
resolution: {integrity: sha512-RZ+JaRCVIS+ZTGnjOnDMr+/0aqDxsBb5rV6aW+Pys4SGELwr5lwE3UbLHRCHqMQ5Ulxj0b/CFBHfbmE175Otog==}
+ '@upstash/[email protected]':
+ resolution: {integrity: sha512-rgJ52TP0eUPLFo4K6TZtiC7qICbJnEwkT+TqaDI1vN8/Hk6qidgNC9dpnUUXCiqfwogty1rlSyBhYfk6PRgXjA==}
+
'@upstash/[email protected]':
resolution: {integrity: sha512-ptQ9xnxtKqmpNK52PCcHCszlPOLxIBfjsv7ty8RoF95pkjctS9rSjTQ3Pl9bx5VFbpDj+0dMXw88WLt6swDkgQ==}
@@ -19774,6 +19780,10 @@ snapshots:
dependencies:
uncrypto: 0.1.3
+ '@upstash/[email protected]':
+ dependencies:
+ '@upstash/vector': 1.2.2
+
'@upstash/[email protected]': {}