The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Add Upstash Search partner search provider
Rafi775399f8d11c
diff --git a/apps/web/.env.example b/apps/web/.env.exampleindex 0d09c879ea..486c67880a 100644--- a/apps/web/.env.example+++ b/apps/web/.env.example@@ -24,10 +24,16 @@ UPSTASH_REDIS_REST_URL=UPSTASH_REDIS_REST_TOKEN=# Full-text partner search+# Supported values: upstash-redis, upstash-searchPARTNER_SEARCH_PROVIDER=PARTNER_SEARCH_INDEX_NAME=partner-search-v1+PARTNER_UPSTASH_SEARCH_INDEX_NAME=partner-search-v1PARTNER_SEARCH_DEBUG=false+# Required when PARTNER_SEARCH_PROVIDER=upstash-search+UPSTASH_SEARCH_REST_URL=+UPSTASH_SEARCH_REST_TOKEN=+# Upstash QStash – required for queues and background jobs# Get your QStash Token here: https://upstash.com/docs/qstash/overall/getstartedQSTASH_URL="https://qstash-us-east-1.upstash.io"diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.tsindex 9d83ad4291..64ae4ddc11 100644--- a/apps/web/lib/api/partners/search/index.ts+++ b/apps/web/lib/api/partners/search/index.ts@@ -4,6 +4,7 @@ export * from "./order-search-results";export * from "./provider";export * from "./providers/mock";export * from "./providers/upstash-redis";+export * from "./providers/upstash-search";export * from "./searchable-values";export * from "./serialize-document";export * from "./sync-documents";diff --git a/apps/web/lib/api/partners/search/provider.ts b/apps/web/lib/api/partners/search/provider.tsindex c40aeb63be..eef9233cb0 100644--- a/apps/web/lib/api/partners/search/provider.ts+++ b/apps/web/lib/api/partners/search/provider.ts@@ -1,7 +1,9 @@-import type { PartnerSearchProvider } from "./types";import { createUpstashRedisPartnerSearchProvider } from "./providers/upstash-redis";+import { createUpstashSearchPartnerSearchProvider } from "./providers/upstash-search";+import type { PartnerSearchProvider } from "./types";let upstashRedisProvider: PartnerSearchProvider | null = null;+let upstashSearchProvider: PartnerSearchProvider | null = null;export function getPartnerSearchProvider(): PartnerSearchProvider | null {const provider = process.env.PARTNER_SEARCH_PROVIDER?.trim();@@ -10,13 +12,19 @@ export function getPartnerSearchProvider(): PartnerSearchProvider | null {return null;}- if (provider !== "upstash-redis") {- throw new Error(`Unsupported partner search provider: ${provider}`);+ if (provider === "upstash-redis") {+ if (!upstashRedisProvider) {+ upstashRedisProvider = createUpstashRedisPartnerSearchProvider();+ }+ return upstashRedisProvider;}- if (!upstashRedisProvider) {- upstashRedisProvider = createUpstashRedisPartnerSearchProvider();+ if (provider === "upstash-search") {+ if (!upstashSearchProvider) {+ upstashSearchProvider = createUpstashSearchPartnerSearchProvider();+ }+ return upstashSearchProvider;}- return upstashRedisProvider;+ throw new Error(`Unsupported partner search provider: ${provider}`);}diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.tsnew file mode 100644index 0000000000..760e782f28--- /dev/null+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts@@ -0,0 +1,671 @@+import { chunk } from "@dub/utils";+import { Search } from "@upstash/search";+import { normalizePartnerSearchQuery } from "../searchable-values";+import type {+ PartnerSearchCountQuery,+ PartnerSearchDocument,+ PartnerSearchGroup,+ PartnerSearchGroupField,+ PartnerSearchListFilter,+ PartnerSearchProvider,+ PartnerSearchQuery,+} from "../types";++const DEFAULT_INDEX_NAME = "partner-search-v1";+const NULL_VALUE = "\0__null:f47ac10b-58cc-4372-a567-0e02b2c3d479__";+const MAX_SEARCH_RESULTS = 100;+const WRITE_BATCH_SIZE = 100;+const DELETE_BATCH_SIZE = 1_000;+const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;+const WAIT_FOR_INDEXING_POLL_MS = 100;+const CANDIDATE_CACHE_TTL_MS = 1_000;+const MAX_CANDIDATE_CACHE_ENTRIES = 100;+const TRANSIENT_RETRY_ATTEMPTS = 2;+const QUERY_OPERATION_TIMEOUT_MS = 1_000;++interface UpstashSearchContent extends Record<string, unknown> {+ partnerId: string;+ name: string;+ email: string;+ companyName: string;+ description: string;+ platforms: string;+ links: string;+ emailNgrams: string;+}++interface UpstashSearchMetadata extends Record<string, unknown> {+ programId: string;+ partnerId: string;+ status: string;+ tenantId: string;+ groupId: string;+ country: string;+ partnerTagIds: string[];+ referredByPartnerId: string;+ totalClicks: number;+ totalLeads: number;+ totalConversions: number;+ totalSaleAmount: number;+ totalCommissions: number;+ netRevenue: number;+ earningsPerClick: number;+ averageLifetimeValue?: number;+ clickToLeadRate?: number;+ clickToConversionRate?: number;+ leadToConversionRate?: number;+ returnOnAdSpend?: number;+ createdAt: number;+}++interface UpstashSearchResult {+ id: string;+ content: UpstashSearchContent;+ metadata?: UpstashSearchMetadata;+ score: number;+}++interface UpstashSearchIndexClient {+ search(params: {+ query: string;+ limit: number;+ filter: string;+ reranking: boolean;+ semanticWeight: number;+ inputEnrichment: boolean;+ }): Promise<UpstashSearchResult[]>;+ upsert(+ documents:+ | {+ id: string;+ content: UpstashSearchContent;+ metadata: UpstashSearchMetadata;+ }+ | {+ id: string;+ content: UpstashSearchContent;+ metadata: UpstashSearchMetadata;+ }[],+ ): Promise<string>;+ delete(documentIds: string[]): Promise<{ deleted: number }>;+ info(): Promise<{ pendingDocumentCount: number; documentCount: number }>;+}++interface CreateUpstashSearchPartnerSearchProviderOptions {+ searchIndex?: UpstashSearchIndexClient;+ indexName?: string;+}++interface CandidateCacheEntry {+ expiresAt: number;+ promise: Promise<UpstashSearchResult[]>;+}++function getIndexName(indexName?: string): string {+ return (+ indexName ??+ process.env.PARTNER_UPSTASH_SEARCH_INDEX_NAME?.trim() ??+ process.env.PARTNER_SEARCH_INDEX_NAME?.trim() ??+ DEFAULT_INDEX_NAME+ );+}++function createSearchIndex(indexName: string): UpstashSearchIndexClient {+ const url = process.env.UPSTASH_SEARCH_REST_URL;+ const token = process.env.UPSTASH_SEARCH_REST_TOKEN;++ if (!url || !token) {+ throw new Error(+ "UPSTASH_SEARCH_REST_URL and UPSTASH_SEARCH_REST_TOKEN are required for partner search.",+ );+ }++ const client = new Search({ url, token, retry: false });+ return client.index<UpstashSearchContent, UpstashSearchMetadata>(indexName);+}++function truncate(value: string | null, maxLength: number): string {+ return value?.slice(0, maxLength) ?? "";+}++function joinValues(values: string[], maxLength: number): string {+ return truncate(values.join(" "), maxLength);+}++function getEmailNgrams(email: string | null): string {+ if (!email) {+ return "";+ }++ const normalized = normalizePartnerSearchQuery(email);+ if (normalized.length < 3) {+ return normalized;+ }++ return Array.from(+ new Set(+ Array.from({ length: normalized.length - 2 }, (_, index) =>+ normalized.slice(index, index + 3),+ ),+ ),+ ).join(" ");+}++function getQueryNgrams(query: string): string | null {+ if (query.length < 3 || /\s/u.test(query)) {+ return null;+ }++ return Array.from(+ new Set(+ Array.from({ length: query.length - 2 }, (_, index) =>+ query.slice(index, index + 3),+ ),+ ),+ ).join(" ");+}++function serializeUpstashSearchDocument(document: PartnerSearchDocument) {+ // Upstash Search limits searchable content to 4,096 characters. Give each+ // assignment field its own budget so a long description cannot crowd out+ // platforms, links, or the email n-grams used for partial email matching.+ const content: UpstashSearchContent = {+ partnerId: truncate(document.partnerId, 50),+ name: truncate(document.name, 200),+ email: truncate(normalizePartnerSearchQuery(document.email ?? ""), 320),+ companyName: truncate(document.companyName, 200),+ description: truncate(document.description, 700),+ platforms: joinValues(+ [...document.platformTypes, ...document.platformIdentifiers],+ 500,+ ),+ links: joinValues(+ [+ ...document.linkDomains,+ ...document.linkKeys,+ ...document.shortLinks,+ ...document.destinationUrls,+ ],+ 850,+ ),+ emailNgrams: truncate(getEmailNgrams(document.email), 800),+ };++ const metadata: UpstashSearchMetadata = {+ programId: document.programId,+ partnerId: document.partnerId,+ status: document.status,+ tenantId: document.tenantId ?? NULL_VALUE,+ groupId: document.groupId ?? NULL_VALUE,+ country: document.country ?? NULL_VALUE,+ partnerTagIds: document.partnerTagIds,+ referredByPartnerId: document.referredByPartnerId ?? NULL_VALUE,+ totalClicks: document.totalClicks,+ totalLeads: document.totalLeads,+ totalConversions: document.totalConversions,+ totalSaleAmount: document.totalSaleAmount,+ totalCommissions: document.totalCommissions,+ netRevenue: document.netRevenue,+ earningsPerClick: document.earningsPerClick,+ ...(document.averageLifetimeValue !== null && {+ averageLifetimeValue: document.averageLifetimeValue,+ }),+ ...(document.clickToLeadRate !== null && {+ clickToLeadRate: document.clickToLeadRate,+ }),+ ...(document.clickToConversionRate !== null && {+ clickToConversionRate: document.clickToConversionRate,+ }),+ ...(document.leadToConversionRate !== null && {+ leadToConversionRate: document.leadToConversionRate,+ }),+ ...(document.returnOnAdSpend !== null && {+ returnOnAdSpend: document.returnOnAdSpend,+ }),+ createdAt: new Date(document.createdAt).getTime(),+ };++ return { id: document.id, content, metadata };+}++function quoteFilterValue(value: string): string {+ return JSON.stringify(value);+}++function buildScalarListFilter(+ field: string,+ filter: PartnerSearchListFilter,+): string {+ const values = filter.values.map(quoteFilterValue).join(", ");+ return `@metadata.${field} ${filter.operator === "IN" ? "IN" : "NOT IN"} (${values})`;+}++function buildArrayListFilter(+ field: string,+ filter: PartnerSearchListFilter,+): string {+ const operator = filter.operator === "IN" ? "CONTAINS" : "NOT CONTAINS";+ const booleanOperator = filter.operator === "IN" ? " OR " : " AND ";+ return `(${filter.values+ .map((value) => `@metadata.${field} ${operator} ${quoteFilterValue(value)}`)+ .join(booleanOperator)})`;+}++function buildUpstashSearchFilter({+ programId,+ filters,+}: PartnerSearchCountQuery): string {+ const conditions = [`@metadata.programId = ${quoteFilterValue(programId)}`];++ if (filters?.status) {+ conditions.push(`@metadata.status = ${quoteFilterValue(filters.status)}`);+ }+ if (filters?.tenantId) {+ conditions.push(+ `@metadata.tenantId = ${quoteFilterValue(filters.tenantId)}`,+ );+ }+ if (filters?.partnerIds?.length) {+ conditions.push(+ buildScalarListFilter("partnerId", {+ values: filters.partnerIds,+ operator: "IN",+ }),+ );+ }++ const listFilters: [+ "groupId" | "country",+ PartnerSearchListFilter | undefined,+ ][] = [+ ["groupId", filters?.groupIds],+ ["country", filters?.countries],+ ];++ for (const [field, listFilter] of listFilters) {+ if (listFilter?.values.length) {+ conditions.push(buildScalarListFilter(field, listFilter));+ }+ }++ if (filters?.partnerTagIds?.values.length) {+ conditions.push(+ buildArrayListFilter("partnerTagIds", filters.partnerTagIds),+ );+ }+ if (filters?.referredByPartnerId) {+ conditions.push(+ `@metadata.referredByPartnerId = ${quoteFilterValue(filters.referredByPartnerId)}`,+ );+ }++ for (const [field, range] of Object.entries(filters?.metrics ?? {})) {+ if (range.min !== undefined) {+ conditions.push(`@metadata.${field} >= ${range.min}`);+ }+ if (range.max !== undefined) {+ conditions.push(`@metadata.${field} <= ${range.max}`);+ }+ }++ return conditions.join(" AND ");+}++function assertCompleteResults(results: UpstashSearchResult[], query: string) {+ // Upstash Search has no count, aggregation, offset, or search cursor API.+ // We can preserve the provider contract only while the complete match set+ // fits below its top-1,000 response limit. Reject capped results so the UI+ // never presents an incomplete count, facet, sorted page, or later page.+ if (results.length >= MAX_SEARCH_RESULTS) {+ throw new Error(+ `Upstash Search returned its ${MAX_SEARCH_RESULTS.toLocaleString()}-result limit for query ${JSON.stringify(query)}. Exact pagination, counts, and facets are unavailable for this query.`,+ );+ }+}++function isTransientError(error: unknown): boolean {+ if (error instanceof TypeError) {+ return true;+ }++ const message = error instanceof Error ? error.message : String(error);+ return /\b(429|500|502|503|504)\b|rate.?limit|timeout|timed out|fetch failed|network|ECONNRESET|ETIMEDOUT/i.test(+ message,+ );+}++function isTimeoutError(error: unknown): boolean {+ const message = error instanceof Error ? error.message : String(error);+ return /timeout|timed out|ETIMEDOUT/i.test(message);+}++async function withTransientRetry<T>(+ operation: () => Promise<T>,+ { retryTimeouts = true }: { retryTimeouts?: boolean } = {},+): Promise<T> {+ for (let attempt = 1; attempt <= TRANSIENT_RETRY_ATTEMPTS; attempt++) {+ try {+ return await operation();+ } catch (error) {+ if (+ attempt === TRANSIENT_RETRY_ATTEMPTS ||+ !isTransientError(error) ||+ (!retryTimeouts && isTimeoutError(error))+ ) {+ throw error;+ }++ await new Promise((resolve) =>+ setTimeout(resolve, 50 * attempt + Math.random() * 25),+ );+ }+ }++ throw new Error("Partner search operation failed.");+}++async function withQueryDeadline<T>(operation: () => Promise<T>): Promise<T> {+ let timeoutId: ReturnType<typeof setTimeout> | undefined;+ const timeout = new Promise<never>((_, reject) => {+ timeoutId = setTimeout(+ () =>+ reject(+ new Error(+ `Partner search query timed out after ${QUERY_OPERATION_TIMEOUT_MS}ms.`,+ ),+ ),+ QUERY_OPERATION_TIMEOUT_MS,+ );+ });++ try {+ return await Promise.race([+ withTransientRetry(operation, { retryTimeouts: false }),+ timeout,+ ]);+ } finally {+ clearTimeout(timeoutId);+ }+}++function mergeSearchResults(+ standardResults: UpstashSearchResult[],+ emailResults: UpstashSearchResult[],+): UpstashSearchResult[] {+ const resultsById = new Map<string, UpstashSearchResult>();++ for (const result of [...standardResults, ...emailResults]) {+ const existing = resultsById.get(result.id);+ if (!existing || result.score > existing.score) {+ resultsById.set(result.id, result);+ }+ }++ return Array.from(resultsById.values()).sort(+ (left, right) =>+ right.score - left.score || left.id.localeCompare(right.id),+ );+}++function compareResults(+ left: UpstashSearchResult,+ right: UpstashSearchResult,+ sort: NonNullable<PartnerSearchQuery["sort"]>,+): number {+ const leftValue = left.metadata?.[sort.field] as number | undefined;+ const rightValue = right.metadata?.[sort.field] as number | undefined;++ if (leftValue === rightValue) {+ return left.id.localeCompare(right.id);+ }+ if (leftValue === undefined) {+ return 1;+ }+ if (rightValue === undefined) {+ return -1;+ }++ const comparison = leftValue < rightValue ? -1 : 1;+ return sort.order === "asc" ? comparison : -comparison;+}++function getGroupValues(+ result: UpstashSearchResult,+ field: PartnerSearchGroupField,+): string[] {+ const metadata = result.metadata;+ if (!metadata) {+ return [];+ }+ if (field === "partnerTagId") {+ return metadata.partnerTagIds;+ }++ const value = metadata[field];+ if (typeof value !== "string") {+ return [];+ }+ if (field === "referredByPartnerId" && value === NULL_VALUE) {+ return [];+ }+ return [value];+}++function mapGroupValue(value: string): string | null {+ return value === NULL_VALUE ? null : value;+}++function logPartnerSearchDebug(+ operation: "search" | "count" | "groupBy",+ details: Record<string, unknown>,+) {+ if (process.env.PARTNER_SEARCH_DEBUG !== "true") {+ return;+ }++ console.log(`[Partner Search Debug] Upstash Search ${operation}`, details);+}++export function createUpstashSearchPartnerSearchProvider({+ searchIndex,+ indexName,+}: CreateUpstashSearchPartnerSearchProviderOptions = {}): PartnerSearchProvider {+ const resolvedIndexName = getIndexName(indexName);+ const index = searchIndex ?? createSearchIndex(resolvedIndexName);+ const candidateCache = new Map<string, CandidateCacheEntry>();++ async function searchCandidates(+ query: PartnerSearchCountQuery,+ { limit, requireComplete }: { limit: number; requireComplete: boolean },+ ): Promise<UpstashSearchResult[]> {+ const normalizedQuery = normalizePartnerSearchQuery(query.query);+ const filter = buildUpstashSearchFilter(query);+ const searchParams = {+ limit,+ filter,+ reranking: false,+ // This table needs literal full-text matching rather than related results+ // that happen to be semantically similar to a name, email, or URL.+ semanticWeight: 0,+ inputEnrichment: false,+ };+ const queryNgrams = getQueryNgrams(normalizedQuery);++ const [standardResults, rawEmailResults] = await withQueryDeadline(() =>+ Promise.all([+ index.search({ ...searchParams, query: normalizedQuery }),+ queryNgrams+ ? index.search({ ...searchParams, query: queryNgrams })+ : Promise.resolve([]),+ ]),+ );++ if (requireComplete) {+ assertCompleteResults(standardResults, normalizedQuery);+ assertCompleteResults(rawEmailResults, normalizedQuery);+ }++ const emailResults = rawEmailResults.filter(({ content }) =>+ content.email.includes(normalizedQuery),+ );+ return mergeSearchResults(standardResults, emailResults);+ }++ function getCandidates(+ query: PartnerSearchCountQuery,+ ): Promise<UpstashSearchResult[]> {+ const now = Date.now();+ const cacheKey = JSON.stringify({+ programId: query.programId,+ query: query.query,+ filters: query.filters,+ });+ const cached = candidateCache.get(cacheKey);+ if (cached && cached.expiresAt > now) {+ return cached.promise;+ }++ if (candidateCache.size >= MAX_CANDIDATE_CACHE_ENTRIES) {+ candidateCache.delete(candidateCache.keys().next().value!);+ }++ const promise = searchCandidates(query, {+ limit: MAX_SEARCH_RESULTS,+ requireComplete: true,+ }).catch((error) => {+ candidateCache.delete(cacheKey);+ throw error;+ });+ candidateCache.set(cacheKey, {+ expiresAt: now + CANDIDATE_CACHE_TTL_MS,+ promise,+ });+ return promise;+ }++ return {+ mode: "relevance-only",++ async search(query) {+ const { sort } = query;+ if (sort) {+ const candidates = [...(await getCandidates(query))];+ candidates.sort((left, right) => compareResults(left, right, sort));++ const offset = (query.page - 1) * query.pageSize;+ const hits = candidates+ .slice(offset, offset + query.pageSize)+ .flatMap(({ id, metadata, score }) =>+ metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],+ );++ logPartnerSearchDebug("search", {+ indexName: resolvedIndexName,+ query,+ candidateCount: candidates.length,+ resultCount: hits.length,+ hits,+ });++ return { hits };+ }++ const requestedResults = query.page * query.pageSize;+ if (requestedResults > MAX_SEARCH_RESULTS) {+ throw new Error(+ `Upstash Search relevance pagination is limited to the first ${MAX_SEARCH_RESULTS.toLocaleString()} results.`,+ );+ }++ // Upstash Search natively returns relevance-ranked results. Fetch enough+ // results to derive the requested page without requiring a complete set.+ const candidates = await searchCandidates(query, {+ limit: requestedResults,+ requireComplete: false,+ });+ const offset = (query.page - 1) * query.pageSize;+ const hits = candidates+ .slice(offset, offset + query.pageSize)+ .flatMap(({ id, metadata, score }) =>+ metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],+ );++ logPartnerSearchDebug("search", {+ indexName: resolvedIndexName,+ query,+ candidateCount: candidates.length,+ resultCount: hits.length,+ hits,+ });++ return { hits };+ },++ async count(query) {+ const count = (await getCandidates(query)).length;+ logPartnerSearchDebug("count", {+ indexName: resolvedIndexName,+ query,+ count,+ });+ return count;+ },++ async groupBy(query, field): Promise<PartnerSearchGroup[]> {+ const groups = new Map<string, number>();+ for (const candidate of await getCandidates(query)) {+ for (const value of getGroupValues(candidate, field)) {+ groups.set(value, (groups.get(value) ?? 0) + 1);+ }+ }++ const result = Array.from(groups, ([value, count]) => ({+ value: mapGroupValue(value),+ count,+ }));+ logPartnerSearchDebug("groupBy", {+ indexName: resolvedIndexName,+ query,+ field,+ groupCount: result.length,+ groups: result,+ });+ return result;+ },++ async waitForIndexing() {+ const deadline = Date.now() + WAIT_FOR_INDEXING_TIMEOUT_MS;+ while (Date.now() < deadline) {+ const { pendingDocumentCount } = await withTransientRetry(() =>+ index.info(),+ );+ if (pendingDocumentCount === 0) {+ return;+ }+ await new Promise((resolve) =>+ setTimeout(resolve, WAIT_FOR_INDEXING_POLL_MS),+ );+ }++ throw new Error(+ `Partner search waitForIndexing timed out after ${WAIT_FOR_INDEXING_TIMEOUT_MS}ms.`,+ );+ },++ async upsert(documents) {+ for (const documentBatch of chunk(documents, WRITE_BATCH_SIZE)) {+ await withTransientRetry(() =>+ index.upsert(documentBatch.map(serializeUpstashSearchDocument)),+ );+ }+ candidateCache.clear();+ },++ async delete(documentIds) {+ for (const documentIdBatch of chunk(documentIds, DELETE_BATCH_SIZE)) {+ await withTransientRetry(() => index.delete(documentIdBatch));+ }+ candidateCache.clear();+ },+ };+}diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.tsindex dea2f68d35..8283d2266f 100644--- a/apps/web/lib/api/partners/search/types.ts+++ b/apps/web/lib/api/partners/search/types.ts@@ -129,6 +129,7 @@ export interface PartnerSearchResult {}export interface PartnerSearchProvider {+ mode?: "full" | "relevance-only";search(query: PartnerSearchQuery): Promise<PartnerSearchResult>;count(query: PartnerSearchCountQuery): Promise<number>;groupBy(diff --git a/apps/web/package.json b/apps/web/package.jsonindex 6bf535220f..f6028eacd7 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -60,6 +60,7 @@"@upstash/qstash": "^2.11.0","@upstash/ratelimit": "^2.0.6","@upstash/redis": "^1.38.2",+ "@upstash/search": "^0.1.7","@upstash/vector": "^1.2.2","@upstash/workflow": "^1.3.0","@vercel/edge-config": "^0.4.1",diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.tsindex d37b2240c3..ca112d0269 100644--- a/apps/web/scripts/dev/benchmark-partner-search.ts+++ b/apps/web/scripts/dev/benchmark-partner-search.ts@@ -251,6 +251,7 @@ async function main() {}const searchCases = await loadSearchCases(options.programId);+ const relevanceOnly = searchProvider.mode === "relevance-only";const runSearch = async (index: number): Promise<BenchmarkResult> => {const searchCase = searchCases[index % searchCases.length];@@ -259,7 +260,9 @@ async function main() {search: searchCase.query,page: 1,pageSize: options.pageSize,- sortBy: "totalSaleAmount" as const,+ sortBy: relevanceOnly+ ? ("relevance" as const)+ : ("totalSaleAmount" as const),sortOrder: "desc" as const,};const startedAt = performance.now();@@ -267,7 +270,9 @@ async function main() {try {const [partners, count] = await Promise.all([getPartners(filters, { searchProvider }),- getPartnersCount<number>(filters, { searchProvider }),+ relevanceOnly+ ? Promise.resolve(null)+ : getPartnersCount<number>(filters, { searchProvider }),]);if (partners.length === 0 || count === 0) {@@ -296,7 +301,9 @@ async function main() {`${options.requests.toLocaleString()} measured requests, ${options.warmupRequests.toLocaleString()} warm-up requests, concurrency ${options.concurrency}`,);console.log(- `Each request runs the partner list and count paths in parallel across ${searchCases.length} search cases.`,+ relevanceOnly+ ? `Each request runs the relevance-ranked partner list path across ${searchCases.length} search cases.`+ : `Each request runs the partner list and count paths in parallel across ${searchCases.length} search cases.`,);const warmupResults = await runWithConcurrency(diff --git a/apps/web/scripts/dev/debug-partner-search.ts b/apps/web/scripts/dev/debug-partner-search.tsindex b50c665462..219aba7ab6 100644--- a/apps/web/scripts/dev/debug-partner-search.ts+++ b/apps/web/scripts/dev/debug-partner-search.ts@@ -167,25 +167,28 @@ async function main() {const providerName = process.env.PARTNER_SEARCH_PROVIDER?.trim();const filters = status ? { status } : undefined;const baseQuery = { programId, query, filters };+ const relevanceOnly = searchProvider.mode === "relevance-only";const startedAt = performance.now();// Step 1: Compare relevance ranking with the website's field-sorted query- const [matchingDocuments, relevanceResult, sortedResult] = await Promise.all([- searchProvider.count(baseQuery),- searchProvider.search({- ...baseQuery,- page: 1,- pageSize: limit,- }),- searchProvider.search({- ...baseQuery,- page: 1,- pageSize: limit,- sort: { field: sortBy, order: sortOrder },- }),- ]);-- const allHits = [...relevanceResult.hits, ...sortedResult.hits];+ const relevanceResult = await searchProvider.search({+ ...baseQuery,+ page: 1,+ pageSize: limit,+ });+ const [matchingDocuments, sortedResult] = relevanceOnly+ ? ([null, null] as const)+ : await Promise.all([+ searchProvider.count(baseQuery),+ searchProvider.search({+ ...baseQuery,+ page: 1,+ pageSize: limit,+ sort: { field: sortBy, order: sortOrder },+ }),+ ]);++ const allHits = [...relevanceResult.hits, ...(sortedResult?.hits ?? [])];// Step 2: Build the canonical search documents from the database for comparisonconst databaseDocuments = await getDatabaseDocuments(allHits);@@ -197,12 +200,17 @@ async function main() {query,normalizedQuery: normalizePartnerSearchQuery(query),status: status ?? "all",- matchingDocuments,+ matchingDocuments: matchingDocuments ?? "not supported",elapsedMs: (performance.now() - startedAt).toFixed(1),});console.log("Provider scores are provider-defined. Compare result order and database values across providers.",);+ if (relevanceOnly) {+ console.log(+ "This provider supports relevance results only, so exact count and explicit field-order comparisons were skipped.",+ );+ }reportResults({label: "Provider relevance order",@@ -211,13 +219,15 @@ async function main() {sortBy,normalizedQuery: normalizePartnerSearchQuery(query),});- reportResults({- label: `${sortBy} ${sortOrder} — explicit field order`,- hits: sortedResult.hits,- databaseDocuments,- sortBy,- normalizedQuery: normalizePartnerSearchQuery(query),- });+ if (sortedResult) {+ reportResults({+ label: `${sortBy} ${sortOrder} — explicit field order`,+ hits: sortedResult.hits,+ databaseDocuments,+ sortBy,+ normalizedQuery: normalizePartnerSearchQuery(query),+ });+ }}main()diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.tsnew file mode 100644index 0000000000..6e3deecd87--- /dev/null+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts@@ -0,0 +1,313 @@+import {+ createUpstashSearchPartnerSearchProvider,+ type PartnerSearchDocument,+} from "@/lib/api/partners/search";+import { beforeEach, describe, expect, it, vi } from "vitest";++const document: PartnerSearchDocument = {+ id: "pge_test",+ programId: "prog_test",+ partnerId: "pn_test",+ name: "Rafi Hasan",+ email: "[email protected]",+ companyName: "Dub Partners",+ description: "Developer tools educator",+ platformTypes: ["website", "youtube", "twitter"],+ platformIdentifiers: ["rafi.dev", "@rafi-youtube", "@rafi-on-x"],+ linkDomains: ["dub.sh"],+ linkKeys: ["rafi"],+ shortLinks: ["https://dub.sh/rafi"],+ destinationUrls: ["https://example.com/referrals/rafi"],+ status: "approved",+ tenantId: null,+ groupId: null,+ country: "CA",+ partnerTagIds: ["ptag_test"],+ referredByPartnerId: null,+ totalClicks: 100,+ totalLeads: 20,+ totalConversions: 10,+ totalSaleAmount: 50_000,+ totalCommissions: 10_000,+ netRevenue: 40_000,+ earningsPerClick: 5,+ averageLifetimeValue: 5_000,+ clickToLeadRate: 0.2,+ clickToConversionRate: 0.1,+ leadToConversionRate: 0.5,+ returnOnAdSpend: 5,+ createdAt: "2026-08-08T00:00:00.000Z",+ updatedAt: "2026-08-08T00:00:00.000Z",+};++const mocks = vi.hoisted(() => ({+ delete: vi.fn(),+ info: vi.fn(),+ search: vi.fn(),+ upsert: vi.fn(),+}));++function createSearchIndexMock() {+ return {+ delete: mocks.delete,+ info: mocks.info,+ search: mocks.search,+ upsert: mocks.upsert,+ };+}++function getSearchResult(+ overrides: {+ id?: string;+ partnerId?: string;+ name?: string;+ email?: string;+ score?: number;+ totalSaleAmount?: number;+ } = {},+) {+ return {+ id: overrides.id ?? document.id,+ content: {+ partnerId: overrides.partnerId ?? document.partnerId,+ name: overrides.name ?? document.name,+ email: overrides.email ?? document.email!,+ companyName: document.companyName!,+ description: document.description!,+ platforms: document.platformIdentifiers.join(" "),+ links: document.destinationUrls.join(" "),+ emailNgrams: "exa xam amp",+ },+ metadata: {+ programId: document.programId,+ partnerId: overrides.partnerId ?? document.partnerId,+ status: document.status,+ tenantId: "null",+ groupId: "null",+ country: document.country!,+ partnerTagIds: document.partnerTagIds,+ referredByPartnerId: "null",+ totalClicks: document.totalClicks,+ totalLeads: document.totalLeads,+ totalConversions: document.totalConversions,+ totalSaleAmount: overrides.totalSaleAmount ?? document.totalSaleAmount,+ totalCommissions: document.totalCommissions,+ netRevenue: document.netRevenue,+ earningsPerClick: document.earningsPerClick,+ averageLifetimeValue: document.averageLifetimeValue!,+ clickToLeadRate: document.clickToLeadRate!,+ clickToConversionRate: document.clickToConversionRate!,+ leadToConversionRate: document.leadToConversionRate!,+ returnOnAdSpend: document.returnOnAdSpend!,+ createdAt: new Date(document.createdAt).getTime(),+ },+ score: overrides.score ?? 0.9,+ };+}++describe("Upstash Search partner search provider", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.delete.mockResolvedValue({ deleted: 0 });+ mocks.info.mockResolvedValue({+ documentCount: 0,+ pendingDocumentCount: 0,+ });+ mocks.search.mockResolvedValue([]);+ mocks.upsert.mockResolvedValue("Success");+ });++ it("stores every searchable assignment field within the content limit", async () => {+ const provider = createUpstashSearchPartnerSearchProvider({+ searchIndex: createSearchIndexMock(),+ indexName: "test-index",+ });++ await provider.upsert([+ {+ ...document,+ description: "description ".repeat(1_000),+ destinationUrls: [`https://example.com/${"path/".repeat(1_000)}`],+ },+ ]);++ const indexedDocument = mocks.upsert.mock.calls[0]![0][0];+ expect(JSON.stringify(indexedDocument.content).length).toBeLessThan(4_096);+ expect(indexedDocument.content).toEqual(+ expect.objectContaining({+ name: document.name,+ email: document.email,+ companyName: document.companyName,+ platforms: expect.stringContaining("rafi-on-x"),+ links: expect.stringContaining("dub.sh"),+ emailNgrams: expect.stringContaining("exa"),+ }),+ );+ expect(indexedDocument.metadata).toEqual(+ expect.objectContaining({+ programId: document.programId,+ partnerId: document.partnerId,+ totalSaleAmount: document.totalSaleAmount,+ }),+ );+ });++ it("adds an exact email n-gram search for partial email matches", async () => {+ const emailResult = getSearchResult();+ mocks.search.mockImplementation(async ({ query }: { query: string }) =>+ query === "exa xam amp" ? [emailResult] : [],+ );+ const provider = createUpstashSearchPartnerSearchProvider({+ searchIndex: createSearchIndexMock(),+ indexName: "test-index",+ });++ await expect(+ provider.search({+ programId: document.programId,+ query: "examp",+ page: 1,+ pageSize: 10,+ }),+ ).resolves.toEqual({+ hits: [+ {+ id: document.id,+ partnerId: document.partnerId,+ score: emailResult.score,+ },+ ],+ });++ expect(mocks.search).toHaveBeenCalledTimes(2);+ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({+ query: "examp",+ inputEnrichment: false,+ reranking: false,+ semanticWeight: 0,+ }),+ );+ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({ query: "exa xam amp" }),+ );+ expect(mocks.search.mock.calls[0]![0].filter).toContain(+ '@metadata.programId = "prog_test"',+ );+ });++ it("returns relevance results without requiring a complete match set", async () => {+ mocks.search.mockResolvedValue(+ Array.from({ length: 100 }, (_, index) =>+ getSearchResult({+ id: `pge_${index}`,+ partnerId: `pn_${index}`,+ score: 1 - index / 100,+ }),+ ),+ );+ const provider = createUpstashSearchPartnerSearchProvider({+ searchIndex: createSearchIndexMock(),+ indexName: "test-index",+ });++ const result = await provider.search({+ programId: document.programId,+ query: "drew moore",+ page: 1,+ pageSize: 10,+ });++ expect(provider.mode).toBe("relevance-only");+ expect(result.hits).toHaveLength(10);+ expect(result.hits[0]).toEqual(+ expect.objectContaining({ id: "pge_0", partnerId: "pn_0" }),+ );+ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({ query: "drew moore", limit: 10 }),+ );+ });++ it("shares candidates across search, count, and grouping", async () => {+ const lower = getSearchResult({+ id: "pge_lower",+ partnerId: "pn_lower",+ score: 0.8,+ totalSaleAmount: 100,+ });+ const higher = getSearchResult({+ id: "pge_higher",+ partnerId: "pn_higher",+ score: 0.7,+ totalSaleAmount: 1_000,+ });+ mocks.search.mockImplementation(async ({ query }: { query: string }) =>+ query === "rafi" ? [lower, higher] : [],+ );+ const provider = createUpstashSearchPartnerSearchProvider({+ searchIndex: createSearchIndexMock(),+ indexName: "test-index",+ });+ const query = { programId: document.programId, query: "rafi" };++ const [searchResult, count, groups] = await Promise.all([+ provider.search({+ ...query,+ page: 1,+ pageSize: 10,+ sort: { field: "totalSaleAmount", order: "desc" },+ }),+ provider.count(query),+ provider.groupBy(query, "country"),+ ]);++ expect(searchResult.hits.map(({ id }) => id)).toEqual([+ "pge_higher",+ "pge_lower",+ ]);+ expect(count).toBe(2);+ expect(groups).toEqual([{ value: "CA", count: 2 }]);+ expect(mocks.search).toHaveBeenCalledTimes(2);+ });++ it("rejects capped result sets instead of returning incomplete counts", async () => {+ mocks.search.mockImplementation(async ({ query }: { query: string }) =>+ query === "rafi"+ ? Array.from({ length: 1_000 }, (_, index) =>+ getSearchResult({+ id: `pge_${index}`,+ partnerId: `pn_${index}`,+ }),+ )+ : [],+ );+ const provider = createUpstashSearchPartnerSearchProvider({+ searchIndex: createSearchIndexMock(),+ indexName: "test-index",+ });++ await expect(+ provider.count({ programId: document.programId, query: "rafi" }),+ ).rejects.toThrow("Exact pagination, counts, and facets are unavailable");+ });++ it("waits until pending documents finish indexing", async () => {+ vi.useFakeTimers();+ mocks.info+ .mockResolvedValueOnce({ documentCount: 1, pendingDocumentCount: 1 })+ .mockResolvedValueOnce({ documentCount: 2, pendingDocumentCount: 0 });+ const provider = createUpstashSearchPartnerSearchProvider({+ searchIndex: createSearchIndexMock(),+ indexName: "test-index",+ });++ try {+ const result = provider.waitForIndexing();+ await vi.advanceTimersByTimeAsync(100);+ await expect(result).resolves.toBeUndefined();+ expect(mocks.info).toHaveBeenCalledTimes(2);+ } finally {+ vi.useRealTimers();+ }+ });+});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex a30e511162..d808ee13eb 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -158,6 +158,9 @@ importers:'@upstash/redis':specifier: ^1.38.2version: 1.38.2+ '@upstash/search':+ specifier: ^0.1.7+ version: 0.1.7'@upstash/vector':specifier: ^1.2.2version: 1.2.2@@ -6039,6 +6042,9 @@ packages:'@upstash/[email protected]':resolution: {integrity: sha512-RZ+JaRCVIS+ZTGnjOnDMr+/0aqDxsBb5rV6aW+Pys4SGELwr5lwE3UbLHRCHqMQ5Ulxj0b/CFBHfbmE175Otog==}+ '@upstash/[email protected]':+ resolution: {integrity: sha512-rgJ52TP0eUPLFo4K6TZtiC7qICbJnEwkT+TqaDI1vN8/Hk6qidgNC9dpnUUXCiqfwogty1rlSyBhYfk6PRgXjA==}+'@upstash/[email protected]':resolution: {integrity: sha512-ptQ9xnxtKqmpNK52PCcHCszlPOLxIBfjsv7ty8RoF95pkjctS9rSjTQ3Pl9bx5VFbpDj+0dMXw88WLt6swDkgQ==}@@ -19774,6 +19780,10 @@ snapshots:dependencies:uncrypto: 0.1.3+ '@upstash/[email protected]':+ dependencies:+ '@upstash/vector': 1.2.2+'@upstash/[email protected]': {}'@upstash/[email protected]([email protected])':