The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Delegate partner search filtering and aggregations to the database
Rafi7ceff9af2be8
diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.tsindex 59f9cf3e6c..c73ab1b49f 100644--- a/apps/web/lib/api/partners/search/providers/upstash-search.ts+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts@@ -6,24 +6,17 @@ import {} from "../constants";import { normalizePartnerSearchQuery } from "../searchable-values";import type {- PartnerSearchCountQuery,+ PartnerSearchCandidateQuery,PartnerSearchDocument,- PartnerSearchGroup,- PartnerSearchGroupField,- PartnerSearchListFilter,PartnerSearchProvider,- PartnerSearchQuery,} from "../types";const DEFAULT_INDEX_NAME = "partner-search-v1";-const NULL_VALUE = "\0__null:f47ac10b-58cc-4372-a567-0e02b2c3d479__";const MAX_SEARCH_RESULTS = PARTNER_SEARCH_CANDIDATE_LIMIT;const WRITE_BATCH_SIZE = 100;const DELETE_BATCH_SIZE = 1_000;const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;const WAIT_FOR_INDEXING_POLL_MS = 100;-const CANDIDATE_CACHE_TTL_MS = 1_000;-const MAX_CANDIDATE_CACHE_ENTRIES = 100;const TRANSIENT_RETRY_ATTEMPTS = 2;const QUERY_OPERATION_TIMEOUT_MS = 1_000;@@ -40,26 +33,6 @@ interface UpstashSearchContent extends Record<string, unknown> {interface UpstashSearchMetadata extends Record<string, unknown> {programId: string;- partnerId: string;- status: string;- tenantId: string;- groupId: string;- country: string;- partnerTagIds: string[];- referredByPartnerId: string;- totalClicks: number;- totalLeads: number;- totalConversions: number;- totalSaleAmount: number;- totalCommissions: number;- netRevenue: number;- earningsPerClick: number;- averageLifetimeValue?: number;- clickToLeadRate?: number;- clickToConversionRate?: number;- leadToConversionRate?: number;- returnOnAdSpend?: number;- createdAt: number;}interface UpstashSearchResult {@@ -100,11 +73,6 @@ interface CreateUpstashSearchPartnerSearchProviderOptions {indexName?: string;}-interface CandidateCacheEntry {- expiresAt: number;- promise: Promise<UpstashSearchResult[]>;-}-function getIndexName(indexName?: string): string {return (indexName ??@@ -195,39 +163,10 @@ function serializeUpstashSearchDocument(document: PartnerSearchDocument) {emailNgrams: truncate(getEmailNgrams(document.email), 800),};- const metadata: UpstashSearchMetadata = {- programId: document.programId,- partnerId: document.partnerId,- status: document.status,- tenantId: document.tenantId ?? NULL_VALUE,- groupId: document.groupId ?? NULL_VALUE,- country: document.country ?? NULL_VALUE,- partnerTagIds: document.partnerTagIds,- referredByPartnerId: document.referredByPartnerId ?? NULL_VALUE,- totalClicks: document.totalClicks,- totalLeads: document.totalLeads,- totalConversions: document.totalConversions,- totalSaleAmount: document.totalSaleAmount,- totalCommissions: document.totalCommissions,- netRevenue: document.netRevenue,- earningsPerClick: document.earningsPerClick,- ...(document.averageLifetimeValue !== null && {- averageLifetimeValue: document.averageLifetimeValue,- }),- ...(document.clickToLeadRate !== null && {- clickToLeadRate: document.clickToLeadRate,- }),- ...(document.clickToConversionRate !== null && {- clickToConversionRate: document.clickToConversionRate,- }),- ...(document.leadToConversionRate !== null && {- leadToConversionRate: document.leadToConversionRate,- }),- ...(document.returnOnAdSpend !== null && {- returnOnAdSpend: document.returnOnAdSpend,- }),- createdAt: new Date(document.createdAt).getTime(),- };+ // Program scope is the only metadata Search needs. The database applies+ // status, group, tag, country, metric, and sorting constraints after it+ // receives the relevance-ranked enrollment IDs.+ const metadata: UpstashSearchMetadata = { programId: document.programId };return { id: document.id, content, metadata };}@@ -236,95 +175,8 @@ function quoteFilterValue(value: string): string {return JSON.stringify(value);}-function buildScalarListFilter(- field: string,- filter: PartnerSearchListFilter,-): string {- const values = filter.values.map(quoteFilterValue).join(", ");- return `@metadata.${field} ${filter.operator === "IN" ? "IN" : "NOT IN"} (${values})`;-}--function buildArrayListFilter(- field: string,- filter: PartnerSearchListFilter,-): string {- const operator = filter.operator === "IN" ? "CONTAINS" : "NOT CONTAINS";- const booleanOperator = filter.operator === "IN" ? " OR " : " AND ";- return `(${filter.values- .map((value) => `@metadata.${field} ${operator} ${quoteFilterValue(value)}`)- .join(booleanOperator)})`;-}--function buildUpstashSearchFilter({- programId,- filters,-}: PartnerSearchCountQuery): string {- const conditions = [`@metadata.programId = ${quoteFilterValue(programId)}`];-- if (filters?.status) {- conditions.push(`@metadata.status = ${quoteFilterValue(filters.status)}`);- }- if (filters?.tenantId) {- conditions.push(- `@metadata.tenantId = ${quoteFilterValue(filters.tenantId)}`,- );- }- if (filters?.partnerIds?.length) {- conditions.push(- buildScalarListFilter("partnerId", {- values: filters.partnerIds,- operator: "IN",- }),- );- }-- const listFilters: [- "groupId" | "country",- PartnerSearchListFilter | undefined,- ][] = [- ["groupId", filters?.groupIds],- ["country", filters?.countries],- ];-- for (const [field, listFilter] of listFilters) {- if (listFilter?.values.length) {- conditions.push(buildScalarListFilter(field, listFilter));- }- }-- if (filters?.partnerTagIds?.values.length) {- conditions.push(- buildArrayListFilter("partnerTagIds", filters.partnerTagIds),- );- }- if (filters?.referredByPartnerId) {- conditions.push(- `@metadata.referredByPartnerId = ${quoteFilterValue(filters.referredByPartnerId)}`,- );- }-- for (const [field, range] of Object.entries(filters?.metrics ?? {})) {- if (range.min !== undefined) {- conditions.push(`@metadata.${field} >= ${range.min}`);- }- if (range.max !== undefined) {- conditions.push(`@metadata.${field} <= ${range.max}`);- }- }-- return conditions.join(" AND ");-}--function assertCompleteResults(results: UpstashSearchResult[], query: string) {- // Upstash Search has no count, aggregation, offset, or search cursor API.- // We can preserve the provider contract only while the complete match set- // fits below its top-1,000 response limit. Reject capped results so the UI- // never presents an incomplete count, facet, sorted page, or later page.- if (results.length >= MAX_SEARCH_RESULTS) {- throw new Error(- `Upstash Search returned its ${MAX_SEARCH_RESULTS.toLocaleString()}-result limit for query ${JSON.stringify(query)}. Exact pagination, counts, and facets are unavailable for this query.`,- );- }+function buildUpstashSearchFilter(programId: string): string {+ return `@metadata.programId = ${quoteFilterValue(programId)}`;}function isTransientError(error: unknown): boolean {@@ -411,54 +263,6 @@ function mergeSearchResults();}-function compareResults(- left: UpstashSearchResult,- right: UpstashSearchResult,- sort: NonNullable<PartnerSearchQuery["sort"]>,-): number {- const leftValue = left.metadata?.[sort.field] as number | undefined;- const rightValue = right.metadata?.[sort.field] as number | undefined;-- if (leftValue === rightValue) {- return left.id.localeCompare(right.id);- }- if (leftValue === undefined) {- return 1;- }- if (rightValue === undefined) {- return -1;- }-- const comparison = leftValue < rightValue ? -1 : 1;- return sort.order === "asc" ? comparison : -comparison;-}--function getGroupValues(- result: UpstashSearchResult,- field: PartnerSearchGroupField,-): string[] {- const metadata = result.metadata;- if (!metadata) {- return [];- }- if (field === "partnerTagId") {- return metadata.partnerTagIds;- }-- const value = metadata[field];- if (typeof value !== "string") {- return [];- }- if (field === "referredByPartnerId" && value === NULL_VALUE) {- return [];- }- return [value];-}--function mapGroupValue(value: string): string | null {- return value === NULL_VALUE ? null : value;-}-function logPartnerSearchDebug(operation: "search" | "count" | "groupBy",details: Record<string, unknown>,@@ -476,14 +280,14 @@ export function createUpstashSearchPartnerSearchProvider({}: CreateUpstashSearchPartnerSearchProviderOptions = {}): PartnerSearchProvider {const resolvedIndexName = getIndexName(indexName);const index = searchIndex ?? createSearchIndex(resolvedIndexName);- const candidateCache = new Map<string, CandidateCacheEntry>();-- async function findSearchCandidates(- query: PartnerSearchCountQuery,- { limit, requireComplete }: { limit: number; requireComplete: boolean },- ): Promise<UpstashSearchResult[]> {- const normalizedQuery = normalizePartnerSearchQuery(query.query);- const filter = buildUpstashSearchFilter(query);++ async function findSearchCandidates({+ programId,+ query,+ limit,+ }: PartnerSearchCandidateQuery): Promise<UpstashSearchResult[]> {+ const normalizedQuery = normalizePartnerSearchQuery(query);+ const filter = buildUpstashSearchFilter(programId);const searchParams = {limit,filter,@@ -504,61 +308,24 @@ export function createUpstashSearchPartnerSearchProvider({]),);- if (requireComplete) {- assertCompleteResults(standardResults, normalizedQuery);- assertCompleteResults(rawEmailResults, normalizedQuery);- }-const emailResults = rawEmailResults.filter(({ content }) =>content.email.includes(normalizedQuery),);return mergeSearchResults(standardResults, emailResults);}- function getCandidates(- query: PartnerSearchCountQuery,- ): Promise<UpstashSearchResult[]> {- const now = Date.now();- const cacheKey = JSON.stringify({- programId: query.programId,- query: query.query,- filters: query.filters,- });- const cached = candidateCache.get(cacheKey);- if (cached && cached.expiresAt > now) {- return cached.promise;- }-- if (candidateCache.size >= MAX_CANDIDATE_CACHE_ENTRIES) {- candidateCache.delete(candidateCache.keys().next().value!);- }-- const promise = findSearchCandidates(query, {- limit: MAX_SEARCH_RESULTS,- requireComplete: true,- }).catch((error) => {- candidateCache.delete(cacheKey);- throw error;- });- candidateCache.set(cacheKey, {- expiresAt: now + CANDIDATE_CACHE_TTL_MS,- promise,- });- return promise;- }-return {async searchCandidates({ programId, query, limit }) {validatePartnerSearchCandidateLimit(limit);- const candidates = await findSearchCandidates(- { programId, query },- { limit, requireComplete: false },- );- const hits = candidates- .slice(0, limit)- .flatMap(({ id, metadata, score }) =>- metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],- );+ const candidates = await findSearchCandidates({+ programId,+ query,+ limit,+ });+ const hits = candidates.slice(0, limit).map(({ id, score }) => ({+ id,+ score,+ }));logPartnerSearchDebug("search", {indexName: resolvedIndexName,@@ -572,48 +339,33 @@ export function createUpstashSearchPartnerSearchProvider({},async search(query) {- const { sort } = query;- if (sort) {- const candidates = [...(await getCandidates(query))];- candidates.sort((left, right) => compareResults(left, right, sort));-- const offset = (query.page - 1) * query.pageSize;- const hits = candidates- .slice(offset, offset + query.pageSize)- .flatMap(({ id, metadata, score }) =>- metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],- );-- logPartnerSearchDebug("search", {- indexName: resolvedIndexName,- query,- candidateCount: candidates.length,- resultCount: hits.length,- hits,- });-- return { hits };+ if (query.sort) {+ throw new Error(+ "Upstash Search partner search supports relevance ordering only.",+ );+ }+ if (query.filters && Object.keys(query.filters).length > 0) {+ throw new Error(+ "Upstash Search partner search filters must be applied by the database.",+ );}- const requestedResults = query.page * query.pageSize;+ const offset = (query.page - 1) * query.pageSize;+ const requestedResults = offset + query.pageSize;if (requestedResults > MAX_SEARCH_RESULTS) {throw new Error(`Upstash Search relevance pagination is limited to the first ${MAX_SEARCH_RESULTS.toLocaleString()} results.`,);}- // Upstash Search natively returns relevance-ranked results. Fetch enough- // results to derive the requested page without requiring a complete set.- const candidates = await findSearchCandidates(query, {+ const candidates = await findSearchCandidates({+ programId: query.programId,+ query: query.query,limit: requestedResults,- requireComplete: false,});- const offset = (query.page - 1) * query.pageSize;const hits = candidates.slice(offset, offset + query.pageSize)- .flatMap(({ id, metadata, score }) =>- metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],- );+ .map(({ id, score }) => ({ id, score }));logPartnerSearchDebug("search", {indexName: resolvedIndexName,@@ -626,36 +378,16 @@ export function createUpstashSearchPartnerSearchProvider({return { hits };},- async count(query) {- const count = (await getCandidates(query)).length;- logPartnerSearchDebug("count", {- indexName: resolvedIndexName,- query,- count,- });- return count;+ async count() {+ throw new Error(+ "Upstash Search partner search counts must be calculated by the database.",+ );},- async groupBy(query, field): Promise<PartnerSearchGroup[]> {- const groups = new Map<string, number>();- for (const candidate of await getCandidates(query)) {- for (const value of getGroupValues(candidate, field)) {- groups.set(value, (groups.get(value) ?? 0) + 1);- }- }-- const result = Array.from(groups, ([value, count]) => ({- value: mapGroupValue(value),- count,- }));- logPartnerSearchDebug("groupBy", {- indexName: resolvedIndexName,- query,- field,- groupCount: result.length,- groups: result,- });- return result;+ async groupBy() {+ throw new Error(+ "Upstash Search partner search groups must be calculated by the database.",+ );},async waitForIndexing() {@@ -683,14 +415,12 @@ export function createUpstashSearchPartnerSearchProvider({index.upsert(documentBatch.map(serializeUpstashSearchDocument)),);}- candidateCache.clear();},async delete(documentIds) {for (const documentIdBatch of chunk(documentIds, DELETE_BATCH_SIZE)) {await withTransientRetry(() => index.delete(documentIdBatch));}- candidateCache.clear();},};}diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.tsindex b337fa1b1c..7dd39c58b5 100644--- a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts@@ -59,17 +59,15 @@ function createSearchIndexMock() {function getSearchResult(overrides: {id?: string;- partnerId?: string;name?: string;email?: string;score?: number;- totalSaleAmount?: number;} = {},) {return {id: overrides.id ?? document.id,content: {- partnerId: overrides.partnerId ?? document.partnerId,+ partnerId: document.partnerId,name: overrides.name ?? document.name,email: overrides.email ?? document.email!,companyName: document.companyName!,@@ -80,26 +78,6 @@ function getSearchResult(},metadata: {programId: document.programId,- partnerId: overrides.partnerId ?? document.partnerId,- status: document.status,- tenantId: "null",- groupId: "null",- country: document.country!,- partnerTagIds: document.partnerTagIds,- referredByPartnerId: "null",- totalClicks: document.totalClicks,- totalLeads: document.totalLeads,- totalConversions: document.totalConversions,- totalSaleAmount: overrides.totalSaleAmount ?? document.totalSaleAmount,- totalCommissions: document.totalCommissions,- netRevenue: document.netRevenue,- earningsPerClick: document.earningsPerClick,- averageLifetimeValue: document.averageLifetimeValue!,- clickToLeadRate: document.clickToLeadRate!,- clickToConversionRate: document.clickToConversionRate!,- leadToConversionRate: document.leadToConversionRate!,- returnOnAdSpend: document.returnOnAdSpend!,- createdAt: new Date(document.createdAt).getTime(),},score: overrides.score ?? 0.9,};@@ -136,6 +114,7 @@ describe("Upstash Search partner search provider", () => {expect(indexedDocument.content).toEqual(expect.objectContaining({name: document.name,+ partnerId: document.partnerId,email: document.email,companyName: document.companyName,platforms: expect.stringContaining("rafi-on-x"),@@ -143,13 +122,9 @@ describe("Upstash Search partner search provider", () => {emailNgrams: expect.stringContaining("exa"),}),);- expect(indexedDocument.metadata).toEqual(- expect.objectContaining({- programId: document.programId,- partnerId: document.partnerId,- totalSaleAmount: document.totalSaleAmount,- }),- );+ expect(indexedDocument.metadata).toEqual({+ programId: document.programId,+ });});it("adds an exact email n-gram search for partial email matches", async () => {@@ -173,7 +148,6 @@ describe("Upstash Search partner search provider", () => {hits: [{id: document.id,- partnerId: document.partnerId,score: emailResult.score,},],@@ -201,7 +175,6 @@ describe("Upstash Search partner search provider", () => {Array.from({ length: 100 }, (_, index) =>getSearchResult({id: `pge_${index}`,- partnerId: `pn_${index}`,score: 1 - index / 100,}),),@@ -220,7 +193,7 @@ describe("Upstash Search partner search provider", () => {expect(result.hits).toHaveLength(10);expect(result.hits[0]).toEqual(- expect.objectContaining({ id: "pge_0", partnerId: "pn_0" }),+ expect.objectContaining({ id: "pge_0", score: 1 }),);expect(mocks.search).toHaveBeenCalledWith(expect.objectContaining({ query: "drew moore", limit: 10 }),@@ -237,67 +210,47 @@ describe("Upstash Search partner search provider", () => {);});- it("shares candidates across search, count, and grouping", async () => {- const lower = getSearchResult({- id: "pge_lower",- partnerId: "pn_lower",- score: 0.8,- totalSaleAmount: 100,- });- const higher = getSearchResult({- id: "pge_higher",- partnerId: "pn_higher",- score: 0.7,- totalSaleAmount: 1_000,- });- mocks.search.mockImplementation(async ({ query }: { query: string }) =>- query === "rafi" ? [lower, higher] : [],- );+ it("leaves filtering and sorting to the database", async () => {const provider = createUpstashSearchPartnerSearchProvider({searchIndex: createSearchIndexMock(),indexName: "test-index",});- const query = { programId: document.programId, query: "rafi" };- const [searchResult, count, groups] = await Promise.all([+ await expect(provider.search({- ...query,+ programId: document.programId,+ query: "rafi",page: 1,pageSize: 10,sort: { field: "totalSaleAmount", order: "desc" },}),- provider.count(query),- provider.groupBy(query, "country"),- ]);+ ).rejects.toThrow("supports relevance ordering only");- expect(searchResult.hits.map(({ id }) => id)).toEqual([- "pge_higher",- "pge_lower",- ]);- expect(count).toBe(2);- expect(groups).toEqual([{ value: "CA", count: 2 }]);- expect(mocks.search).toHaveBeenCalledTimes(2);+ await expect(+ provider.search({+ programId: document.programId,+ query: "rafi",+ filters: { status: "approved" },+ page: 1,+ pageSize: 10,+ }),+ ).rejects.toThrow("filters must be applied by the database");});- it("rejects capped result sets instead of returning incomplete counts", async () => {- mocks.search.mockImplementation(async ({ query }: { query: string }) =>- query === "rafi"- ? Array.from({ length: 1_000 }, (_, index) =>- getSearchResult({- id: `pge_${index}`,- partnerId: `pn_${index}`,- }),- )- : [],- );+ it("leaves counts and grouping to the database", async () => {const provider = createUpstashSearchPartnerSearchProvider({searchIndex: createSearchIndexMock(),indexName: "test-index",});+ const query = { programId: document.programId, query: "rafi" };- await expect(- provider.count({ programId: document.programId, query: "rafi" }),- ).rejects.toThrow("Exact pagination, counts, and facets are unavailable");+ await expect(provider.count(query)).rejects.toThrow(+ "counts must be calculated by the database",+ );+ await expect(provider.groupBy(query, "country")).rejects.toThrow(+ "groups must be calculated by the database",+ );+ expect(mocks.search).not.toHaveBeenCalled();});it("waits until pending documents finish indexing", async () => {