Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Delegate partner search filtering and aggregations to the database

Rafi7ceff9af2be8
diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.ts
index 59f9cf3e6c..c73ab1b49f 100644
--- a/apps/web/lib/api/partners/search/providers/upstash-search.ts
+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts
@@ -6,24 +6,17 @@ import {
} from "../constants";
import { normalizePartnerSearchQuery } from "../searchable-values";
import type {
- PartnerSearchCountQuery,
+ PartnerSearchCandidateQuery,
PartnerSearchDocument,
- PartnerSearchGroup,
- PartnerSearchGroupField,
- PartnerSearchListFilter,
PartnerSearchProvider,
- PartnerSearchQuery,
} from "../types";
const DEFAULT_INDEX_NAME = "partner-search-v1";
-const NULL_VALUE = "\0__null:f47ac10b-58cc-4372-a567-0e02b2c3d479__";
const MAX_SEARCH_RESULTS = PARTNER_SEARCH_CANDIDATE_LIMIT;
const WRITE_BATCH_SIZE = 100;
const DELETE_BATCH_SIZE = 1_000;
const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;
const WAIT_FOR_INDEXING_POLL_MS = 100;
-const CANDIDATE_CACHE_TTL_MS = 1_000;
-const MAX_CANDIDATE_CACHE_ENTRIES = 100;
const TRANSIENT_RETRY_ATTEMPTS = 2;
const QUERY_OPERATION_TIMEOUT_MS = 1_000;
@@ -40,26 +33,6 @@ interface UpstashSearchContent extends Record<string, unknown> {
interface UpstashSearchMetadata extends Record<string, unknown> {
programId: string;
- partnerId: string;
- status: string;
- tenantId: string;
- groupId: string;
- country: string;
- partnerTagIds: string[];
- referredByPartnerId: string;
- totalClicks: number;
- totalLeads: number;
- totalConversions: number;
- totalSaleAmount: number;
- totalCommissions: number;
- netRevenue: number;
- earningsPerClick: number;
- averageLifetimeValue?: number;
- clickToLeadRate?: number;
- clickToConversionRate?: number;
- leadToConversionRate?: number;
- returnOnAdSpend?: number;
- createdAt: number;
}
interface UpstashSearchResult {
@@ -100,11 +73,6 @@ interface CreateUpstashSearchPartnerSearchProviderOptions {
indexName?: string;
}
-interface CandidateCacheEntry {
- expiresAt: number;
- promise: Promise<UpstashSearchResult[]>;
-}
-
function getIndexName(indexName?: string): string {
return (
indexName ??
@@ -195,39 +163,10 @@ function serializeUpstashSearchDocument(document: PartnerSearchDocument) {
emailNgrams: truncate(getEmailNgrams(document.email), 800),
};
- const metadata: UpstashSearchMetadata = {
- programId: document.programId,
- partnerId: document.partnerId,
- status: document.status,
- tenantId: document.tenantId ?? NULL_VALUE,
- groupId: document.groupId ?? NULL_VALUE,
- country: document.country ?? NULL_VALUE,
- partnerTagIds: document.partnerTagIds,
- referredByPartnerId: document.referredByPartnerId ?? NULL_VALUE,
- totalClicks: document.totalClicks,
- totalLeads: document.totalLeads,
- totalConversions: document.totalConversions,
- totalSaleAmount: document.totalSaleAmount,
- totalCommissions: document.totalCommissions,
- netRevenue: document.netRevenue,
- earningsPerClick: document.earningsPerClick,
- ...(document.averageLifetimeValue !== null && {
- averageLifetimeValue: document.averageLifetimeValue,
- }),
- ...(document.clickToLeadRate !== null && {
- clickToLeadRate: document.clickToLeadRate,
- }),
- ...(document.clickToConversionRate !== null && {
- clickToConversionRate: document.clickToConversionRate,
- }),
- ...(document.leadToConversionRate !== null && {
- leadToConversionRate: document.leadToConversionRate,
- }),
- ...(document.returnOnAdSpend !== null && {
- returnOnAdSpend: document.returnOnAdSpend,
- }),
- createdAt: new Date(document.createdAt).getTime(),
- };
+ // Program scope is the only metadata Search needs. The database applies
+ // status, group, tag, country, metric, and sorting constraints after it
+ // receives the relevance-ranked enrollment IDs.
+ const metadata: UpstashSearchMetadata = { programId: document.programId };
return { id: document.id, content, metadata };
}
@@ -236,95 +175,8 @@ function quoteFilterValue(value: string): string {
return JSON.stringify(value);
}
-function buildScalarListFilter(
- field: string,
- filter: PartnerSearchListFilter,
-): string {
- const values = filter.values.map(quoteFilterValue).join(", ");
- return `@metadata.${field} ${filter.operator === "IN" ? "IN" : "NOT IN"} (${values})`;
-}
-
-function buildArrayListFilter(
- field: string,
- filter: PartnerSearchListFilter,
-): string {
- const operator = filter.operator === "IN" ? "CONTAINS" : "NOT CONTAINS";
- const booleanOperator = filter.operator === "IN" ? " OR " : " AND ";
- return `(${filter.values
- .map((value) => `@metadata.${field} ${operator} ${quoteFilterValue(value)}`)
- .join(booleanOperator)})`;
-}
-
-function buildUpstashSearchFilter({
- programId,
- filters,
-}: PartnerSearchCountQuery): string {
- const conditions = [`@metadata.programId = ${quoteFilterValue(programId)}`];
-
- if (filters?.status) {
- conditions.push(`@metadata.status = ${quoteFilterValue(filters.status)}`);
- }
- if (filters?.tenantId) {
- conditions.push(
- `@metadata.tenantId = ${quoteFilterValue(filters.tenantId)}`,
- );
- }
- if (filters?.partnerIds?.length) {
- conditions.push(
- buildScalarListFilter("partnerId", {
- values: filters.partnerIds,
- operator: "IN",
- }),
- );
- }
-
- const listFilters: [
- "groupId" | "country",
- PartnerSearchListFilter | undefined,
- ][] = [
- ["groupId", filters?.groupIds],
- ["country", filters?.countries],
- ];
-
- for (const [field, listFilter] of listFilters) {
- if (listFilter?.values.length) {
- conditions.push(buildScalarListFilter(field, listFilter));
- }
- }
-
- if (filters?.partnerTagIds?.values.length) {
- conditions.push(
- buildArrayListFilter("partnerTagIds", filters.partnerTagIds),
- );
- }
- if (filters?.referredByPartnerId) {
- conditions.push(
- `@metadata.referredByPartnerId = ${quoteFilterValue(filters.referredByPartnerId)}`,
- );
- }
-
- for (const [field, range] of Object.entries(filters?.metrics ?? {})) {
- if (range.min !== undefined) {
- conditions.push(`@metadata.${field} >= ${range.min}`);
- }
- if (range.max !== undefined) {
- conditions.push(`@metadata.${field} <= ${range.max}`);
- }
- }
-
- return conditions.join(" AND ");
-}
-
-function assertCompleteResults(results: UpstashSearchResult[], query: string) {
- // Upstash Search has no count, aggregation, offset, or search cursor API.
- // We can preserve the provider contract only while the complete match set
- // fits below its top-1,000 response limit. Reject capped results so the UI
- // never presents an incomplete count, facet, sorted page, or later page.
- if (results.length >= MAX_SEARCH_RESULTS) {
- throw new Error(
- `Upstash Search returned its ${MAX_SEARCH_RESULTS.toLocaleString()}-result limit for query ${JSON.stringify(query)}. Exact pagination, counts, and facets are unavailable for this query.`,
- );
- }
+function buildUpstashSearchFilter(programId: string): string {
+ return `@metadata.programId = ${quoteFilterValue(programId)}`;
}
function isTransientError(error: unknown): boolean {
@@ -411,54 +263,6 @@ function mergeSearchResults(
);
}
-function compareResults(
- left: UpstashSearchResult,
- right: UpstashSearchResult,
- sort: NonNullable<PartnerSearchQuery["sort"]>,
-): number {
- const leftValue = left.metadata?.[sort.field] as number | undefined;
- const rightValue = right.metadata?.[sort.field] as number | undefined;
-
- if (leftValue === rightValue) {
- return left.id.localeCompare(right.id);
- }
- if (leftValue === undefined) {
- return 1;
- }
- if (rightValue === undefined) {
- return -1;
- }
-
- const comparison = leftValue < rightValue ? -1 : 1;
- return sort.order === "asc" ? comparison : -comparison;
-}
-
-function getGroupValues(
- result: UpstashSearchResult,
- field: PartnerSearchGroupField,
-): string[] {
- const metadata = result.metadata;
- if (!metadata) {
- return [];
- }
- if (field === "partnerTagId") {
- return metadata.partnerTagIds;
- }
-
- const value = metadata[field];
- if (typeof value !== "string") {
- return [];
- }
- if (field === "referredByPartnerId" && value === NULL_VALUE) {
- return [];
- }
- return [value];
-}
-
-function mapGroupValue(value: string): string | null {
- return value === NULL_VALUE ? null : value;
-}
-
function logPartnerSearchDebug(
operation: "search" | "count" | "groupBy",
details: Record<string, unknown>,
@@ -476,14 +280,14 @@ export function createUpstashSearchPartnerSearchProvider({
}: CreateUpstashSearchPartnerSearchProviderOptions = {}): PartnerSearchProvider {
const resolvedIndexName = getIndexName(indexName);
const index = searchIndex ?? createSearchIndex(resolvedIndexName);
- const candidateCache = new Map<string, CandidateCacheEntry>();
-
- async function findSearchCandidates(
- query: PartnerSearchCountQuery,
- { limit, requireComplete }: { limit: number; requireComplete: boolean },
- ): Promise<UpstashSearchResult[]> {
- const normalizedQuery = normalizePartnerSearchQuery(query.query);
- const filter = buildUpstashSearchFilter(query);
+
+ async function findSearchCandidates({
+ programId,
+ query,
+ limit,
+ }: PartnerSearchCandidateQuery): Promise<UpstashSearchResult[]> {
+ const normalizedQuery = normalizePartnerSearchQuery(query);
+ const filter = buildUpstashSearchFilter(programId);
const searchParams = {
limit,
filter,
@@ -504,61 +308,24 @@ export function createUpstashSearchPartnerSearchProvider({
]),
);
- if (requireComplete) {
- assertCompleteResults(standardResults, normalizedQuery);
- assertCompleteResults(rawEmailResults, normalizedQuery);
- }
-
const emailResults = rawEmailResults.filter(({ content }) =>
content.email.includes(normalizedQuery),
);
return mergeSearchResults(standardResults, emailResults);
}
- function getCandidates(
- query: PartnerSearchCountQuery,
- ): Promise<UpstashSearchResult[]> {
- const now = Date.now();
- const cacheKey = JSON.stringify({
- programId: query.programId,
- query: query.query,
- filters: query.filters,
- });
- const cached = candidateCache.get(cacheKey);
- if (cached && cached.expiresAt > now) {
- return cached.promise;
- }
-
- if (candidateCache.size >= MAX_CANDIDATE_CACHE_ENTRIES) {
- candidateCache.delete(candidateCache.keys().next().value!);
- }
-
- const promise = findSearchCandidates(query, {
- limit: MAX_SEARCH_RESULTS,
- requireComplete: true,
- }).catch((error) => {
- candidateCache.delete(cacheKey);
- throw error;
- });
- candidateCache.set(cacheKey, {
- expiresAt: now + CANDIDATE_CACHE_TTL_MS,
- promise,
- });
- return promise;
- }
-
return {
async searchCandidates({ programId, query, limit }) {
validatePartnerSearchCandidateLimit(limit);
- const candidates = await findSearchCandidates(
- { programId, query },
- { limit, requireComplete: false },
- );
- const hits = candidates
- .slice(0, limit)
- .flatMap(({ id, metadata, score }) =>
- metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],
- );
+ const candidates = await findSearchCandidates({
+ programId,
+ query,
+ limit,
+ });
+ const hits = candidates.slice(0, limit).map(({ id, score }) => ({
+ id,
+ score,
+ }));
logPartnerSearchDebug("search", {
indexName: resolvedIndexName,
@@ -572,48 +339,33 @@ export function createUpstashSearchPartnerSearchProvider({
},
async search(query) {
- const { sort } = query;
- if (sort) {
- const candidates = [...(await getCandidates(query))];
- candidates.sort((left, right) => compareResults(left, right, sort));
-
- const offset = (query.page - 1) * query.pageSize;
- const hits = candidates
- .slice(offset, offset + query.pageSize)
- .flatMap(({ id, metadata, score }) =>
- metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],
- );
-
- logPartnerSearchDebug("search", {
- indexName: resolvedIndexName,
- query,
- candidateCount: candidates.length,
- resultCount: hits.length,
- hits,
- });
-
- return { hits };
+ if (query.sort) {
+ throw new Error(
+ "Upstash Search partner search supports relevance ordering only.",
+ );
+ }
+ if (query.filters && Object.keys(query.filters).length > 0) {
+ throw new Error(
+ "Upstash Search partner search filters must be applied by the database.",
+ );
}
- const requestedResults = query.page * query.pageSize;
+ const offset = (query.page - 1) * query.pageSize;
+ const requestedResults = offset + query.pageSize;
if (requestedResults > MAX_SEARCH_RESULTS) {
throw new Error(
`Upstash Search relevance pagination is limited to the first ${MAX_SEARCH_RESULTS.toLocaleString()} results.`,
);
}
- // Upstash Search natively returns relevance-ranked results. Fetch enough
- // results to derive the requested page without requiring a complete set.
- const candidates = await findSearchCandidates(query, {
+ const candidates = await findSearchCandidates({
+ programId: query.programId,
+ query: query.query,
limit: requestedResults,
- requireComplete: false,
});
- const offset = (query.page - 1) * query.pageSize;
const hits = candidates
.slice(offset, offset + query.pageSize)
- .flatMap(({ id, metadata, score }) =>
- metadata ? [{ id, partnerId: metadata.partnerId, score }] : [],
- );
+ .map(({ id, score }) => ({ id, score }));
logPartnerSearchDebug("search", {
indexName: resolvedIndexName,
@@ -626,36 +378,16 @@ export function createUpstashSearchPartnerSearchProvider({
return { hits };
},
- async count(query) {
- const count = (await getCandidates(query)).length;
- logPartnerSearchDebug("count", {
- indexName: resolvedIndexName,
- query,
- count,
- });
- return count;
+ async count() {
+ throw new Error(
+ "Upstash Search partner search counts must be calculated by the database.",
+ );
},
- async groupBy(query, field): Promise<PartnerSearchGroup[]> {
- const groups = new Map<string, number>();
- for (const candidate of await getCandidates(query)) {
- for (const value of getGroupValues(candidate, field)) {
- groups.set(value, (groups.get(value) ?? 0) + 1);
- }
- }
-
- const result = Array.from(groups, ([value, count]) => ({
- value: mapGroupValue(value),
- count,
- }));
- logPartnerSearchDebug("groupBy", {
- indexName: resolvedIndexName,
- query,
- field,
- groupCount: result.length,
- groups: result,
- });
- return result;
+ async groupBy() {
+ throw new Error(
+ "Upstash Search partner search groups must be calculated by the database.",
+ );
},
async waitForIndexing() {
@@ -683,14 +415,12 @@ export function createUpstashSearchPartnerSearchProvider({
index.upsert(documentBatch.map(serializeUpstashSearchDocument)),
);
}
- candidateCache.clear();
},
async delete(documentIds) {
for (const documentIdBatch of chunk(documentIds, DELETE_BATCH_SIZE)) {
await withTransientRetry(() => index.delete(documentIdBatch));
}
- candidateCache.clear();
},
};
}
diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
index b337fa1b1c..7dd39c58b5 100644
--- a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
@@ -59,17 +59,15 @@ function createSearchIndexMock() {
function getSearchResult(
overrides: {
id?: string;
- partnerId?: string;
name?: string;
email?: string;
score?: number;
- totalSaleAmount?: number;
} = {},
) {
return {
id: overrides.id ?? document.id,
content: {
- partnerId: overrides.partnerId ?? document.partnerId,
+ partnerId: document.partnerId,
name: overrides.name ?? document.name,
email: overrides.email ?? document.email!,
companyName: document.companyName!,
@@ -80,26 +78,6 @@ function getSearchResult(
},
metadata: {
programId: document.programId,
- partnerId: overrides.partnerId ?? document.partnerId,
- status: document.status,
- tenantId: "null",
- groupId: "null",
- country: document.country!,
- partnerTagIds: document.partnerTagIds,
- referredByPartnerId: "null",
- totalClicks: document.totalClicks,
- totalLeads: document.totalLeads,
- totalConversions: document.totalConversions,
- totalSaleAmount: overrides.totalSaleAmount ?? document.totalSaleAmount,
- totalCommissions: document.totalCommissions,
- netRevenue: document.netRevenue,
- earningsPerClick: document.earningsPerClick,
- averageLifetimeValue: document.averageLifetimeValue!,
- clickToLeadRate: document.clickToLeadRate!,
- clickToConversionRate: document.clickToConversionRate!,
- leadToConversionRate: document.leadToConversionRate!,
- returnOnAdSpend: document.returnOnAdSpend!,
- createdAt: new Date(document.createdAt).getTime(),
},
score: overrides.score ?? 0.9,
};
@@ -136,6 +114,7 @@ describe("Upstash Search partner search provider", () => {
expect(indexedDocument.content).toEqual(
expect.objectContaining({
name: document.name,
+ partnerId: document.partnerId,
email: document.email,
companyName: document.companyName,
platforms: expect.stringContaining("rafi-on-x"),
@@ -143,13 +122,9 @@ describe("Upstash Search partner search provider", () => {
emailNgrams: expect.stringContaining("exa"),
}),
);
- expect(indexedDocument.metadata).toEqual(
- expect.objectContaining({
- programId: document.programId,
- partnerId: document.partnerId,
- totalSaleAmount: document.totalSaleAmount,
- }),
- );
+ expect(indexedDocument.metadata).toEqual({
+ programId: document.programId,
+ });
});
it("adds an exact email n-gram search for partial email matches", async () => {
@@ -173,7 +148,6 @@ describe("Upstash Search partner search provider", () => {
hits: [
{
id: document.id,
- partnerId: document.partnerId,
score: emailResult.score,
},
],
@@ -201,7 +175,6 @@ describe("Upstash Search partner search provider", () => {
Array.from({ length: 100 }, (_, index) =>
getSearchResult({
id: `pge_${index}`,
- partnerId: `pn_${index}`,
score: 1 - index / 100,
}),
),
@@ -220,7 +193,7 @@ describe("Upstash Search partner search provider", () => {
expect(result.hits).toHaveLength(10);
expect(result.hits[0]).toEqual(
- expect.objectContaining({ id: "pge_0", partnerId: "pn_0" }),
+ expect.objectContaining({ id: "pge_0", score: 1 }),
);
expect(mocks.search).toHaveBeenCalledWith(
expect.objectContaining({ query: "drew moore", limit: 10 }),
@@ -237,67 +210,47 @@ describe("Upstash Search partner search provider", () => {
);
});
- it("shares candidates across search, count, and grouping", async () => {
- const lower = getSearchResult({
- id: "pge_lower",
- partnerId: "pn_lower",
- score: 0.8,
- totalSaleAmount: 100,
- });
- const higher = getSearchResult({
- id: "pge_higher",
- partnerId: "pn_higher",
- score: 0.7,
- totalSaleAmount: 1_000,
- });
- mocks.search.mockImplementation(async ({ query }: { query: string }) =>
- query === "rafi" ? [lower, higher] : [],
- );
+ it("leaves filtering and sorting to the database", async () => {
const provider = createUpstashSearchPartnerSearchProvider({
searchIndex: createSearchIndexMock(),
indexName: "test-index",
});
- const query = { programId: document.programId, query: "rafi" };
- const [searchResult, count, groups] = await Promise.all([
+ await expect(
provider.search({
- ...query,
+ programId: document.programId,
+ query: "rafi",
page: 1,
pageSize: 10,
sort: { field: "totalSaleAmount", order: "desc" },
}),
- provider.count(query),
- provider.groupBy(query, "country"),
- ]);
+ ).rejects.toThrow("supports relevance ordering only");
- expect(searchResult.hits.map(({ id }) => id)).toEqual([
- "pge_higher",
- "pge_lower",
- ]);
- expect(count).toBe(2);
- expect(groups).toEqual([{ value: "CA", count: 2 }]);
- expect(mocks.search).toHaveBeenCalledTimes(2);
+ await expect(
+ provider.search({
+ programId: document.programId,
+ query: "rafi",
+ filters: { status: "approved" },
+ page: 1,
+ pageSize: 10,
+ }),
+ ).rejects.toThrow("filters must be applied by the database");
});
- it("rejects capped result sets instead of returning incomplete counts", async () => {
- mocks.search.mockImplementation(async ({ query }: { query: string }) =>
- query === "rafi"
- ? Array.from({ length: 1_000 }, (_, index) =>
- getSearchResult({
- id: `pge_${index}`,
- partnerId: `pn_${index}`,
- }),
- )
- : [],
- );
+ it("leaves counts and grouping to the database", async () => {
const provider = createUpstashSearchPartnerSearchProvider({
searchIndex: createSearchIndexMock(),
indexName: "test-index",
});
+ const query = { programId: document.programId, query: "rafi" };
- await expect(
- provider.count({ programId: document.programId, query: "rafi" }),
- ).rejects.toThrow("Exact pagination, counts, and facets are unavailable");
+ await expect(provider.count(query)).rejects.toThrow(
+ "counts must be calculated by the database",
+ );
+ await expect(provider.groupBy(query, "country")).rejects.toThrow(
+ "groups must be calculated by the database",
+ );
+ expect(mocks.search).not.toHaveBeenCalled();
});
it("waits until pending documents finish indexing", async () => {