Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Simplify Upstash Redis partner search to relevance-only

Rafif92307bafd43
diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
index fd8c9fbebc..c0f6da9274 100644
--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts
+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
@@ -5,29 +5,21 @@ import {
type InferFilterFromSchema,
type SearchIndex,
} from "@upstash/redis";
-import { validatePartnerSearchCandidateLimit } from "../constants";
+import {
+ PARTNER_SEARCH_CANDIDATE_LIMIT,
+ validatePartnerSearchCandidateLimit,
+} from "../constants";
import {
getPartnerSearchableValues,
normalizePartnerSearchQuery,
} from "../searchable-values";
import type {
PartnerSearchCandidateQuery,
- PartnerSearchCountQuery,
PartnerSearchDocument,
- PartnerSearchGroup,
- PartnerSearchGroupField,
- PartnerSearchListFilter,
- PartnerSearchMetricField,
PartnerSearchProvider,
- PartnerSearchQuery,
} from "../types";
const DEFAULT_INDEX_NAME = "partner-search-v1";
-// Sentinel value for nullable keyword fields. Redis Search keyword fields
-// cannot store null, so we use a value that will never appear in real data
-// to represent absent values and map it back to null in query results.
-const NULL_VALUE = "\0__null:f47ac10b-58cc-4372-a567-0e02b2c3d479__";
-const MAX_GROUPS = 1_000;
const TRANSIENT_RETRY_ATTEMPTS = 2;
// Keep the full query operation within the one-second latency target while
// leaving a small window to retry failures that return quickly
@@ -36,41 +28,11 @@ const QUERY_OPERATION_TIMEOUT_MS = 1_000;
const WRITE_REQUEST_TIMEOUT_MS = 10_000;
const WRITE_BATCH_SIZE = 100;
const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;
-const DOCUMENT_TYPE_PARTNER = "partner";
-const DOCUMENT_TYPE_TAG = "tag";
-
-// Redis Search keyword fields accept one string value rather than an array of tag IDs
-// The partner document stores tag IDs as searchable text for filtering,
-// but Redis Search cannot use that text to group and count individual tags
-// To support tag grouping, we add one shadow document per tag with a scalar
-// partnerTagId that the provider can aggregate with $terms
+
export const upstashPartnerSearchSchema = s.object({
- id: s.keyword(),
programId: s.keyword(),
- partnerId: s.keyword(),
- documentType: s.keyword(),
searchText: s.string().noStem(),
emailNgrams: s.string().noStem(),
- status: s.keyword(),
- tenantId: s.keyword(),
- groupId: s.keyword(),
- country: s.keyword(),
- partnerTagIds: s.string().noStem(),
- partnerTagId: s.keyword(),
- referredByPartnerId: s.keyword(),
- totalClicks: s.number("F64"),
- totalLeads: s.number("F64"),
- totalConversions: s.number("F64"),
- totalSaleAmount: s.number("F64"),
- totalCommissions: s.number("F64"),
- netRevenue: s.number("F64"),
- earningsPerClick: s.number("F64"),
- averageLifetimeValue: s.number("F64"),
- clickToLeadRate: s.number("F64"),
- clickToConversionRate: s.number("F64"),
- leadToConversionRate: s.number("F64"),
- returnOnAdSpend: s.number("F64"),
- createdAt: s.date().fast(),
});
type UpstashPartnerSearchSchema = typeof upstashPartnerSearchSchema;
@@ -79,33 +41,9 @@ type UpstashPartnerSearchFilter =
type UpstashPartnerSearchIndex = SearchIndex<UpstashPartnerSearchSchema>;
interface UpstashPartnerSearchDocument extends Record<string, unknown> {
- id: string;
programId: string;
- partnerId: string;
- documentType: typeof DOCUMENT_TYPE_PARTNER | typeof DOCUMENT_TYPE_TAG;
searchText: string;
emailNgrams: string;
- status: string;
- tenantId: string;
- groupId: string;
- country: string;
- partnerTagIds: string;
- partnerTagIdsRaw: string[];
- partnerTagId: string;
- referredByPartnerId: string;
- totalClicks: number;
- totalLeads: number;
- totalConversions: number;
- totalSaleAmount: number;
- totalCommissions: number;
- netRevenue: number;
- earningsPerClick: number;
- averageLifetimeValue?: number;
- clickToLeadRate?: number;
- clickToConversionRate?: number;
- leadToConversionRate?: number;
- returnOnAdSpend?: number;
- createdAt: string;
}
interface CreateUpstashRedisPartnerSearchProviderOptions {
@@ -134,19 +72,11 @@ function getIndexName(indexName?: string): string {
}
function getDocumentPrefix(indexName: string): string {
- return `${indexName}:`;
+ return `${indexName}:partner:`;
}
function getDocumentKey(indexName: string, documentId: string): string {
- return `${getDocumentPrefix(indexName)}partner:${documentId}`;
-}
-
-function getTagDocumentKey(
- indexName: string,
- documentId: string,
- partnerTagId: string,
-): string {
- return `${getDocumentPrefix(indexName)}tag:${documentId}:${partnerTagId}`;
+ return `${getDocumentPrefix(indexName)}${documentId}`;
}
function createRedisClient(requestTimeoutMs: number): Redis {
@@ -204,68 +134,16 @@ function getQueryNgrams(query: string): string[] {
function serializeUpstashDocument(
document: PartnerSearchDocument,
- documentType: typeof DOCUMENT_TYPE_PARTNER | typeof DOCUMENT_TYPE_TAG,
- partnerTagId = NULL_VALUE,
): UpstashPartnerSearchDocument {
return {
- id: document.id,
programId: document.programId,
- partnerId: document.partnerId,
- documentType,
searchText: getPartnerSearchableValues(document)
.map(normalizePartnerSearchQuery)
.join(" "),
emailNgrams: getEmailNgrams(document.email),
- status: document.status,
- tenantId: document.tenantId ?? NULL_VALUE,
- groupId: document.groupId ?? NULL_VALUE,
- country: document.country ?? NULL_VALUE,
- partnerTagIds: document.partnerTagIds.join(" "),
- partnerTagIdsRaw: document.partnerTagIds,
- partnerTagId,
- referredByPartnerId: document.referredByPartnerId ?? NULL_VALUE,
- totalClicks: document.totalClicks,
- totalLeads: document.totalLeads,
- totalConversions: document.totalConversions,
- totalSaleAmount: document.totalSaleAmount,
- totalCommissions: document.totalCommissions,
- netRevenue: document.netRevenue,
- earningsPerClick: document.earningsPerClick,
- ...(document.averageLifetimeValue !== null && {
- averageLifetimeValue: document.averageLifetimeValue,
- }),
- ...(document.clickToLeadRate !== null && {
- clickToLeadRate: document.clickToLeadRate,
- }),
- ...(document.clickToConversionRate !== null && {
- clickToConversionRate: document.clickToConversionRate,
- }),
- ...(document.leadToConversionRate !== null && {
- leadToConversionRate: document.leadToConversionRate,
- }),
- ...(document.returnOnAdSpend !== null && {
- returnOnAdSpend: document.returnOnAdSpend,
- }),
- createdAt: document.createdAt,
};
}
-function buildListFilter(
- field: "groupId" | "country" | "partnerTagIds",
- filter: PartnerSearchListFilter,
-): {
- include?: UpstashPartnerSearchFilter;
- exclude?: UpstashPartnerSearchFilter;
-} {
- const condition = {
- [field]: { $in: filter.values },
- } as UpstashPartnerSearchFilter;
-
- return filter.operator === "IN"
- ? { include: condition }
- : { exclude: condition };
-}
-
function buildTextAlternatives(query: string): UpstashPartnerSearchFilter[][] {
// $smart handles word-boundary and prefix matching via the inverted index.
// Email n-grams handle infix/substring matching (e.g. "examp" in "[email protected]").
@@ -288,75 +166,21 @@ function buildTextAlternatives(query: string): UpstashPartnerSearchFilter[][] {
return alternatives;
}
-function buildUpstashFilter(
- { programId, query, filters }: PartnerSearchCountQuery,
- documentType:
- | typeof DOCUMENT_TYPE_PARTNER
- | typeof DOCUMENT_TYPE_TAG = DOCUMENT_TYPE_PARTNER,
-): UpstashPartnerSearchFilter {
- const must: UpstashPartnerSearchFilter[] = [
- { documentType: { $eq: documentType } },
- { programId: { $eq: programId } },
- ];
- const mustNot: UpstashPartnerSearchFilter[] = [];
-
- if (filters?.status) {
- must.push({ status: { $eq: filters.status } });
- }
- if (filters?.tenantId) {
- must.push({ tenantId: { $eq: filters.tenantId } });
- }
- if (filters?.partnerIds?.length) {
- must.push({ partnerId: { $in: filters.partnerIds } });
- }
-
- const listFilters: [
- "groupId" | "country" | "partnerTagIds",
- PartnerSearchListFilter | undefined,
- ][] = [
- ["groupId", filters?.groupIds],
- ["country", filters?.countries],
- ["partnerTagIds", filters?.partnerTagIds],
- ];
-
- for (const [field, listFilter] of listFilters) {
- if (!listFilter) {
- continue;
- }
-
- const { include, exclude } = buildListFilter(field, listFilter);
- if (include) {
- must.push(include);
- }
- if (exclude) {
- mustNot.push(exclude);
- }
- }
-
- if (filters?.referredByPartnerId) {
- must.push({
- referredByPartnerId: { $eq: filters.referredByPartnerId },
- });
- }
-
- for (const [field, range] of Object.entries(filters?.metrics ?? {})) {
- const condition = {
- ...(range.min !== undefined && { $gte: range.min }),
- ...(range.max !== undefined && { $lte: range.max }),
- };
- must.push({
- [field as PartnerSearchMetricField]: condition,
- } as UpstashPartnerSearchFilter);
- }
-
- // At the root, $should requires at least one text alternative to match.
- // Combining $should with $must would make text matching an optional score
- // boost, allowing unrelated documents through when results use orderBy.
+function buildUpstashFilter({
+ programId,
+ query,
+}: Pick<
+ PartnerSearchCandidateQuery,
+ "programId" | "query"
+>): UpstashPartnerSearchFilter {
+ // Each branch includes the program scope because a root-level $must would
+ // turn the sibling $should clauses into optional score boosts. Keeping both
+ // requirements in every branch prevents unrelated program documents from
+ // being returned with low scores.
return {
$should: buildTextAlternatives(normalizePartnerSearchQuery(query)).map(
(textConditions) => ({
- $must: [...must, ...textConditions],
- ...(mustNot.length > 0 && { $mustNot: mustNot }),
+ $must: [{ programId: { $eq: programId } }, ...textConditions],
}),
),
} as UpstashPartnerSearchFilter;
@@ -429,16 +253,6 @@ async function withQueryDeadline<T>(operation: () => Promise<T>): Promise<T> {
}
}
-function getGroupIndexField(
- field: PartnerSearchGroupField,
-): "status" | "country" | "groupId" | "partnerTagId" | "referredByPartnerId" {
- return field;
-}
-
-function mapGroupValue(value: string): string | null {
- return value === NULL_VALUE ? null : value;
-}
-
function getSchemaSignature(schema: Record<string, Record<string, unknown>>) {
return JSON.stringify(
Object.fromEntries(
@@ -480,71 +294,20 @@ function validateIndexDescription(
!hasExpectedSchema
) {
throw new Error(
- `Partner search index ${indexName} does not match the configured schema. Create a new versioned index.`,
+ `Partner search index ${indexName} does not match the configured schema. Delete and recreate it before backfilling.`,
);
}
}
-function getStoredDocument(
- value: UpstashPartnerSearchDocument[] | null,
-): UpstashPartnerSearchDocument | null {
- return value?.[0] ?? null;
-}
-
-async function getStoredDocuments(
- redisClient: Redis,
- indexName: string,
- documentIds: string[],
-): Promise<(UpstashPartnerSearchDocument | null)[]> {
- if (documentIds.length === 0) {
- return [];
- }
-
- const values = await withTransientRetry(() =>
- redisClient.json.mget<(UpstashPartnerSearchDocument[] | null)[]>(
- documentIds.map((documentId) => getDocumentKey(indexName, documentId)),
- "$",
- ),
- );
-
- return values.map(getStoredDocument);
-}
-
-function getStaleTagDocumentKeys(
- indexName: string,
- documents: PartnerSearchDocument[],
- storedDocuments: (UpstashPartnerSearchDocument | null)[],
-): string[] {
- return documents.flatMap((document, index) => {
- const currentTagIds = new Set(document.partnerTagIds);
- return (storedDocuments[index]?.partnerTagIdsRaw ?? [])
- .filter((partnerTagId) => !currentTagIds.has(partnerTagId))
- .map((partnerTagId) =>
- getTagDocumentKey(indexName, document.id, partnerTagId),
- );
- });
-}
-
function getUpsertEntries(
indexName: string,
documents: PartnerSearchDocument[],
) {
- return documents.flatMap((document) => [
- {
- key: getDocumentKey(indexName, document.id),
- path: "$",
- value: serializeUpstashDocument(document, DOCUMENT_TYPE_PARTNER),
- },
- ...document.partnerTagIds.map((partnerTagId) => ({
- key: getTagDocumentKey(indexName, document.id, partnerTagId),
- path: "$",
- value: serializeUpstashDocument(
- document,
- DOCUMENT_TYPE_TAG,
- partnerTagId,
- ),
- })),
- ]);
+ return documents.map((document) => ({
+ key: getDocumentKey(indexName, document.id),
+ path: "$",
+ value: serializeUpstashDocument(document),
+ }));
}
async function upsertDocumentBatch(
@@ -552,30 +315,9 @@ async function upsertDocumentBatch(
indexName: string,
documents: PartnerSearchDocument[],
) {
- const storedDocuments = await getStoredDocuments(
- redisClient,
- indexName,
- documents.map(({ id }) => id),
- );
- const staleTagDocumentKeys = getStaleTagDocumentKeys(
- indexName,
- documents,
- storedDocuments,
- );
const upsertEntries = getUpsertEntries(indexName, documents);
- // Keep stale tag cleanup and document upserts in one transaction so another
- // synchronization cannot modify the same keys between these operations
- await withTransientRetry(async () => {
- const transaction = redisClient.multi();
-
- if (staleTagDocumentKeys.length > 0) {
- transaction.del(...staleTagDocumentKeys);
- }
-
- transaction.json.mset(...upsertEntries);
- await transaction.exec();
- });
+ await withTransientRetry(() => redisClient.json.mset(...upsertEntries));
}
async function deleteDocumentBatch(
@@ -583,23 +325,9 @@ async function deleteDocumentBatch(
indexName: string,
documentIds: string[],
) {
- // Read the batch once and derive exact tag keys rather than scanning the
- // entire Redis keyspace for each partner
- const storedDocuments = await getStoredDocuments(
- redisClient,
- indexName,
- documentIds,
- );
- const tagDocumentKeys = storedDocuments.flatMap((document, index) =>
- (document?.partnerTagIdsRaw ?? []).map((partnerTagId) =>
- getTagDocumentKey(indexName, documentIds[index]!, partnerTagId),
- ),
- );
-
await withTransientRetry(() =>
redisClient.del(
...documentIds.map((documentId) => getDocumentKey(indexName, documentId)),
- ...tagDocumentKeys,
),
);
}
@@ -655,139 +383,76 @@ export function createUpstashRedisPartnerSearchProvider({
schema: upstashPartnerSearchSchema,
});
- return {
- async searchCandidates({
- programId,
- query,
- limit,
- }: PartnerSearchCandidateQuery) {
- validatePartnerSearchCandidateLimit(limit);
- const filter = buildUpstashFilter({ programId, query });
- const results = await withQueryDeadline(() =>
- queryIndex.query({
- filter,
- limit,
- select: { id: true, partnerId: true },
- }),
- );
- const hits = results.map(({ data, score }) => ({
- id: data.id,
- partnerId: data.partnerId,
- score,
- }));
-
- logPartnerSearchDebug("search", {
- indexName: resolvedIndexName,
- operation: "searchCandidates",
- query: { programId, query, limit },
- filter,
- resultCount: hits.length,
- hits,
- });
-
- return { hits };
- },
-
- async search(query: PartnerSearchQuery) {
- const filter = buildUpstashFilter(query);
- const offset = (query.page - 1) * query.pageSize;
- const orderBy = query.sort
- ? ({
- [query.sort.field]: query.sort.order.toUpperCase(),
- } as Record<string, "ASC" | "DESC">)
- : undefined;
-
- const results = await withQueryDeadline(() =>
- queryIndex.query({
- filter,
- limit: query.pageSize,
- offset,
- select: { id: true, partnerId: true },
- ...(orderBy && { orderBy }),
- }),
- );
- const hits = results.map(({ key, data, score }) => ({
- key,
- id: data.id,
- partnerId: data.partnerId,
- score,
- }));
-
- logPartnerSearchDebug("search", {
- indexName: resolvedIndexName,
- query,
- filter,
- resultCount: hits.length,
- hits,
- });
-
- return {
- hits: hits.map(({ id, partnerId, score }) => ({
- id,
- partnerId,
- score,
- })),
- };
- },
-
- async count(query) {
- const result = await withQueryDeadline(() =>
- queryIndex.count({ filter: buildUpstashFilter(query) }),
- );
+ async function findCandidates({
+ programId,
+ query,
+ limit,
+ }: PartnerSearchCandidateQuery) {
+ validatePartnerSearchCandidateLimit(limit);
+ const filter = buildUpstashFilter({ programId, query });
+ const results = await withQueryDeadline(() =>
+ queryIndex.query({ filter, limit, select: {} }),
+ );
+ const hits = results.map(({ key, score }) => ({
+ id: key.slice(getDocumentPrefix(resolvedIndexName).length),
+ score,
+ }));
+
+ logPartnerSearchDebug("search", {
+ indexName: resolvedIndexName,
+ operation: "searchCandidates",
+ query: { programId, query, limit },
+ filter,
+ resultCount: hits.length,
+ hits,
+ });
- logPartnerSearchDebug("count", {
- indexName: resolvedIndexName,
- query,
- count: result.count,
- });
+ return { hits };
+ }
- return result.count;
- },
+ return {
+ mode: "relevance-only",
- async groupBy(query, field): Promise<PartnerSearchGroup[]> {
- const indexField = getGroupIndexField(field);
- const documentType =
- field === "partnerTagId" ? DOCUMENT_TYPE_TAG : DOCUMENT_TYPE_PARTNER;
- const result = await withQueryDeadline(() =>
- queryIndex.aggregate({
- filter: buildUpstashFilter(query, documentType),
- aggregations: {
- groups: {
- $terms: {
- field: indexField,
- size: MAX_GROUPS,
- },
- },
- },
- }),
- );
+ searchCandidates: findCandidates,
- const { buckets, sumOtherDocCount } = result.groups;
+ async search(query) {
+ if (query.sort) {
+ throw new Error(
+ "Upstash Redis partner search supports relevance ordering only.",
+ );
+ }
+ if (query.filters && Object.keys(query.filters).length > 0) {
+ throw new Error(
+ "Upstash Redis partner search filters must be applied by the database.",
+ );
+ }
- if (buckets.length >= MAX_GROUPS) {
- console.warn(
- `[Partner Search] groupBy("${field}") returned ${buckets.length} buckets (limit: ${MAX_GROUPS}). ` +
- `Results may be truncated (${sumOtherDocCount ?? "unknown"} docs in unlisted groups).`,
+ const offset = (query.page - 1) * query.pageSize;
+ const requestedResults = offset + query.pageSize;
+ if (requestedResults > PARTNER_SEARCH_CANDIDATE_LIMIT) {
+ throw new Error(
+ `Upstash Redis relevance pagination is limited to the first ${PARTNER_SEARCH_CANDIDATE_LIMIT.toLocaleString()} results.`,
);
}
- const groups = buckets.flatMap(({ key, docCount }) => {
- const value = mapGroupValue(key);
- if (field === "referredByPartnerId" && value === null) {
- return [];
- }
- return [{ value, count: docCount }];
+ const result = await findCandidates({
+ programId: query.programId,
+ query: query.query,
+ limit: requestedResults,
});
+ return { hits: result.hits.slice(offset, requestedResults) };
+ },
- logPartnerSearchDebug("groupBy", {
- indexName: resolvedIndexName,
- query,
- field,
- groupCount: groups.length,
- groups,
- });
+ async count() {
+ throw new Error(
+ "Upstash Redis partner search counts must be calculated by the database.",
+ );
+ },
- return groups;
+ async groupBy() {
+ throw new Error(
+ "Upstash Redis partner search groups must be calculated by the database.",
+ );
},
async waitForIndexing() {
diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.ts
index 32f2c8d313..d57d40f7ea 100644
--- a/apps/web/lib/api/partners/search/types.ts
+++ b/apps/web/lib/api/partners/search/types.ts
@@ -120,7 +120,7 @@ export interface PartnerSearchGroup {
export interface PartnerSearchHit {
id: string;
- partnerId: string;
+ partnerId?: string;
score?: number;
}
diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
index 2a60189fd4..7ec63a5438 100644
--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
@@ -45,23 +45,17 @@ const document: PartnerSearchDocument = {
};
const mocks = vi.hoisted(() => ({
- aggregate: vi.fn(),
- count: vi.fn(),
createIndex: vi.fn(),
del: vi.fn(),
describe: vi.fn(),
index: vi.fn(),
- jsonMget: vi.fn(),
jsonMset: vi.fn(),
- multi: vi.fn(),
query: vi.fn(),
waitIndexing: vi.fn(),
}));
function createRedisMock(): Redis {
const searchIndex = {
- aggregate: mocks.aggregate,
- count: mocks.count,
describe: mocks.describe,
query: mocks.query,
waitIndexing: mocks.waitIndexing,
@@ -73,32 +67,12 @@ function createRedisMock(): Redis {
const redisMock = {
del: mocks.del,
json: {
- mget: mocks.jsonMget,
mset: mocks.jsonMset,
},
search: {
createIndex: mocks.createIndex,
index: mocks.index,
},
- multi() {
- mocks.multi();
- const commands: Array<() => void> = [];
- return {
- del: (...args: unknown[]) => {
- commands.push(() => mocks.del(...args));
- },
- json: {
- mset: (...args: unknown[]) => {
- commands.push(() => mocks.jsonMset(...args));
- },
- },
- async exec() {
- for (const cmd of commands) {
- await cmd();
- }
- },
- };
- },
} as unknown as Redis;
return redisMock;
@@ -107,18 +81,13 @@ function createRedisMock(): Redis {
describe("Upstash Redis partner search provider", () => {
beforeEach(() => {
vi.clearAllMocks();
- mocks.aggregate.mockResolvedValue({
- groups: { buckets: [], sumOtherDocCount: 0 },
- });
- mocks.count.mockResolvedValue({ count: 0 });
mocks.del.mockResolvedValue(0);
mocks.describe.mockResolvedValue({
name: "test-index",
dataType: "json",
- prefixes: ["test-index:"],
+ prefixes: ["test-index:partner:"],
schema: upstashPartnerSearchSchema,
});
- mocks.jsonMget.mockResolvedValue([]);
mocks.jsonMset.mockResolvedValue("OK");
mocks.query.mockResolvedValue([]);
mocks.waitIndexing.mockResolvedValue(1);
@@ -129,7 +98,7 @@ describe("Upstash Redis partner search provider", () => {
{
key: "test-index:partner:pge_test",
score: 4,
- data: { id: document.id, partnerId: document.partnerId },
+ data: {},
},
]);
const provider = createUpstashRedisPartnerSearchProvider({
@@ -144,13 +113,13 @@ describe("Upstash Redis partner search provider", () => {
limit: PARTNER_SEARCH_CANDIDATE_LIMIT,
}),
).resolves.toEqual({
- hits: [{ id: document.id, partnerId: document.partnerId, score: 4 }],
+ hits: [{ id: document.id, score: 4 }],
});
expect(mocks.query).toHaveBeenCalledWith(
expect.objectContaining({
limit: PARTNER_SEARCH_CANDIDATE_LIMIT,
- select: { id: true, partnerId: true },
+ select: {},
}),
);
const request = mocks.query.mock.calls[0]![0];
@@ -161,111 +130,87 @@ describe("Upstash Redis partner search provider", () => {
);
});
- it("searches within the program and supports partial email matching", async () => {
- mocks.query.mockResolvedValue([
- {
- key: "test-index:partner:pge_test",
- score: 2,
- data: { id: document.id, partnerId: document.partnerId },
- },
- ]);
+ it("requires program-scoped text or partial email matches", async () => {
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
indexName: "test-index",
});
- const result = await provider.search({
+ await provider.searchCandidates({
programId: document.programId,
query: "examp",
- page: 3,
- pageSize: 25,
- sort: { field: "totalSaleAmount", order: "desc" },
+ limit: 25,
});
- expect(result).toEqual({
- hits: [{ id: document.id, partnerId: document.partnerId, score: 2 }],
- });
- expect(mocks.count).not.toHaveBeenCalled();
- expect(mocks.query).toHaveBeenCalledWith(
- expect.objectContaining({
- limit: 25,
- offset: 50,
- orderBy: { totalSaleAmount: "DESC" },
- select: { id: true, partnerId: true },
- }),
- );
-
const filter = mocks.query.mock.calls[0]![0].filter;
expect(filter.$must).toBeUndefined();
expect(filter.$should).toHaveLength(2);
- expect(JSON.stringify(filter)).toContain('"programId":{"$eq":"prog_test"}');
- expect(JSON.stringify(filter)).toContain(
- '"documentType":{"$eq":"partner"}',
- );
+ for (const branch of filter.$should) {
+ expect(branch.$must).toEqual(
+ expect.arrayContaining([{ programId: { $eq: "prog_test" } }]),
+ );
+ }
expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"exa"}');
expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"xam"}');
expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"amp"}');
});
- it("passes list exclusions and metric ranges to Upstash", async () => {
+ it("exposes relevance-only behavior during the transition", async () => {
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
indexName: "test-index",
});
- await provider.count({
- programId: document.programId,
- query: "rafi",
- filters: {
- countries: { values: ["US", "CA"], operator: "NOT_IN" },
- partnerTagIds: { values: ["ptag_test"], operator: "IN" },
- metrics: { totalSaleAmount: { min: 100, max: 1_000 } },
- },
- });
-
- const filter = mocks.count.mock.calls[0]![0].filter;
- expect(filter.$should).toHaveLength(2);
- for (const branch of filter.$should) {
- expect(branch.$must).toEqual(
- expect.arrayContaining([
- { partnerTagIds: { $in: ["ptag_test"] } },
- { totalSaleAmount: { $gte: 100, $lte: 1_000 } },
- ]),
- );
- expect(branch.$mustNot).toEqual([{ country: { $in: ["US", "CA"] } }]);
- }
+ expect(provider.mode).toBe("relevance-only");
+ await expect(
+ provider.count({ programId: document.programId, query: "rafi" }),
+ ).rejects.toThrow("counts must be calculated by the database");
+ await expect(
+ provider.groupBy(
+ { programId: document.programId, query: "rafi" },
+ "country",
+ ),
+ ).rejects.toThrow("groups must be calculated by the database");
});
it("retries a transient provider error", async () => {
- mocks.count
+ mocks.query
.mockRejectedValueOnce(new Error("Upstash returned 503"))
- .mockResolvedValueOnce({ count: 1 });
+ .mockResolvedValueOnce([]);
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
indexName: "test-index",
});
await expect(
- provider.count({ programId: document.programId, query: "rafi" }),
- ).resolves.toBe(1);
- expect(mocks.count).toHaveBeenCalledTimes(2);
+ provider.searchCandidates({
+ programId: document.programId,
+ query: "rafi",
+ limit: 10,
+ }),
+ ).resolves.toEqual({ hits: [] });
+ expect(mocks.query).toHaveBeenCalledTimes(2);
});
it("does not retry a permanent provider error", async () => {
- mocks.count.mockRejectedValue(new Error("Invalid search filter"));
+ mocks.query.mockRejectedValue(new Error("Invalid search filter"));
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
indexName: "test-index",
});
await expect(
- provider.count({ programId: document.programId, query: "rafi" }),
+ provider.searchCandidates({
+ programId: document.programId,
+ query: "rafi",
+ limit: 10,
+ }),
).rejects.toThrow("Invalid search filter");
- expect(mocks.count).toHaveBeenCalledTimes(1);
+ expect(mocks.query).toHaveBeenCalledTimes(1);
});
it("does not retry a request timeout after it consumes the query budget", async () => {
- mocks.count.mockRejectedValue(
+ mocks.query.mockRejectedValue(
new DOMException(
"The operation was aborted due to timeout",
"TimeoutError",
@@ -277,16 +222,19 @@ describe("Upstash Redis partner search provider", () => {
});
await expect(
- provider.count({ programId: document.programId, query: "rafi" }),
+ provider.searchCandidates({
+ programId: document.programId,
+ query: "rafi",
+ limit: 10,
+ }),
).rejects.toThrow("The operation was aborted due to timeout");
- expect(mocks.count).toHaveBeenCalledTimes(1);
+ expect(mocks.query).toHaveBeenCalledTimes(1);
});
it("allows query latency within the one-second deadline", async () => {
vi.useFakeTimers();
- mocks.count.mockImplementation(
- () =>
- new Promise((resolve) => setTimeout(() => resolve({ count: 1 }), 500)),
+ mocks.query.mockImplementation(
+ () => new Promise((resolve) => setTimeout(() => resolve([]), 500)),
);
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
@@ -294,13 +242,14 @@ describe("Upstash Redis partner search provider", () => {
});
try {
- const result = provider.count({
+ const result = provider.searchCandidates({
programId: document.programId,
query: "rafi",
+ limit: 10,
});
await vi.advanceTimersByTimeAsync(500);
- await expect(result).resolves.toBe(1);
+ await expect(result).resolves.toEqual({ hits: [] });
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.useRealTimers();
@@ -309,7 +258,7 @@ describe("Upstash Redis partner search provider", () => {
it("bounds the total query operation to one second", async () => {
vi.useFakeTimers();
- mocks.count.mockImplementation(() => new Promise(() => {}));
+ mocks.query.mockImplementation(() => new Promise(() => {}));
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
indexName: "test-index",
@@ -317,9 +266,10 @@ describe("Upstash Redis partner search provider", () => {
try {
const result = expect(
- provider.count({
+ provider.searchCandidates({
programId: document.programId,
query: "rafi",
+ limit: 10,
}),
).rejects.toThrow("Partner search query timed out after 1000ms");
await vi.advanceTimersByTimeAsync(1_000);
@@ -331,8 +281,7 @@ describe("Upstash Redis partner search provider", () => {
}
});
- it("writes the partner document and tag shadow documents", async () => {
- mocks.jsonMget.mockResolvedValue([null]);
+ it("writes one lean search document per enrollment", async () => {
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
indexName: "test-index",
@@ -340,62 +289,29 @@ describe("Upstash Redis partner search provider", () => {
await provider.upsert([document]);
- expect(mocks.jsonMget).toHaveBeenCalledWith(
- ["test-index:partner:pge_test"],
- "$",
- );
const entries = mocks.jsonMset.mock.calls[0]!;
- expect(entries).toHaveLength(2);
- expect(entries[0]).toEqual(
- expect.objectContaining({
- key: "test-index:partner:pge_test",
- path: "$",
- value: expect.objectContaining({
- documentType: "partner",
- partnerTagIds: "ptag_test",
- partnerTagIdsRaw: ["ptag_test"],
- tenantId: expect.stringContaining("__null:"),
- }),
- }),
- );
- expect(entries[1]).toEqual(
- expect.objectContaining({
- key: "test-index:tag:pge_test:ptag_test",
- value: expect.objectContaining({
- documentType: "tag",
- partnerTagId: "ptag_test",
- }),
- }),
- );
+ expect(entries).toHaveLength(1);
+ expect(entries[0]).toEqual({
+ key: "test-index:partner:pge_test",
+ path: "$",
+ value: {
+ programId: "prog_test",
+ searchText: expect.any(String),
+ emailNgrams: expect.any(String),
+ },
+ });
expect(entries[0].value.searchText).toContain("[email protected]");
expect(entries[0].value.searchText).toContain("dub partners");
expect(entries[0].value.searchText).toContain("rafi-on-x");
expect(entries[0].value.searchText).toContain("referrals/rafi");
expect(entries[0].value.emailNgrams).toContain("exa");
- });
-
- it("removes stale tag shadow documents before updating", async () => {
- mocks.jsonMget.mockResolvedValue([
- [{ partnerTagIdsRaw: ["ptag_old", "ptag_test"] }],
- ]);
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await provider.upsert([document]);
-
- expect(mocks.multi).toHaveBeenCalledOnce();
- expect(mocks.del).toHaveBeenCalledWith("test-index:tag:pge_test:ptag_old");
- expect(mocks.del.mock.invocationCallOrder[0]).toBeLessThan(
- mocks.jsonMset.mock.invocationCallOrder[0]!,
- );
+ expect(entries[0].value).not.toHaveProperty("partnerId");
+ expect(entries[0].value).not.toHaveProperty("status");
+ expect(entries[0].value).not.toHaveProperty("partnerTagIds");
+ expect(entries[0].value).not.toHaveProperty("totalSaleAmount");
});
it("bounds Upstash write batches", async () => {
- mocks.jsonMget.mockImplementation(async (keys: string[]) =>
- keys.map(() => null),
- );
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
indexName: "test-index",
@@ -408,16 +324,12 @@ describe("Upstash Redis partner search provider", () => {
await provider.upsert(documents);
- expect(mocks.jsonMget).toHaveBeenCalledTimes(2);
expect(mocks.jsonMset).toHaveBeenCalledTimes(2);
expect(mocks.jsonMset.mock.calls[0]).toHaveLength(100);
expect(mocks.jsonMset.mock.calls[1]).toHaveLength(1);
});
- it("deletes the partner document and its tag shadow documents", async () => {
- mocks.jsonMget.mockResolvedValue([
- [{ partnerTagIdsRaw: ["ptag_one", "ptag_two"] }],
- ]);
+ it("deletes the enrollment document directly", async () => {
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
indexName: "test-index",
@@ -425,41 +337,7 @@ describe("Upstash Redis partner search provider", () => {
await provider.delete([document.id]);
- expect(mocks.jsonMget).toHaveBeenCalledWith(
- ["test-index:partner:pge_test"],
- "$",
- );
- expect(mocks.del).toHaveBeenCalledWith(
- "test-index:partner:pge_test",
- "test-index:tag:pge_test:ptag_one",
- "test-index:tag:pge_test:ptag_two",
- );
- });
-
- it("groups partner tags using tag shadow documents", async () => {
- mocks.aggregate.mockResolvedValue({
- groups: {
- buckets: [{ key: "ptag_test", docCount: 12 }],
- sumOtherDocCount: 0,
- },
- });
- const provider = createUpstashRedisPartnerSearchProvider({
- redisClient: createRedisMock(),
- indexName: "test-index",
- });
-
- await expect(
- provider.groupBy(
- { programId: document.programId, query: "rafi" },
- "partnerTagId",
- ),
- ).resolves.toEqual([{ value: "ptag_test", count: 12 }]);
-
- const request = mocks.aggregate.mock.calls[0]![0];
- expect(request.aggregations.groups.$terms.field).toBe("partnerTagId");
- expect(JSON.stringify(request.filter)).toContain(
- '"documentType":{"$eq":"tag"}',
- );
+ expect(mocks.del).toHaveBeenCalledWith("test-index:partner:pge_test");
});
it("waits for pending index updates", async () => {
@@ -491,18 +369,23 @@ describe("Upstash Redis partner search provider", () => {
expect.objectContaining({
name: "test-index",
dataType: "json",
- prefix: "test-index:",
+ prefix: "test-index:partner:",
existsOk: true,
skipInitialScan: true,
}),
);
+ expect(Object.keys(upstashPartnerSearchSchema).sort()).toEqual([
+ "emailNgrams",
+ "programId",
+ "searchText",
+ ]);
});
it("rejects an existing index with a stale schema", async () => {
mocks.describe.mockResolvedValue({
name: "test-index",
dataType: "json",
- prefixes: ["test-index:"],
+ prefixes: ["test-index:partner:"],
schema: { id: { type: "KEYWORD" } },
});
@@ -511,6 +394,6 @@ describe("Upstash Redis partner search provider", () => {
redisClient: createRedisMock(),
indexName: "test-index",
}),
- ).rejects.toThrow("Create a new versioned index");
+ ).rejects.toThrow("Delete and recreate it before backfilling");
});
});