The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Simplify Upstash Redis partner search to relevance-only
Rafif92307bafd43
diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.tsindex fd8c9fbebc..c0f6da9274 100644--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts@@ -5,29 +5,21 @@ import {type InferFilterFromSchema,type SearchIndex,} from "@upstash/redis";-import { validatePartnerSearchCandidateLimit } from "../constants";+import {+ PARTNER_SEARCH_CANDIDATE_LIMIT,+ validatePartnerSearchCandidateLimit,+} from "../constants";import {getPartnerSearchableValues,normalizePartnerSearchQuery,} from "../searchable-values";import type {PartnerSearchCandidateQuery,- PartnerSearchCountQuery,PartnerSearchDocument,- PartnerSearchGroup,- PartnerSearchGroupField,- PartnerSearchListFilter,- PartnerSearchMetricField,PartnerSearchProvider,- PartnerSearchQuery,} from "../types";const DEFAULT_INDEX_NAME = "partner-search-v1";-// Sentinel value for nullable keyword fields. Redis Search keyword fields-// cannot store null, so we use a value that will never appear in real data-// to represent absent values and map it back to null in query results.-const NULL_VALUE = "\0__null:f47ac10b-58cc-4372-a567-0e02b2c3d479__";-const MAX_GROUPS = 1_000;const TRANSIENT_RETRY_ATTEMPTS = 2;// Keep the full query operation within the one-second latency target while// leaving a small window to retry failures that return quickly@@ -36,41 +28,11 @@ const QUERY_OPERATION_TIMEOUT_MS = 1_000;const WRITE_REQUEST_TIMEOUT_MS = 10_000;const WRITE_BATCH_SIZE = 100;const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;-const DOCUMENT_TYPE_PARTNER = "partner";-const DOCUMENT_TYPE_TAG = "tag";--// Redis Search keyword fields accept one string value rather than an array of tag IDs-// The partner document stores tag IDs as searchable text for filtering,-// but Redis Search cannot use that text to group and count individual tags-// To support tag grouping, we add one shadow document per tag with a scalar-// partnerTagId that the provider can aggregate with $terms+export const upstashPartnerSearchSchema = s.object({- id: s.keyword(),programId: s.keyword(),- partnerId: s.keyword(),- documentType: s.keyword(),searchText: s.string().noStem(),emailNgrams: s.string().noStem(),- status: s.keyword(),- tenantId: s.keyword(),- groupId: s.keyword(),- country: s.keyword(),- partnerTagIds: s.string().noStem(),- partnerTagId: s.keyword(),- referredByPartnerId: s.keyword(),- totalClicks: s.number("F64"),- totalLeads: s.number("F64"),- totalConversions: s.number("F64"),- totalSaleAmount: s.number("F64"),- totalCommissions: s.number("F64"),- netRevenue: s.number("F64"),- earningsPerClick: s.number("F64"),- averageLifetimeValue: s.number("F64"),- clickToLeadRate: s.number("F64"),- clickToConversionRate: s.number("F64"),- leadToConversionRate: s.number("F64"),- returnOnAdSpend: s.number("F64"),- createdAt: s.date().fast(),});type UpstashPartnerSearchSchema = typeof upstashPartnerSearchSchema;@@ -79,33 +41,9 @@ type UpstashPartnerSearchFilter =type UpstashPartnerSearchIndex = SearchIndex<UpstashPartnerSearchSchema>;interface UpstashPartnerSearchDocument extends Record<string, unknown> {- id: string;programId: string;- partnerId: string;- documentType: typeof DOCUMENT_TYPE_PARTNER | typeof DOCUMENT_TYPE_TAG;searchText: string;emailNgrams: string;- status: string;- tenantId: string;- groupId: string;- country: string;- partnerTagIds: string;- partnerTagIdsRaw: string[];- partnerTagId: string;- referredByPartnerId: string;- totalClicks: number;- totalLeads: number;- totalConversions: number;- totalSaleAmount: number;- totalCommissions: number;- netRevenue: number;- earningsPerClick: number;- averageLifetimeValue?: number;- clickToLeadRate?: number;- clickToConversionRate?: number;- leadToConversionRate?: number;- returnOnAdSpend?: number;- createdAt: string;}interface CreateUpstashRedisPartnerSearchProviderOptions {@@ -134,19 +72,11 @@ function getIndexName(indexName?: string): string {}function getDocumentPrefix(indexName: string): string {- return `${indexName}:`;+ return `${indexName}:partner:`;}function getDocumentKey(indexName: string, documentId: string): string {- return `${getDocumentPrefix(indexName)}partner:${documentId}`;-}--function getTagDocumentKey(- indexName: string,- documentId: string,- partnerTagId: string,-): string {- return `${getDocumentPrefix(indexName)}tag:${documentId}:${partnerTagId}`;+ return `${getDocumentPrefix(indexName)}${documentId}`;}function createRedisClient(requestTimeoutMs: number): Redis {@@ -204,68 +134,16 @@ function getQueryNgrams(query: string): string[] {function serializeUpstashDocument(document: PartnerSearchDocument,- documentType: typeof DOCUMENT_TYPE_PARTNER | typeof DOCUMENT_TYPE_TAG,- partnerTagId = NULL_VALUE,): UpstashPartnerSearchDocument {return {- id: document.id,programId: document.programId,- partnerId: document.partnerId,- documentType,searchText: getPartnerSearchableValues(document).map(normalizePartnerSearchQuery).join(" "),emailNgrams: getEmailNgrams(document.email),- status: document.status,- tenantId: document.tenantId ?? NULL_VALUE,- groupId: document.groupId ?? NULL_VALUE,- country: document.country ?? NULL_VALUE,- partnerTagIds: document.partnerTagIds.join(" "),- partnerTagIdsRaw: document.partnerTagIds,- partnerTagId,- referredByPartnerId: document.referredByPartnerId ?? NULL_VALUE,- totalClicks: document.totalClicks,- totalLeads: document.totalLeads,- totalConversions: document.totalConversions,- totalSaleAmount: document.totalSaleAmount,- totalCommissions: document.totalCommissions,- netRevenue: document.netRevenue,- earningsPerClick: document.earningsPerClick,- ...(document.averageLifetimeValue !== null && {- averageLifetimeValue: document.averageLifetimeValue,- }),- ...(document.clickToLeadRate !== null && {- clickToLeadRate: document.clickToLeadRate,- }),- ...(document.clickToConversionRate !== null && {- clickToConversionRate: document.clickToConversionRate,- }),- ...(document.leadToConversionRate !== null && {- leadToConversionRate: document.leadToConversionRate,- }),- ...(document.returnOnAdSpend !== null && {- returnOnAdSpend: document.returnOnAdSpend,- }),- createdAt: document.createdAt,};}-function buildListFilter(- field: "groupId" | "country" | "partnerTagIds",- filter: PartnerSearchListFilter,-): {- include?: UpstashPartnerSearchFilter;- exclude?: UpstashPartnerSearchFilter;-} {- const condition = {- [field]: { $in: filter.values },- } as UpstashPartnerSearchFilter;-- return filter.operator === "IN"- ? { include: condition }- : { exclude: condition };-}-function buildTextAlternatives(query: string): UpstashPartnerSearchFilter[][] {// $smart handles word-boundary and prefix matching via the inverted index.// Email n-grams handle infix/substring matching (e.g. "examp" in "[email protected]").@@ -288,75 +166,21 @@ function buildTextAlternatives(query: string): UpstashPartnerSearchFilter[][] {return alternatives;}-function buildUpstashFilter(- { programId, query, filters }: PartnerSearchCountQuery,- documentType:- | typeof DOCUMENT_TYPE_PARTNER- | typeof DOCUMENT_TYPE_TAG = DOCUMENT_TYPE_PARTNER,-): UpstashPartnerSearchFilter {- const must: UpstashPartnerSearchFilter[] = [- { documentType: { $eq: documentType } },- { programId: { $eq: programId } },- ];- const mustNot: UpstashPartnerSearchFilter[] = [];-- if (filters?.status) {- must.push({ status: { $eq: filters.status } });- }- if (filters?.tenantId) {- must.push({ tenantId: { $eq: filters.tenantId } });- }- if (filters?.partnerIds?.length) {- must.push({ partnerId: { $in: filters.partnerIds } });- }-- const listFilters: [- "groupId" | "country" | "partnerTagIds",- PartnerSearchListFilter | undefined,- ][] = [- ["groupId", filters?.groupIds],- ["country", filters?.countries],- ["partnerTagIds", filters?.partnerTagIds],- ];-- for (const [field, listFilter] of listFilters) {- if (!listFilter) {- continue;- }-- const { include, exclude } = buildListFilter(field, listFilter);- if (include) {- must.push(include);- }- if (exclude) {- mustNot.push(exclude);- }- }-- if (filters?.referredByPartnerId) {- must.push({- referredByPartnerId: { $eq: filters.referredByPartnerId },- });- }-- for (const [field, range] of Object.entries(filters?.metrics ?? {})) {- const condition = {- ...(range.min !== undefined && { $gte: range.min }),- ...(range.max !== undefined && { $lte: range.max }),- };- must.push({- [field as PartnerSearchMetricField]: condition,- } as UpstashPartnerSearchFilter);- }-- // At the root, $should requires at least one text alternative to match.- // Combining $should with $must would make text matching an optional score- // boost, allowing unrelated documents through when results use orderBy.+function buildUpstashFilter({+ programId,+ query,+}: Pick<+ PartnerSearchCandidateQuery,+ "programId" | "query"+>): UpstashPartnerSearchFilter {+ // Each branch includes the program scope because a root-level $must would+ // turn the sibling $should clauses into optional score boosts. Keeping both+ // requirements in every branch prevents unrelated program documents from+ // being returned with low scores.return {$should: buildTextAlternatives(normalizePartnerSearchQuery(query)).map((textConditions) => ({- $must: [...must, ...textConditions],- ...(mustNot.length > 0 && { $mustNot: mustNot }),+ $must: [{ programId: { $eq: programId } }, ...textConditions],}),),} as UpstashPartnerSearchFilter;@@ -429,16 +253,6 @@ async function withQueryDeadline<T>(operation: () => Promise<T>): Promise<T> {}}-function getGroupIndexField(- field: PartnerSearchGroupField,-): "status" | "country" | "groupId" | "partnerTagId" | "referredByPartnerId" {- return field;-}--function mapGroupValue(value: string): string | null {- return value === NULL_VALUE ? null : value;-}-function getSchemaSignature(schema: Record<string, Record<string, unknown>>) {return JSON.stringify(Object.fromEntries(@@ -480,71 +294,20 @@ function validateIndexDescription(!hasExpectedSchema) {throw new Error(- `Partner search index ${indexName} does not match the configured schema. Create a new versioned index.`,+ `Partner search index ${indexName} does not match the configured schema. Delete and recreate it before backfilling.`,);}}-function getStoredDocument(- value: UpstashPartnerSearchDocument[] | null,-): UpstashPartnerSearchDocument | null {- return value?.[0] ?? null;-}--async function getStoredDocuments(- redisClient: Redis,- indexName: string,- documentIds: string[],-): Promise<(UpstashPartnerSearchDocument | null)[]> {- if (documentIds.length === 0) {- return [];- }-- const values = await withTransientRetry(() =>- redisClient.json.mget<(UpstashPartnerSearchDocument[] | null)[]>(- documentIds.map((documentId) => getDocumentKey(indexName, documentId)),- "$",- ),- );-- return values.map(getStoredDocument);-}--function getStaleTagDocumentKeys(- indexName: string,- documents: PartnerSearchDocument[],- storedDocuments: (UpstashPartnerSearchDocument | null)[],-): string[] {- return documents.flatMap((document, index) => {- const currentTagIds = new Set(document.partnerTagIds);- return (storedDocuments[index]?.partnerTagIdsRaw ?? [])- .filter((partnerTagId) => !currentTagIds.has(partnerTagId))- .map((partnerTagId) =>- getTagDocumentKey(indexName, document.id, partnerTagId),- );- });-}-function getUpsertEntries(indexName: string,documents: PartnerSearchDocument[],) {- return documents.flatMap((document) => [- {- key: getDocumentKey(indexName, document.id),- path: "$",- value: serializeUpstashDocument(document, DOCUMENT_TYPE_PARTNER),- },- ...document.partnerTagIds.map((partnerTagId) => ({- key: getTagDocumentKey(indexName, document.id, partnerTagId),- path: "$",- value: serializeUpstashDocument(- document,- DOCUMENT_TYPE_TAG,- partnerTagId,- ),- })),- ]);+ return documents.map((document) => ({+ key: getDocumentKey(indexName, document.id),+ path: "$",+ value: serializeUpstashDocument(document),+ }));}async function upsertDocumentBatch(@@ -552,30 +315,9 @@ async function upsertDocumentBatch(indexName: string,documents: PartnerSearchDocument[],) {- const storedDocuments = await getStoredDocuments(- redisClient,- indexName,- documents.map(({ id }) => id),- );- const staleTagDocumentKeys = getStaleTagDocumentKeys(- indexName,- documents,- storedDocuments,- );const upsertEntries = getUpsertEntries(indexName, documents);- // Keep stale tag cleanup and document upserts in one transaction so another- // synchronization cannot modify the same keys between these operations- await withTransientRetry(async () => {- const transaction = redisClient.multi();-- if (staleTagDocumentKeys.length > 0) {- transaction.del(...staleTagDocumentKeys);- }-- transaction.json.mset(...upsertEntries);- await transaction.exec();- });+ await withTransientRetry(() => redisClient.json.mset(...upsertEntries));}async function deleteDocumentBatch(@@ -583,23 +325,9 @@ async function deleteDocumentBatch(indexName: string,documentIds: string[],) {- // Read the batch once and derive exact tag keys rather than scanning the- // entire Redis keyspace for each partner- const storedDocuments = await getStoredDocuments(- redisClient,- indexName,- documentIds,- );- const tagDocumentKeys = storedDocuments.flatMap((document, index) =>- (document?.partnerTagIdsRaw ?? []).map((partnerTagId) =>- getTagDocumentKey(indexName, documentIds[index]!, partnerTagId),- ),- );-await withTransientRetry(() =>redisClient.del(...documentIds.map((documentId) => getDocumentKey(indexName, documentId)),- ...tagDocumentKeys,),);}@@ -655,139 +383,76 @@ export function createUpstashRedisPartnerSearchProvider({schema: upstashPartnerSearchSchema,});- return {- async searchCandidates({- programId,- query,- limit,- }: PartnerSearchCandidateQuery) {- validatePartnerSearchCandidateLimit(limit);- const filter = buildUpstashFilter({ programId, query });- const results = await withQueryDeadline(() =>- queryIndex.query({- filter,- limit,- select: { id: true, partnerId: true },- }),- );- const hits = results.map(({ data, score }) => ({- id: data.id,- partnerId: data.partnerId,- score,- }));-- logPartnerSearchDebug("search", {- indexName: resolvedIndexName,- operation: "searchCandidates",- query: { programId, query, limit },- filter,- resultCount: hits.length,- hits,- });-- return { hits };- },-- async search(query: PartnerSearchQuery) {- const filter = buildUpstashFilter(query);- const offset = (query.page - 1) * query.pageSize;- const orderBy = query.sort- ? ({- [query.sort.field]: query.sort.order.toUpperCase(),- } as Record<string, "ASC" | "DESC">)- : undefined;-- const results = await withQueryDeadline(() =>- queryIndex.query({- filter,- limit: query.pageSize,- offset,- select: { id: true, partnerId: true },- ...(orderBy && { orderBy }),- }),- );- const hits = results.map(({ key, data, score }) => ({- key,- id: data.id,- partnerId: data.partnerId,- score,- }));-- logPartnerSearchDebug("search", {- indexName: resolvedIndexName,- query,- filter,- resultCount: hits.length,- hits,- });-- return {- hits: hits.map(({ id, partnerId, score }) => ({- id,- partnerId,- score,- })),- };- },-- async count(query) {- const result = await withQueryDeadline(() =>- queryIndex.count({ filter: buildUpstashFilter(query) }),- );+ async function findCandidates({+ programId,+ query,+ limit,+ }: PartnerSearchCandidateQuery) {+ validatePartnerSearchCandidateLimit(limit);+ const filter = buildUpstashFilter({ programId, query });+ const results = await withQueryDeadline(() =>+ queryIndex.query({ filter, limit, select: {} }),+ );+ const hits = results.map(({ key, score }) => ({+ id: key.slice(getDocumentPrefix(resolvedIndexName).length),+ score,+ }));++ logPartnerSearchDebug("search", {+ indexName: resolvedIndexName,+ operation: "searchCandidates",+ query: { programId, query, limit },+ filter,+ resultCount: hits.length,+ hits,+ });- logPartnerSearchDebug("count", {- indexName: resolvedIndexName,- query,- count: result.count,- });+ return { hits };+ }- return result.count;- },+ return {+ mode: "relevance-only",- async groupBy(query, field): Promise<PartnerSearchGroup[]> {- const indexField = getGroupIndexField(field);- const documentType =- field === "partnerTagId" ? DOCUMENT_TYPE_TAG : DOCUMENT_TYPE_PARTNER;- const result = await withQueryDeadline(() =>- queryIndex.aggregate({- filter: buildUpstashFilter(query, documentType),- aggregations: {- groups: {- $terms: {- field: indexField,- size: MAX_GROUPS,- },- },- },- }),- );+ searchCandidates: findCandidates,- const { buckets, sumOtherDocCount } = result.groups;+ async search(query) {+ if (query.sort) {+ throw new Error(+ "Upstash Redis partner search supports relevance ordering only.",+ );+ }+ if (query.filters && Object.keys(query.filters).length > 0) {+ throw new Error(+ "Upstash Redis partner search filters must be applied by the database.",+ );+ }- if (buckets.length >= MAX_GROUPS) {- console.warn(- `[Partner Search] groupBy("${field}") returned ${buckets.length} buckets (limit: ${MAX_GROUPS}). ` +- `Results may be truncated (${sumOtherDocCount ?? "unknown"} docs in unlisted groups).`,+ const offset = (query.page - 1) * query.pageSize;+ const requestedResults = offset + query.pageSize;+ if (requestedResults > PARTNER_SEARCH_CANDIDATE_LIMIT) {+ throw new Error(+ `Upstash Redis relevance pagination is limited to the first ${PARTNER_SEARCH_CANDIDATE_LIMIT.toLocaleString()} results.`,);}- const groups = buckets.flatMap(({ key, docCount }) => {- const value = mapGroupValue(key);- if (field === "referredByPartnerId" && value === null) {- return [];- }- return [{ value, count: docCount }];+ const result = await findCandidates({+ programId: query.programId,+ query: query.query,+ limit: requestedResults,});+ return { hits: result.hits.slice(offset, requestedResults) };+ },- logPartnerSearchDebug("groupBy", {- indexName: resolvedIndexName,- query,- field,- groupCount: groups.length,- groups,- });+ async count() {+ throw new Error(+ "Upstash Redis partner search counts must be calculated by the database.",+ );+ },- return groups;+ async groupBy() {+ throw new Error(+ "Upstash Redis partner search groups must be calculated by the database.",+ );},async waitForIndexing() {diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.tsindex 32f2c8d313..d57d40f7ea 100644--- a/apps/web/lib/api/partners/search/types.ts+++ b/apps/web/lib/api/partners/search/types.ts@@ -120,7 +120,7 @@ export interface PartnerSearchGroup {export interface PartnerSearchHit {id: string;- partnerId: string;+ partnerId?: string;score?: number;}diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.tsindex 2a60189fd4..7ec63a5438 100644--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts@@ -45,23 +45,17 @@ const document: PartnerSearchDocument = {};const mocks = vi.hoisted(() => ({- aggregate: vi.fn(),- count: vi.fn(),createIndex: vi.fn(),del: vi.fn(),describe: vi.fn(),index: vi.fn(),- jsonMget: vi.fn(),jsonMset: vi.fn(),- multi: vi.fn(),query: vi.fn(),waitIndexing: vi.fn(),}));function createRedisMock(): Redis {const searchIndex = {- aggregate: mocks.aggregate,- count: mocks.count,describe: mocks.describe,query: mocks.query,waitIndexing: mocks.waitIndexing,@@ -73,32 +67,12 @@ function createRedisMock(): Redis {const redisMock = {del: mocks.del,json: {- mget: mocks.jsonMget,mset: mocks.jsonMset,},search: {createIndex: mocks.createIndex,index: mocks.index,},- multi() {- mocks.multi();- const commands: Array<() => void> = [];- return {- del: (...args: unknown[]) => {- commands.push(() => mocks.del(...args));- },- json: {- mset: (...args: unknown[]) => {- commands.push(() => mocks.jsonMset(...args));- },- },- async exec() {- for (const cmd of commands) {- await cmd();- }- },- };- },} as unknown as Redis;return redisMock;@@ -107,18 +81,13 @@ function createRedisMock(): Redis {describe("Upstash Redis partner search provider", () => {beforeEach(() => {vi.clearAllMocks();- mocks.aggregate.mockResolvedValue({- groups: { buckets: [], sumOtherDocCount: 0 },- });- mocks.count.mockResolvedValue({ count: 0 });mocks.del.mockResolvedValue(0);mocks.describe.mockResolvedValue({name: "test-index",dataType: "json",- prefixes: ["test-index:"],+ prefixes: ["test-index:partner:"],schema: upstashPartnerSearchSchema,});- mocks.jsonMget.mockResolvedValue([]);mocks.jsonMset.mockResolvedValue("OK");mocks.query.mockResolvedValue([]);mocks.waitIndexing.mockResolvedValue(1);@@ -129,7 +98,7 @@ describe("Upstash Redis partner search provider", () => {{key: "test-index:partner:pge_test",score: 4,- data: { id: document.id, partnerId: document.partnerId },+ data: {},},]);const provider = createUpstashRedisPartnerSearchProvider({@@ -144,13 +113,13 @@ describe("Upstash Redis partner search provider", () => {limit: PARTNER_SEARCH_CANDIDATE_LIMIT,}),).resolves.toEqual({- hits: [{ id: document.id, partnerId: document.partnerId, score: 4 }],+ hits: [{ id: document.id, score: 4 }],});expect(mocks.query).toHaveBeenCalledWith(expect.objectContaining({limit: PARTNER_SEARCH_CANDIDATE_LIMIT,- select: { id: true, partnerId: true },+ select: {},}),);const request = mocks.query.mock.calls[0]![0];@@ -161,111 +130,87 @@ describe("Upstash Redis partner search provider", () => {);});- it("searches within the program and supports partial email matching", async () => {- mocks.query.mockResolvedValue([- {- key: "test-index:partner:pge_test",- score: 2,- data: { id: document.id, partnerId: document.partnerId },- },- ]);+ it("requires program-scoped text or partial email matches", async () => {const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),indexName: "test-index",});- const result = await provider.search({+ await provider.searchCandidates({programId: document.programId,query: "examp",- page: 3,- pageSize: 25,- sort: { field: "totalSaleAmount", order: "desc" },+ limit: 25,});- expect(result).toEqual({- hits: [{ id: document.id, partnerId: document.partnerId, score: 2 }],- });- expect(mocks.count).not.toHaveBeenCalled();- expect(mocks.query).toHaveBeenCalledWith(- expect.objectContaining({- limit: 25,- offset: 50,- orderBy: { totalSaleAmount: "DESC" },- select: { id: true, partnerId: true },- }),- );-const filter = mocks.query.mock.calls[0]![0].filter;expect(filter.$must).toBeUndefined();expect(filter.$should).toHaveLength(2);- expect(JSON.stringify(filter)).toContain('"programId":{"$eq":"prog_test"}');- expect(JSON.stringify(filter)).toContain(- '"documentType":{"$eq":"partner"}',- );+ for (const branch of filter.$should) {+ expect(branch.$must).toEqual(+ expect.arrayContaining([{ programId: { $eq: "prog_test" } }]),+ );+ }expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"exa"}');expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"xam"}');expect(JSON.stringify(filter)).toContain('"emailNgrams":{"$eq":"amp"}');});- it("passes list exclusions and metric ranges to Upstash", async () => {+ it("exposes relevance-only behavior during the transition", async () => {const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),indexName: "test-index",});- await provider.count({- programId: document.programId,- query: "rafi",- filters: {- countries: { values: ["US", "CA"], operator: "NOT_IN" },- partnerTagIds: { values: ["ptag_test"], operator: "IN" },- metrics: { totalSaleAmount: { min: 100, max: 1_000 } },- },- });-- const filter = mocks.count.mock.calls[0]![0].filter;- expect(filter.$should).toHaveLength(2);- for (const branch of filter.$should) {- expect(branch.$must).toEqual(- expect.arrayContaining([- { partnerTagIds: { $in: ["ptag_test"] } },- { totalSaleAmount: { $gte: 100, $lte: 1_000 } },- ]),- );- expect(branch.$mustNot).toEqual([{ country: { $in: ["US", "CA"] } }]);- }+ expect(provider.mode).toBe("relevance-only");+ await expect(+ provider.count({ programId: document.programId, query: "rafi" }),+ ).rejects.toThrow("counts must be calculated by the database");+ await expect(+ provider.groupBy(+ { programId: document.programId, query: "rafi" },+ "country",+ ),+ ).rejects.toThrow("groups must be calculated by the database");});it("retries a transient provider error", async () => {- mocks.count+ mocks.query.mockRejectedValueOnce(new Error("Upstash returned 503"))- .mockResolvedValueOnce({ count: 1 });+ .mockResolvedValueOnce([]);const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),indexName: "test-index",});await expect(- provider.count({ programId: document.programId, query: "rafi" }),- ).resolves.toBe(1);- expect(mocks.count).toHaveBeenCalledTimes(2);+ provider.searchCandidates({+ programId: document.programId,+ query: "rafi",+ limit: 10,+ }),+ ).resolves.toEqual({ hits: [] });+ expect(mocks.query).toHaveBeenCalledTimes(2);});it("does not retry a permanent provider error", async () => {- mocks.count.mockRejectedValue(new Error("Invalid search filter"));+ mocks.query.mockRejectedValue(new Error("Invalid search filter"));const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),indexName: "test-index",});await expect(- provider.count({ programId: document.programId, query: "rafi" }),+ provider.searchCandidates({+ programId: document.programId,+ query: "rafi",+ limit: 10,+ }),).rejects.toThrow("Invalid search filter");- expect(mocks.count).toHaveBeenCalledTimes(1);+ expect(mocks.query).toHaveBeenCalledTimes(1);});it("does not retry a request timeout after it consumes the query budget", async () => {- mocks.count.mockRejectedValue(+ mocks.query.mockRejectedValue(new DOMException("The operation was aborted due to timeout","TimeoutError",@@ -277,16 +222,19 @@ describe("Upstash Redis partner search provider", () => {});await expect(- provider.count({ programId: document.programId, query: "rafi" }),+ provider.searchCandidates({+ programId: document.programId,+ query: "rafi",+ limit: 10,+ }),).rejects.toThrow("The operation was aborted due to timeout");- expect(mocks.count).toHaveBeenCalledTimes(1);+ expect(mocks.query).toHaveBeenCalledTimes(1);});it("allows query latency within the one-second deadline", async () => {vi.useFakeTimers();- mocks.count.mockImplementation(- () =>- new Promise((resolve) => setTimeout(() => resolve({ count: 1 }), 500)),+ mocks.query.mockImplementation(+ () => new Promise((resolve) => setTimeout(() => resolve([]), 500)),);const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),@@ -294,13 +242,14 @@ describe("Upstash Redis partner search provider", () => {});try {- const result = provider.count({+ const result = provider.searchCandidates({programId: document.programId,query: "rafi",+ limit: 10,});await vi.advanceTimersByTimeAsync(500);- await expect(result).resolves.toBe(1);+ await expect(result).resolves.toEqual({ hits: [] });expect(vi.getTimerCount()).toBe(0);} finally {vi.useRealTimers();@@ -309,7 +258,7 @@ describe("Upstash Redis partner search provider", () => {it("bounds the total query operation to one second", async () => {vi.useFakeTimers();- mocks.count.mockImplementation(() => new Promise(() => {}));+ mocks.query.mockImplementation(() => new Promise(() => {}));const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),indexName: "test-index",@@ -317,9 +266,10 @@ describe("Upstash Redis partner search provider", () => {try {const result = expect(- provider.count({+ provider.searchCandidates({programId: document.programId,query: "rafi",+ limit: 10,}),).rejects.toThrow("Partner search query timed out after 1000ms");await vi.advanceTimersByTimeAsync(1_000);@@ -331,8 +281,7 @@ describe("Upstash Redis partner search provider", () => {}});- it("writes the partner document and tag shadow documents", async () => {- mocks.jsonMget.mockResolvedValue([null]);+ it("writes one lean search document per enrollment", async () => {const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),indexName: "test-index",@@ -340,62 +289,29 @@ describe("Upstash Redis partner search provider", () => {await provider.upsert([document]);- expect(mocks.jsonMget).toHaveBeenCalledWith(- ["test-index:partner:pge_test"],- "$",- );const entries = mocks.jsonMset.mock.calls[0]!;- expect(entries).toHaveLength(2);- expect(entries[0]).toEqual(- expect.objectContaining({- key: "test-index:partner:pge_test",- path: "$",- value: expect.objectContaining({- documentType: "partner",- partnerTagIds: "ptag_test",- partnerTagIdsRaw: ["ptag_test"],- tenantId: expect.stringContaining("__null:"),- }),- }),- );- expect(entries[1]).toEqual(- expect.objectContaining({- key: "test-index:tag:pge_test:ptag_test",- value: expect.objectContaining({- documentType: "tag",- partnerTagId: "ptag_test",- }),- }),- );+ expect(entries).toHaveLength(1);+ expect(entries[0]).toEqual({+ key: "test-index:partner:pge_test",+ path: "$",+ value: {+ programId: "prog_test",+ searchText: expect.any(String),+ emailNgrams: expect.any(String),+ },+ });expect(entries[0].value.searchText).toContain("[email protected]");expect(entries[0].value.searchText).toContain("dub partners");expect(entries[0].value.searchText).toContain("rafi-on-x");expect(entries[0].value.searchText).toContain("referrals/rafi");expect(entries[0].value.emailNgrams).toContain("exa");- });-- it("removes stale tag shadow documents before updating", async () => {- mocks.jsonMget.mockResolvedValue([- [{ partnerTagIdsRaw: ["ptag_old", "ptag_test"] }],- ]);- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await provider.upsert([document]);-- expect(mocks.multi).toHaveBeenCalledOnce();- expect(mocks.del).toHaveBeenCalledWith("test-index:tag:pge_test:ptag_old");- expect(mocks.del.mock.invocationCallOrder[0]).toBeLessThan(- mocks.jsonMset.mock.invocationCallOrder[0]!,- );+ expect(entries[0].value).not.toHaveProperty("partnerId");+ expect(entries[0].value).not.toHaveProperty("status");+ expect(entries[0].value).not.toHaveProperty("partnerTagIds");+ expect(entries[0].value).not.toHaveProperty("totalSaleAmount");});it("bounds Upstash write batches", async () => {- mocks.jsonMget.mockImplementation(async (keys: string[]) =>- keys.map(() => null),- );const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),indexName: "test-index",@@ -408,16 +324,12 @@ describe("Upstash Redis partner search provider", () => {await provider.upsert(documents);- expect(mocks.jsonMget).toHaveBeenCalledTimes(2);expect(mocks.jsonMset).toHaveBeenCalledTimes(2);expect(mocks.jsonMset.mock.calls[0]).toHaveLength(100);expect(mocks.jsonMset.mock.calls[1]).toHaveLength(1);});- it("deletes the partner document and its tag shadow documents", async () => {- mocks.jsonMget.mockResolvedValue([- [{ partnerTagIdsRaw: ["ptag_one", "ptag_two"] }],- ]);+ it("deletes the enrollment document directly", async () => {const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),indexName: "test-index",@@ -425,41 +337,7 @@ describe("Upstash Redis partner search provider", () => {await provider.delete([document.id]);- expect(mocks.jsonMget).toHaveBeenCalledWith(- ["test-index:partner:pge_test"],- "$",- );- expect(mocks.del).toHaveBeenCalledWith(- "test-index:partner:pge_test",- "test-index:tag:pge_test:ptag_one",- "test-index:tag:pge_test:ptag_two",- );- });-- it("groups partner tags using tag shadow documents", async () => {- mocks.aggregate.mockResolvedValue({- groups: {- buckets: [{ key: "ptag_test", docCount: 12 }],- sumOtherDocCount: 0,- },- });- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });-- await expect(- provider.groupBy(- { programId: document.programId, query: "rafi" },- "partnerTagId",- ),- ).resolves.toEqual([{ value: "ptag_test", count: 12 }]);-- const request = mocks.aggregate.mock.calls[0]![0];- expect(request.aggregations.groups.$terms.field).toBe("partnerTagId");- expect(JSON.stringify(request.filter)).toContain(- '"documentType":{"$eq":"tag"}',- );+ expect(mocks.del).toHaveBeenCalledWith("test-index:partner:pge_test");});it("waits for pending index updates", async () => {@@ -491,18 +369,23 @@ describe("Upstash Redis partner search provider", () => {expect.objectContaining({name: "test-index",dataType: "json",- prefix: "test-index:",+ prefix: "test-index:partner:",existsOk: true,skipInitialScan: true,}),);+ expect(Object.keys(upstashPartnerSearchSchema).sort()).toEqual([+ "emailNgrams",+ "programId",+ "searchText",+ ]);});it("rejects an existing index with a stale schema", async () => {mocks.describe.mockResolvedValue({name: "test-index",dataType: "json",- prefixes: ["test-index:"],+ prefixes: ["test-index:partner:"],schema: { id: { type: "KEYWORD" } },});@@ -511,6 +394,6 @@ describe("Upstash Redis partner search provider", () => {redisClient: createRedisMock(),indexName: "test-index",}),- ).rejects.toThrow("Create a new versioned index");+ ).rejects.toThrow("Delete and recreate it before backfilling");});});