The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Refactor upstash redis and upstash search for better code organization
Rafi0bc7998b4fe0
diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.tsindex 1291ae42dc..4b77ca6093 100644--- a/apps/web/lib/api/partners/search/index.ts+++ b/apps/web/lib/api/partners/search/index.ts@@ -3,8 +3,6 @@ export * from "./build-search-query";export * from "./find-candidates";export * from "./order-search-results";export * from "./provider";-export * from "./providers/upstash-redis";-export * from "./providers/upstash-search";export * from "./reconcile";export * from "./searchable-values";export * from "./serialize-document";diff --git a/apps/web/lib/api/partners/search/providers/resilience.ts b/apps/web/lib/api/partners/search/providers/resilience.tsnew file mode 100644index 0000000000..0b4595c696--- /dev/null+++ b/apps/web/lib/api/partners/search/providers/resilience.ts@@ -0,0 +1,69 @@+const TRANSIENT_RETRY_ATTEMPTS = 2;++function isTransientError(error: unknown): boolean {+ if (error instanceof TypeError) {+ return true;+ }++ const message = error instanceof Error ? error.message : String(error);+ return /\b(429|500|502|503|504)\b|rate.?limit|timeout|timed out|fetch failed|network|ECONNRESET|ETIMEDOUT/i.test(+ message,+ );+}++function isTimeoutError(error: unknown): boolean {+ const message = error instanceof Error ? error.message : String(error);+ return /timeout|timed out|ETIMEDOUT/i.test(message);+}++export async function withTransientRetry<T>(+ operation: () => Promise<T>,+ { retryTimeouts = true }: { retryTimeouts?: boolean } = {},+): Promise<T> {+ for (let attempt = 1; attempt <= TRANSIENT_RETRY_ATTEMPTS; attempt++) {+ try {+ return await operation();+ } catch (error) {+ if (+ attempt === TRANSIENT_RETRY_ATTEMPTS ||+ !isTransientError(error) ||+ (!retryTimeouts && isTimeoutError(error))+ ) {+ throw error;+ }++ await new Promise((resolve) =>+ setTimeout(resolve, 50 * attempt + Math.random() * 25),+ );+ }+ }++ throw new Error("Partner search operation failed.");+}++export async function withQueryDeadline<T>(+ operation: () => Promise<T>,+ timeoutMs: number,+): Promise<T> {+ let timeoutId: ReturnType<typeof setTimeout> | undefined;+ const timeout = new Promise<never>((_, reject) => {+ timeoutId = setTimeout(+ () =>+ reject(+ new Error(`Partner search query timed out after ${timeoutMs}ms.`),+ ),+ timeoutMs,+ );+ });++ try {+ // A request timeout consumes nearly the full SLA budget, so only retry+ // transient failures such as rate limits and 503s that return quickly.+ return await Promise.race([+ withTransientRetry(operation, { retryTimeouts: false }),+ timeout,+ ]);+ } finally {+ clearTimeout(timeoutId);+ }+}diff --git a/apps/web/lib/api/partners/search/providers/shared.ts b/apps/web/lib/api/partners/search/providers/shared.tsnew file mode 100644index 0000000000..fc2fece416--- /dev/null+++ b/apps/web/lib/api/partners/search/providers/shared.ts@@ -0,0 +1,69 @@+import { normalizePartnerSearchQuery } from "../searchable-values";++const NGRAM_SIZE = 3;++/**+ * Three-character tokens, which is what lets a partial email match without a+ * leading-wildcard query. Providers package these differently — Redis Search+ * requires each one as a separate `$eq`, Upstash Search takes them as a single+ * query string — so this returns the tokens and leaves the packaging to them.+ */+function toNgrams(value: string): string[] {+ return Array.from(+ new Set(+ Array.from({ length: value.length - (NGRAM_SIZE - 1) }, (_, index) =>+ value.slice(index, index + NGRAM_SIZE),+ ),+ ),+ );+}++/** N-grams of an indexed email, as the space-separated field both providers store. */+export function getEmailNgrams(email: string | null): string {+ if (!email) {+ return "";+ }++ const normalized = normalizePartnerSearchQuery(email);+ if (normalized.length < NGRAM_SIZE) {+ return normalized;+ }++ return toNgrams(normalized).join(" ");+}++/**+ * N-grams of a search term, or none when the term cannot use the n-gram path.+ * Multi-word queries are excluded because the stored field holds n-grams of a+ * single address.+ */+export function getQueryNgrams(query: string): string[] {+ if (query.length < NGRAM_SIZE || /\s/u.test(query)) {+ return [];+ }++ return toNgrams(query);+}++export function logPartnerSearchDebug(+ provider: string,+ details: Record<string, unknown>,+) {+ if (process.env.PARTNER_SEARCH_DEBUG !== "true") {+ return;+ }++ console.log(`[Partner Search Debug] ${provider}`, details);+}++/**+ * First non-blank wins. `||` rather than `??` because a blank value has to fall+ * through to the default, otherwise an empty env var silently targets an index+ * named "".+ */+export function resolveIndexName(+ candidates: (string | undefined)[],+ fallback: string,+): string {+ return candidates.map((value) => value?.trim()).find(Boolean) || fallback;+}diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.tsindex 74b6822838..196a5a3126 100644--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts@@ -15,9 +15,15 @@ import type {PartnerSearchProvider,} from "../types";import { validatePartnerSearchCandidateLimit } from "../types";+import { withQueryDeadline, withTransientRetry } from "./resilience";+import {+ getEmailNgrams,+ getQueryNgrams,+ logPartnerSearchDebug,+ resolveIndexName,+} from "./shared";const DEFAULT_INDEX_NAME = "partner-search-v1";-const TRANSIENT_RETRY_ATTEMPTS = 2;// Keep the full query operation within the one-second latency target while// leaving a small window to retry failures that return quicklyconst QUERY_REQUEST_TIMEOUT_MS = 900;@@ -49,21 +55,10 @@ interface CreateUpstashRedisPartnerSearchProviderOptions {indexName?: string;}-function logPartnerSearchDebug(details: Record<string, unknown>) {- if (process.env.PARTNER_SEARCH_DEBUG !== "true") {- return;- }-- console.log("[Partner Search Debug] Upstash search", details);-}-function getIndexName(indexName?: string): string {- // `||` rather than `??`: a blank value has to fall through to the default,- // otherwise an empty env var silently targets an index named "".- return (- indexName?.trim() ||- process.env.PARTNER_SEARCH_INDEX_NAME?.trim() ||- DEFAULT_INDEX_NAME+ return resolveIndexName(+ [indexName, process.env.PARTNER_SEARCH_INDEX_NAME],+ DEFAULT_INDEX_NAME,);}@@ -94,40 +89,6 @@ function createRedisClient(requestTimeoutMs: number): Redis {});}-function getEmailNgrams(email: string | null): string {- if (!email) {- return "";- }-- const normalized = normalizePartnerSearchQuery(email);- if (normalized.length < 3) {- return normalized;- }-- // Three-character tokens support partial email matches without a leading-wildcard regex query- return Array.from(- new Set(- Array.from({ length: normalized.length - 2 }, (_, index) =>- normalized.slice(index, index + 3),- ),- ),- ).join(" ");-}--function getQueryNgrams(query: string): string[] {- if (query.length < 3 || /\s/u.test(query)) {- return [];- }-- return Array.from(- new Set(- Array.from({ length: query.length - 2 }, (_, index) =>- query.slice(index, index + 3),- ),- ),- );-}-function serializeUpstashDocument(document: PartnerSearchDocument,): UpstashPartnerSearchDocument {@@ -182,73 +143,6 @@ function buildUpstashFilter({} as UpstashPartnerSearchFilter;}-function isTransientError(error: unknown): boolean {- if (error instanceof TypeError) {- return true;- }-- const message = error instanceof Error ? error.message : String(error);- return /\b(429|500|502|503|504)\b|rate.?limit|timeout|timed out|fetch failed|network|ECONNRESET|ETIMEDOUT/i.test(- message,- );-}--function isTimeoutError(error: unknown): boolean {- const message = error instanceof Error ? error.message : String(error);- return /timeout|timed out|ETIMEDOUT/i.test(message);-}--async function withTransientRetry<T>(- operation: () => Promise<T>,- { retryTimeouts = true }: { retryTimeouts?: boolean } = {},-): Promise<T> {- for (let attempt = 1; attempt <= TRANSIENT_RETRY_ATTEMPTS; attempt++) {- try {- return await operation();- } catch (error) {- if (- attempt === TRANSIENT_RETRY_ATTEMPTS ||- !isTransientError(error) ||- (!retryTimeouts && isTimeoutError(error))- ) {- throw error;- }-- await new Promise((resolve) =>- setTimeout(resolve, 50 * attempt + Math.random() * 25),- );- }- }-- throw new Error("Partner search operation failed.");-}--async function withQueryDeadline<T>(operation: () => Promise<T>): Promise<T> {- let timeoutId: ReturnType<typeof setTimeout> | undefined;- const timeout = new Promise<never>((_, reject) => {- timeoutId = setTimeout(- () =>- reject(- new Error(- `Partner search query timed out after ${QUERY_OPERATION_TIMEOUT_MS}ms.`,- ),- ),- QUERY_OPERATION_TIMEOUT_MS,- );- });-- try {- // A request timeout consumes nearly the full SLA budget, so only retry- // transient failures such as rate limits and 503s that return quickly- return await Promise.race([- withTransientRetry(operation, { retryTimeouts: false }),- timeout,- ]);- } finally {- clearTimeout(timeoutId);- }-}-function getSchemaSignature(schema: Record<string, Record<string, unknown>>) {return JSON.stringify(Object.fromEntries(@@ -388,15 +282,16 @@ export function createUpstashRedisPartnerSearchProvider({}: PartnerSearchCandidateQuery) {validatePartnerSearchCandidateLimit(limit);const filter = buildUpstashFilter({ programId, query });- const results = await withQueryDeadline(() =>- queryIndex.query({ filter, limit, select: {} }),+ const results = await withQueryDeadline(+ () => queryIndex.query({ filter, limit, select: {} }),+ QUERY_OPERATION_TIMEOUT_MS,);const hits = results.map(({ key, score }) => ({id: key.slice(getDocumentPrefix(resolvedIndexName).length),score,}));- logPartnerSearchDebug({+ logPartnerSearchDebug("Upstash Redis Search", {indexName: resolvedIndexName,operation: "searchCandidates",query: { programId, query, limit },diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.tsindex f2e96b831c..145c991be9 100644--- a/apps/web/lib/api/partners/search/providers/upstash-search.ts+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts@@ -7,13 +7,19 @@ import type {PartnerSearchProvider,} from "../types";import { validatePartnerSearchCandidateLimit } from "../types";+import { withQueryDeadline, withTransientRetry } from "./resilience";+import {+ getEmailNgrams,+ getQueryNgrams,+ logPartnerSearchDebug,+ resolveIndexName,+} from "./shared";const DEFAULT_INDEX_NAME = "partner-search-v1";const WRITE_BATCH_SIZE = 100;const DELETE_BATCH_SIZE = 1_000;const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;const WAIT_FOR_INDEXING_POLL_MS = 100;-const TRANSIENT_RETRY_ATTEMPTS = 2;const QUERY_OPERATION_TIMEOUT_MS = 1_000;interface UpstashSearchContent extends Record<string, unknown> {@@ -75,13 +81,13 @@ interface CreateUpstashSearchPartnerSearchProviderOptions {}function getIndexName(indexName?: string): string {- // `||` rather than `??`: a blank value has to fall through to the default,- // otherwise an empty env var silently targets an index named "".- return (- indexName?.trim() ||- process.env.PARTNER_UPSTASH_SEARCH_INDEX_NAME?.trim() ||- process.env.PARTNER_SEARCH_INDEX_NAME?.trim() ||- DEFAULT_INDEX_NAME+ return resolveIndexName(+ [+ indexName,+ process.env.PARTNER_UPSTASH_SEARCH_INDEX_NAME,+ process.env.PARTNER_SEARCH_INDEX_NAME,+ ],+ DEFAULT_INDEX_NAME,);}@@ -107,39 +113,6 @@ function joinValues(values: string[], maxLength: number): string {return truncate(values.join(" "), maxLength);}-function getEmailNgrams(email: string | null): string {- if (!email) {- return "";- }-- const normalized = normalizePartnerSearchQuery(email);- if (normalized.length < 3) {- return normalized;- }-- return Array.from(- new Set(- Array.from({ length: normalized.length - 2 }, (_, index) =>- normalized.slice(index, index + 3),- ),- ),- ).join(" ");-}--function getQueryNgrams(query: string): string | null {- if (query.length < 3 || /\s/u.test(query)) {- return null;- }-- return Array.from(- new Set(- Array.from({ length: query.length - 2 }, (_, index) =>- query.slice(index, index + 3),- ),- ),- ).join(" ");-}-function serializeUpstashSearchDocument(document: PartnerSearchDocument) {// Upstash Search limits searchable content to 4,096 characters. Give each// assignment field its own budget so a long description cannot crowd out@@ -182,71 +155,6 @@ function buildUpstashSearchFilter(programId: string): string {return `@metadata.programId = ${quoteFilterValue(programId)}`;}-function isTransientError(error: unknown): boolean {- if (error instanceof TypeError) {- return true;- }-- const message = error instanceof Error ? error.message : String(error);- return /\b(429|500|502|503|504)\b|rate.?limit|timeout|timed out|fetch failed|network|ECONNRESET|ETIMEDOUT/i.test(- message,- );-}--function isTimeoutError(error: unknown): boolean {- const message = error instanceof Error ? error.message : String(error);- return /timeout|timed out|ETIMEDOUT/i.test(message);-}--async function withTransientRetry<T>(- operation: () => Promise<T>,- { retryTimeouts = true }: { retryTimeouts?: boolean } = {},-): Promise<T> {- for (let attempt = 1; attempt <= TRANSIENT_RETRY_ATTEMPTS; attempt++) {- try {- return await operation();- } catch (error) {- if (- attempt === TRANSIENT_RETRY_ATTEMPTS ||- !isTransientError(error) ||- (!retryTimeouts && isTimeoutError(error))- ) {- throw error;- }-- await new Promise((resolve) =>- setTimeout(resolve, 50 * attempt + Math.random() * 25),- );- }- }-- throw new Error("Partner search operation failed.");-}--async function withQueryDeadline<T>(operation: () => Promise<T>): Promise<T> {- let timeoutId: ReturnType<typeof setTimeout> | undefined;- const timeout = new Promise<never>((_, reject) => {- timeoutId = setTimeout(- () =>- reject(- new Error(- `Partner search query timed out after ${QUERY_OPERATION_TIMEOUT_MS}ms.`,- ),- ),- QUERY_OPERATION_TIMEOUT_MS,- );- });-- try {- return await Promise.race([- withTransientRetry(operation, { retryTimeouts: false }),- timeout,- ]);- } finally {- clearTimeout(timeoutId);- }-}-function mergeSearchResults(standardResults: UpstashSearchResult[],emailResults: UpstashSearchResult[],@@ -266,14 +174,6 @@ function mergeSearchResults();}-function logPartnerSearchDebug(details: Record<string, unknown>) {- if (process.env.PARTNER_SEARCH_DEBUG !== "true") {- return;- }-- console.log("[Partner Search Debug] Upstash Search", details);-}-export async function deleteUpstashSearchPartnerSearchIndex({searchIndex,indexName,@@ -315,13 +215,17 @@ export function createUpstashSearchPartnerSearchProvider({};const queryNgrams = getQueryNgrams(normalizedQuery);- const [standardResults, rawEmailResults] = await withQueryDeadline(() =>- Promise.all([- index.search({ ...searchParams, query: normalizedQuery }),- queryNgrams- ? index.search({ ...searchParams, query: queryNgrams })- : Promise.resolve([]),- ]),+ const [standardResults, rawEmailResults] = await withQueryDeadline(+ () =>+ Promise.all([+ index.search({ ...searchParams, query: normalizedQuery }),+ // Upstash Search takes the n-grams as one query string, unlike Redis+ // Search which needs each as a separate condition.+ queryNgrams.length > 0+ ? index.search({ ...searchParams, query: queryNgrams.join(" ") })+ : Promise.resolve([]),+ ]),+ QUERY_OPERATION_TIMEOUT_MS,);const emailResults = rawEmailResults.filter(({ content }) =>@@ -343,7 +247,7 @@ export function createUpstashSearchPartnerSearchProvider({score,}));- logPartnerSearchDebug({+ logPartnerSearchDebug("Upstash Search", {indexName: resolvedIndexName,operation: "searchCandidates",query: { programId, query, limit },diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.tsindex 8b9fe26db0..1e8c11e5dc 100644--- a/apps/web/scripts/partners/backfill-partner-search.ts+++ b/apps/web/scripts/partners/backfill-partner-search.ts@@ -18,11 +18,11 @@import {backfillPartnerSearch,- createUpstashRedisPartnerSearchIndex,getPartnerSearchProviderName,reconcilePartnerSearchIndex,type PartnerSearchBackfillProgress,} from "@/lib/api/partners/search";+import { createUpstashRedisPartnerSearchIndex } from "@/lib/api/partners/search/providers/upstash-redis";import { prisma } from "@/lib/prisma";import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";import "dotenv-flow/config";diff --git a/apps/web/scripts/partners/create-partner-search-index.ts b/apps/web/scripts/partners/create-partner-search-index.tsindex e351d21de4..3a581423bf 100644--- a/apps/web/scripts/partners/create-partner-search-index.ts+++ b/apps/web/scripts/partners/create-partner-search-index.ts@@ -1,7 +1,5 @@-import {- createUpstashRedisPartnerSearchIndex,- getPartnerSearchProviderName,-} from "@/lib/api/partners/search";+import { getPartnerSearchProviderName } from "@/lib/api/partners/search";+import { createUpstashRedisPartnerSearchIndex } from "@/lib/api/partners/search/providers/upstash-redis";import "dotenv-flow/config";async function main() {diff --git a/apps/web/scripts/partners/delete-partner-search-index.ts b/apps/web/scripts/partners/delete-partner-search-index.tsindex 4ee71d2ad7..9c9053b6aa 100644--- a/apps/web/scripts/partners/delete-partner-search-index.ts+++ b/apps/web/scripts/partners/delete-partner-search-index.ts@@ -1,7 +1,5 @@-import {- deleteUpstashSearchPartnerSearchIndex,- getPartnerSearchProviderName,-} from "@/lib/api/partners/search";+import { getPartnerSearchProviderName } from "@/lib/api/partners/search";+import { deleteUpstashSearchPartnerSearchIndex } from "@/lib/api/partners/search/providers/upstash-search";import { chunk } from "@dub/utils";import { Redis } from "@upstash/redis";import "dotenv-flow/config";diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.tsindex f7674483e2..33b3c7c318 100644--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts@@ -1,10 +1,12 @@import {- createUpstashRedisPartnerSearchIndex,- createUpstashRedisPartnerSearchProvider,PARTNER_SEARCH_CANDIDATE_LIMIT,type PartnerSearchDocument,- upstashPartnerSearchSchema,} from "@/lib/api/partners/search";+import {+ createUpstashRedisPartnerSearchIndex,+ createUpstashRedisPartnerSearchProvider,+ upstashPartnerSearchSchema,+} from "@/lib/api/partners/search/providers/upstash-redis";import type { Redis } from "@upstash/redis";import { beforeEach, describe, expect, it, vi } from "vitest";diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.tsindex 50be76b5ac..cc44306c3d 100644--- a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts@@ -1,8 +1,8 @@+import { type PartnerSearchDocument } from "@/lib/api/partners/search";import {createUpstashSearchPartnerSearchProvider,deleteUpstashSearchPartnerSearchIndex,- type PartnerSearchDocument,-} from "@/lib/api/partners/search";+} from "@/lib/api/partners/search/providers/upstash-search";import { beforeEach, describe, expect, it, vi } from "vitest";const document: PartnerSearchDocument = {