Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Refactor upstash redis and upstash search for better code organization

Rafi0bc7998b4fe0
diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.ts
index 1291ae42dc..4b77ca6093 100644
--- a/apps/web/lib/api/partners/search/index.ts
+++ b/apps/web/lib/api/partners/search/index.ts
@@ -3,8 +3,6 @@ export * from "./build-search-query";
export * from "./find-candidates";
export * from "./order-search-results";
export * from "./provider";
-export * from "./providers/upstash-redis";
-export * from "./providers/upstash-search";
export * from "./reconcile";
export * from "./searchable-values";
export * from "./serialize-document";
diff --git a/apps/web/lib/api/partners/search/providers/resilience.ts b/apps/web/lib/api/partners/search/providers/resilience.ts
new file mode 100644
index 0000000000..0b4595c696
--- /dev/null
+++ b/apps/web/lib/api/partners/search/providers/resilience.ts
@@ -0,0 +1,69 @@
+const TRANSIENT_RETRY_ATTEMPTS = 2;
+
+function isTransientError(error: unknown): boolean {
+ if (error instanceof TypeError) {
+ return true;
+ }
+
+ const message = error instanceof Error ? error.message : String(error);
+ return /\b(429|500|502|503|504)\b|rate.?limit|timeout|timed out|fetch failed|network|ECONNRESET|ETIMEDOUT/i.test(
+ message,
+ );
+}
+
+function isTimeoutError(error: unknown): boolean {
+ const message = error instanceof Error ? error.message : String(error);
+ return /timeout|timed out|ETIMEDOUT/i.test(message);
+}
+
+export async function withTransientRetry<T>(
+ operation: () => Promise<T>,
+ { retryTimeouts = true }: { retryTimeouts?: boolean } = {},
+): Promise<T> {
+ for (let attempt = 1; attempt <= TRANSIENT_RETRY_ATTEMPTS; attempt++) {
+ try {
+ return await operation();
+ } catch (error) {
+ if (
+ attempt === TRANSIENT_RETRY_ATTEMPTS ||
+ !isTransientError(error) ||
+ (!retryTimeouts && isTimeoutError(error))
+ ) {
+ throw error;
+ }
+
+ await new Promise((resolve) =>
+ setTimeout(resolve, 50 * attempt + Math.random() * 25),
+ );
+ }
+ }
+
+ throw new Error("Partner search operation failed.");
+}
+
+export async function withQueryDeadline<T>(
+ operation: () => Promise<T>,
+ timeoutMs: number,
+): Promise<T> {
+ let timeoutId: ReturnType<typeof setTimeout> | undefined;
+ const timeout = new Promise<never>((_, reject) => {
+ timeoutId = setTimeout(
+ () =>
+ reject(
+ new Error(`Partner search query timed out after ${timeoutMs}ms.`),
+ ),
+ timeoutMs,
+ );
+ });
+
+ try {
+ // A request timeout consumes nearly the full SLA budget, so only retry
+ // transient failures such as rate limits and 503s that return quickly.
+ return await Promise.race([
+ withTransientRetry(operation, { retryTimeouts: false }),
+ timeout,
+ ]);
+ } finally {
+ clearTimeout(timeoutId);
+ }
+}
diff --git a/apps/web/lib/api/partners/search/providers/shared.ts b/apps/web/lib/api/partners/search/providers/shared.ts
new file mode 100644
index 0000000000..fc2fece416
--- /dev/null
+++ b/apps/web/lib/api/partners/search/providers/shared.ts
@@ -0,0 +1,69 @@
+import { normalizePartnerSearchQuery } from "../searchable-values";
+
+const NGRAM_SIZE = 3;
+
+/**
+ * Three-character tokens, which is what lets a partial email match without a
+ * leading-wildcard query. Providers package these differently — Redis Search
+ * requires each one as a separate `$eq`, Upstash Search takes them as a single
+ * query string — so this returns the tokens and leaves the packaging to them.
+ */
+function toNgrams(value: string): string[] {
+ return Array.from(
+ new Set(
+ Array.from({ length: value.length - (NGRAM_SIZE - 1) }, (_, index) =>
+ value.slice(index, index + NGRAM_SIZE),
+ ),
+ ),
+ );
+}
+
+/** N-grams of an indexed email, as the space-separated field both providers store. */
+export function getEmailNgrams(email: string | null): string {
+ if (!email) {
+ return "";
+ }
+
+ const normalized = normalizePartnerSearchQuery(email);
+ if (normalized.length < NGRAM_SIZE) {
+ return normalized;
+ }
+
+ return toNgrams(normalized).join(" ");
+}
+
+/**
+ * N-grams of a search term, or none when the term cannot use the n-gram path.
+ * Multi-word queries are excluded because the stored field holds n-grams of a
+ * single address.
+ */
+export function getQueryNgrams(query: string): string[] {
+ if (query.length < NGRAM_SIZE || /\s/u.test(query)) {
+ return [];
+ }
+
+ return toNgrams(query);
+}
+
+export function logPartnerSearchDebug(
+ provider: string,
+ details: Record<string, unknown>,
+) {
+ if (process.env.PARTNER_SEARCH_DEBUG !== "true") {
+ return;
+ }
+
+ console.log(`[Partner Search Debug] ${provider}`, details);
+}
+
+/**
+ * First non-blank wins. `||` rather than `??` because a blank value has to fall
+ * through to the default, otherwise an empty env var silently targets an index
+ * named "".
+ */
+export function resolveIndexName(
+ candidates: (string | undefined)[],
+ fallback: string,
+): string {
+ return candidates.map((value) => value?.trim()).find(Boolean) || fallback;
+}
diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
index 74b6822838..196a5a3126 100644
--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts
+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
@@ -15,9 +15,15 @@ import type {
PartnerSearchProvider,
} from "../types";
import { validatePartnerSearchCandidateLimit } from "../types";
+import { withQueryDeadline, withTransientRetry } from "./resilience";
+import {
+ getEmailNgrams,
+ getQueryNgrams,
+ logPartnerSearchDebug,
+ resolveIndexName,
+} from "./shared";
const DEFAULT_INDEX_NAME = "partner-search-v1";
-const TRANSIENT_RETRY_ATTEMPTS = 2;
// Keep the full query operation within the one-second latency target while
// leaving a small window to retry failures that return quickly
const QUERY_REQUEST_TIMEOUT_MS = 900;
@@ -49,21 +55,10 @@ interface CreateUpstashRedisPartnerSearchProviderOptions {
indexName?: string;
}
-function logPartnerSearchDebug(details: Record<string, unknown>) {
- if (process.env.PARTNER_SEARCH_DEBUG !== "true") {
- return;
- }
-
- console.log("[Partner Search Debug] Upstash search", details);
-}
-
function getIndexName(indexName?: string): string {
- // `||` rather than `??`: a blank value has to fall through to the default,
- // otherwise an empty env var silently targets an index named "".
- return (
- indexName?.trim() ||
- process.env.PARTNER_SEARCH_INDEX_NAME?.trim() ||
- DEFAULT_INDEX_NAME
+ return resolveIndexName(
+ [indexName, process.env.PARTNER_SEARCH_INDEX_NAME],
+ DEFAULT_INDEX_NAME,
);
}
@@ -94,40 +89,6 @@ function createRedisClient(requestTimeoutMs: number): Redis {
});
}
-function getEmailNgrams(email: string | null): string {
- if (!email) {
- return "";
- }
-
- const normalized = normalizePartnerSearchQuery(email);
- if (normalized.length < 3) {
- return normalized;
- }
-
- // Three-character tokens support partial email matches without a leading-wildcard regex query
- return Array.from(
- new Set(
- Array.from({ length: normalized.length - 2 }, (_, index) =>
- normalized.slice(index, index + 3),
- ),
- ),
- ).join(" ");
-}
-
-function getQueryNgrams(query: string): string[] {
- if (query.length < 3 || /\s/u.test(query)) {
- return [];
- }
-
- return Array.from(
- new Set(
- Array.from({ length: query.length - 2 }, (_, index) =>
- query.slice(index, index + 3),
- ),
- ),
- );
-}
-
function serializeUpstashDocument(
document: PartnerSearchDocument,
): UpstashPartnerSearchDocument {
@@ -182,73 +143,6 @@ function buildUpstashFilter({
} as UpstashPartnerSearchFilter;
}
-function isTransientError(error: unknown): boolean {
- if (error instanceof TypeError) {
- return true;
- }
-
- const message = error instanceof Error ? error.message : String(error);
- return /\b(429|500|502|503|504)\b|rate.?limit|timeout|timed out|fetch failed|network|ECONNRESET|ETIMEDOUT/i.test(
- message,
- );
-}
-
-function isTimeoutError(error: unknown): boolean {
- const message = error instanceof Error ? error.message : String(error);
- return /timeout|timed out|ETIMEDOUT/i.test(message);
-}
-
-async function withTransientRetry<T>(
- operation: () => Promise<T>,
- { retryTimeouts = true }: { retryTimeouts?: boolean } = {},
-): Promise<T> {
- for (let attempt = 1; attempt <= TRANSIENT_RETRY_ATTEMPTS; attempt++) {
- try {
- return await operation();
- } catch (error) {
- if (
- attempt === TRANSIENT_RETRY_ATTEMPTS ||
- !isTransientError(error) ||
- (!retryTimeouts && isTimeoutError(error))
- ) {
- throw error;
- }
-
- await new Promise((resolve) =>
- setTimeout(resolve, 50 * attempt + Math.random() * 25),
- );
- }
- }
-
- throw new Error("Partner search operation failed.");
-}
-
-async function withQueryDeadline<T>(operation: () => Promise<T>): Promise<T> {
- let timeoutId: ReturnType<typeof setTimeout> | undefined;
- const timeout = new Promise<never>((_, reject) => {
- timeoutId = setTimeout(
- () =>
- reject(
- new Error(
- `Partner search query timed out after ${QUERY_OPERATION_TIMEOUT_MS}ms.`,
- ),
- ),
- QUERY_OPERATION_TIMEOUT_MS,
- );
- });
-
- try {
- // A request timeout consumes nearly the full SLA budget, so only retry
- // transient failures such as rate limits and 503s that return quickly
- return await Promise.race([
- withTransientRetry(operation, { retryTimeouts: false }),
- timeout,
- ]);
- } finally {
- clearTimeout(timeoutId);
- }
-}
-
function getSchemaSignature(schema: Record<string, Record<string, unknown>>) {
return JSON.stringify(
Object.fromEntries(
@@ -388,15 +282,16 @@ export function createUpstashRedisPartnerSearchProvider({
}: PartnerSearchCandidateQuery) {
validatePartnerSearchCandidateLimit(limit);
const filter = buildUpstashFilter({ programId, query });
- const results = await withQueryDeadline(() =>
- queryIndex.query({ filter, limit, select: {} }),
+ const results = await withQueryDeadline(
+ () => queryIndex.query({ filter, limit, select: {} }),
+ QUERY_OPERATION_TIMEOUT_MS,
);
const hits = results.map(({ key, score }) => ({
id: key.slice(getDocumentPrefix(resolvedIndexName).length),
score,
}));
- logPartnerSearchDebug({
+ logPartnerSearchDebug("Upstash Redis Search", {
indexName: resolvedIndexName,
operation: "searchCandidates",
query: { programId, query, limit },
diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.ts
index f2e96b831c..145c991be9 100644
--- a/apps/web/lib/api/partners/search/providers/upstash-search.ts
+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts
@@ -7,13 +7,19 @@ import type {
PartnerSearchProvider,
} from "../types";
import { validatePartnerSearchCandidateLimit } from "../types";
+import { withQueryDeadline, withTransientRetry } from "./resilience";
+import {
+ getEmailNgrams,
+ getQueryNgrams,
+ logPartnerSearchDebug,
+ resolveIndexName,
+} from "./shared";
const DEFAULT_INDEX_NAME = "partner-search-v1";
const WRITE_BATCH_SIZE = 100;
const DELETE_BATCH_SIZE = 1_000;
const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;
const WAIT_FOR_INDEXING_POLL_MS = 100;
-const TRANSIENT_RETRY_ATTEMPTS = 2;
const QUERY_OPERATION_TIMEOUT_MS = 1_000;
interface UpstashSearchContent extends Record<string, unknown> {
@@ -75,13 +81,13 @@ interface CreateUpstashSearchPartnerSearchProviderOptions {
}
function getIndexName(indexName?: string): string {
- // `||` rather than `??`: a blank value has to fall through to the default,
- // otherwise an empty env var silently targets an index named "".
- return (
- indexName?.trim() ||
- process.env.PARTNER_UPSTASH_SEARCH_INDEX_NAME?.trim() ||
- process.env.PARTNER_SEARCH_INDEX_NAME?.trim() ||
- DEFAULT_INDEX_NAME
+ return resolveIndexName(
+ [
+ indexName,
+ process.env.PARTNER_UPSTASH_SEARCH_INDEX_NAME,
+ process.env.PARTNER_SEARCH_INDEX_NAME,
+ ],
+ DEFAULT_INDEX_NAME,
);
}
@@ -107,39 +113,6 @@ function joinValues(values: string[], maxLength: number): string {
return truncate(values.join(" "), maxLength);
}
-function getEmailNgrams(email: string | null): string {
- if (!email) {
- return "";
- }
-
- const normalized = normalizePartnerSearchQuery(email);
- if (normalized.length < 3) {
- return normalized;
- }
-
- return Array.from(
- new Set(
- Array.from({ length: normalized.length - 2 }, (_, index) =>
- normalized.slice(index, index + 3),
- ),
- ),
- ).join(" ");
-}
-
-function getQueryNgrams(query: string): string | null {
- if (query.length < 3 || /\s/u.test(query)) {
- return null;
- }
-
- return Array.from(
- new Set(
- Array.from({ length: query.length - 2 }, (_, index) =>
- query.slice(index, index + 3),
- ),
- ),
- ).join(" ");
-}
-
function serializeUpstashSearchDocument(document: PartnerSearchDocument) {
// Upstash Search limits searchable content to 4,096 characters. Give each
// assignment field its own budget so a long description cannot crowd out
@@ -182,71 +155,6 @@ function buildUpstashSearchFilter(programId: string): string {
return `@metadata.programId = ${quoteFilterValue(programId)}`;
}
-function isTransientError(error: unknown): boolean {
- if (error instanceof TypeError) {
- return true;
- }
-
- const message = error instanceof Error ? error.message : String(error);
- return /\b(429|500|502|503|504)\b|rate.?limit|timeout|timed out|fetch failed|network|ECONNRESET|ETIMEDOUT/i.test(
- message,
- );
-}
-
-function isTimeoutError(error: unknown): boolean {
- const message = error instanceof Error ? error.message : String(error);
- return /timeout|timed out|ETIMEDOUT/i.test(message);
-}
-
-async function withTransientRetry<T>(
- operation: () => Promise<T>,
- { retryTimeouts = true }: { retryTimeouts?: boolean } = {},
-): Promise<T> {
- for (let attempt = 1; attempt <= TRANSIENT_RETRY_ATTEMPTS; attempt++) {
- try {
- return await operation();
- } catch (error) {
- if (
- attempt === TRANSIENT_RETRY_ATTEMPTS ||
- !isTransientError(error) ||
- (!retryTimeouts && isTimeoutError(error))
- ) {
- throw error;
- }
-
- await new Promise((resolve) =>
- setTimeout(resolve, 50 * attempt + Math.random() * 25),
- );
- }
- }
-
- throw new Error("Partner search operation failed.");
-}
-
-async function withQueryDeadline<T>(operation: () => Promise<T>): Promise<T> {
- let timeoutId: ReturnType<typeof setTimeout> | undefined;
- const timeout = new Promise<never>((_, reject) => {
- timeoutId = setTimeout(
- () =>
- reject(
- new Error(
- `Partner search query timed out after ${QUERY_OPERATION_TIMEOUT_MS}ms.`,
- ),
- ),
- QUERY_OPERATION_TIMEOUT_MS,
- );
- });
-
- try {
- return await Promise.race([
- withTransientRetry(operation, { retryTimeouts: false }),
- timeout,
- ]);
- } finally {
- clearTimeout(timeoutId);
- }
-}
-
function mergeSearchResults(
standardResults: UpstashSearchResult[],
emailResults: UpstashSearchResult[],
@@ -266,14 +174,6 @@ function mergeSearchResults(
);
}
-function logPartnerSearchDebug(details: Record<string, unknown>) {
- if (process.env.PARTNER_SEARCH_DEBUG !== "true") {
- return;
- }
-
- console.log("[Partner Search Debug] Upstash Search", details);
-}
-
export async function deleteUpstashSearchPartnerSearchIndex({
searchIndex,
indexName,
@@ -315,13 +215,17 @@ export function createUpstashSearchPartnerSearchProvider({
};
const queryNgrams = getQueryNgrams(normalizedQuery);
- const [standardResults, rawEmailResults] = await withQueryDeadline(() =>
- Promise.all([
- index.search({ ...searchParams, query: normalizedQuery }),
- queryNgrams
- ? index.search({ ...searchParams, query: queryNgrams })
- : Promise.resolve([]),
- ]),
+ const [standardResults, rawEmailResults] = await withQueryDeadline(
+ () =>
+ Promise.all([
+ index.search({ ...searchParams, query: normalizedQuery }),
+ // Upstash Search takes the n-grams as one query string, unlike Redis
+ // Search which needs each as a separate condition.
+ queryNgrams.length > 0
+ ? index.search({ ...searchParams, query: queryNgrams.join(" ") })
+ : Promise.resolve([]),
+ ]),
+ QUERY_OPERATION_TIMEOUT_MS,
);
const emailResults = rawEmailResults.filter(({ content }) =>
@@ -343,7 +247,7 @@ export function createUpstashSearchPartnerSearchProvider({
score,
}));
- logPartnerSearchDebug({
+ logPartnerSearchDebug("Upstash Search", {
indexName: resolvedIndexName,
operation: "searchCandidates",
query: { programId, query, limit },
diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.ts
index 8b9fe26db0..1e8c11e5dc 100644
--- a/apps/web/scripts/partners/backfill-partner-search.ts
+++ b/apps/web/scripts/partners/backfill-partner-search.ts
@@ -18,11 +18,11 @@
import {
backfillPartnerSearch,
- createUpstashRedisPartnerSearchIndex,
getPartnerSearchProviderName,
reconcilePartnerSearchIndex,
type PartnerSearchBackfillProgress,
} from "@/lib/api/partners/search";
+import { createUpstashRedisPartnerSearchIndex } from "@/lib/api/partners/search/providers/upstash-redis";
import { prisma } from "@/lib/prisma";
import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";
import "dotenv-flow/config";
diff --git a/apps/web/scripts/partners/create-partner-search-index.ts b/apps/web/scripts/partners/create-partner-search-index.ts
index e351d21de4..3a581423bf 100644
--- a/apps/web/scripts/partners/create-partner-search-index.ts
+++ b/apps/web/scripts/partners/create-partner-search-index.ts
@@ -1,7 +1,5 @@
-import {
- createUpstashRedisPartnerSearchIndex,
- getPartnerSearchProviderName,
-} from "@/lib/api/partners/search";
+import { getPartnerSearchProviderName } from "@/lib/api/partners/search";
+import { createUpstashRedisPartnerSearchIndex } from "@/lib/api/partners/search/providers/upstash-redis";
import "dotenv-flow/config";
async function main() {
diff --git a/apps/web/scripts/partners/delete-partner-search-index.ts b/apps/web/scripts/partners/delete-partner-search-index.ts
index 4ee71d2ad7..9c9053b6aa 100644
--- a/apps/web/scripts/partners/delete-partner-search-index.ts
+++ b/apps/web/scripts/partners/delete-partner-search-index.ts
@@ -1,7 +1,5 @@
-import {
- deleteUpstashSearchPartnerSearchIndex,
- getPartnerSearchProviderName,
-} from "@/lib/api/partners/search";
+import { getPartnerSearchProviderName } from "@/lib/api/partners/search";
+import { deleteUpstashSearchPartnerSearchIndex } from "@/lib/api/partners/search/providers/upstash-search";
import { chunk } from "@dub/utils";
import { Redis } from "@upstash/redis";
import "dotenv-flow/config";
diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
index f7674483e2..33b3c7c318 100644
--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
@@ -1,10 +1,12 @@
import {
- createUpstashRedisPartnerSearchIndex,
- createUpstashRedisPartnerSearchProvider,
PARTNER_SEARCH_CANDIDATE_LIMIT,
type PartnerSearchDocument,
- upstashPartnerSearchSchema,
} from "@/lib/api/partners/search";
+import {
+ createUpstashRedisPartnerSearchIndex,
+ createUpstashRedisPartnerSearchProvider,
+ upstashPartnerSearchSchema,
+} from "@/lib/api/partners/search/providers/upstash-redis";
import type { Redis } from "@upstash/redis";
import { beforeEach, describe, expect, it, vi } from "vitest";
diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
index 50be76b5ac..cc44306c3d 100644
--- a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
@@ -1,8 +1,8 @@
+import { type PartnerSearchDocument } from "@/lib/api/partners/search";
import {
createUpstashSearchPartnerSearchProvider,
deleteUpstashSearchPartnerSearchIndex,
- type PartnerSearchDocument,
-} from "@/lib/api/partners/search";
+} from "@/lib/api/partners/search/providers/upstash-search";
import { beforeEach, describe, expect, it, vi } from "vitest";
const document: PartnerSearchDocument = {