Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Upstash redis optimization

Rafie48d8211dc82
diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
index 9d27f6aed0..86fe07ca11 100644
--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts
+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
@@ -21,12 +21,16 @@ import type {
} from "../types";
const DEFAULT_INDEX_NAME = "partner-search-v1";
-const NULL_VALUE = "__none__";
+// Sentinel value for nullable keyword fields. Redis Search keyword fields
+// cannot store null, so we use a value that will never appear in real data
+// to represent absent values and map it back to null in query results.
+const NULL_VALUE = "\0__null:f47ac10b-58cc-4372-a567-0e02b2c3d479__";
const MAX_GROUPS = 1_000;
const TRANSIENT_RETRY_ATTEMPTS = 2;
const QUERY_REQUEST_TIMEOUT_MS = 400;
const WRITE_REQUEST_TIMEOUT_MS = 10_000;
const WRITE_BATCH_SIZE = 100;
+const WAIT_FOR_INDEXING_TIMEOUT_MS = 30_000;
const DOCUMENT_TYPE_PARTNER = "partner";
const DOCUMENT_TYPE_TAG = "tag";
@@ -182,9 +186,7 @@ function getQueryNgrams(query: string): string[] {
);
}
-function escapeRegex(value: string): string {
- return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
+
function serializeUpstashDocument(
document: PartnerSearchDocument,
@@ -251,14 +253,14 @@ function buildListFilter(
}
function buildTextFilter(query: string): UpstashPartnerSearchFilter {
+ // $smart handles word-boundary and prefix matching via the inverted index.
+ // Email n-grams handle infix/substring matching (e.g. "examp" in "[email protected]").
+ // A $regex path was intentionally omitted because regex queries scan the
+ // inverted index linearly, which degrades p99 latency at 100K+ documents.
const alternatives: UpstashPartnerSearchFilter[] = [
{ searchText: { $smart: query } },
];
- if (!/\s/u.test(query)) {
- alternatives.push({ searchText: { $regex: `${escapeRegex(query)}.*` } });
- }
-
const emailNgrams = getQueryNgrams(query);
if (emailNgrams.length > 0) {
alternatives.push({
@@ -499,14 +501,40 @@ async function upsertDocumentBatch(
documents,
storedDocuments,
);
+ const upsertEntries = getUpsertEntries(indexName, documents);
- if (staleTagDocumentKeys.length > 0) {
- await withTransientRetry(() => redisClient.del(...staleTagDocumentKeys));
- }
+ // Pipeline the stale tag cleanup and document upsert into a single
+ // atomic call so a crash between the two cannot leave orphaned state.
+ await withTransientRetry(async () => {
+ const pipeline = redisClient.pipeline();
- await withTransientRetry(() =>
- redisClient.json.mset(...getUpsertEntries(indexName, documents)),
- );
+ if (staleTagDocumentKeys.length > 0) {
+ pipeline.del(...staleTagDocumentKeys);
+ }
+
+ pipeline.json.mset(...upsertEntries);
+ await pipeline.exec();
+ });
+}
+
+async function scanTagKeys(
+ redisClient: Redis,
+ indexName: string,
+ documentId: string,
+): Promise<string[]> {
+ const pattern = `${getDocumentPrefix(indexName)}tag:${documentId}:*`;
+ const keys: string[] = [];
+ let cursor = 0;
+
+ do {
+ const [nextCursor, batch] = await withTransientRetry(() =>
+ redisClient.scan(cursor, { match: pattern, count: 100 }),
+ );
+ cursor = nextCursor;
+ keys.push(...batch);
+ } while (cursor !== 0);
+
+ return keys;
}
async function deleteDocumentBatch(
@@ -514,23 +542,22 @@ async function deleteDocumentBatch(
indexName: string,
documentIds: string[],
) {
- const storedDocuments = await getStoredDocuments(
- redisClient,
- indexName,
- documentIds,
- );
- const tagDocumentKeys = storedDocuments.flatMap((document, index) =>
- (document?.partnerTagIdsRaw ?? []).map((partnerTagId) =>
- getTagDocumentKey(indexName, documentIds[index]!, partnerTagId),
- ),
+ // Discover tag shadow keys via SCAN instead of reading document contents,
+ // saving a JSON.MGET round-trip per batch.
+ const tagKeys = (
+ await Promise.all(
+ documentIds.map((id) => scanTagKeys(redisClient, indexName, id)),
+ )
+ ).flat();
+
+ const partnerKeys = documentIds.map((id) =>
+ getDocumentKey(indexName, id),
);
- await withTransientRetry(() =>
- redisClient.del(
- ...documentIds.map((documentId) => getDocumentKey(indexName, documentId)),
- ...tagDocumentKeys,
- ),
- );
+ const allKeys = [...partnerKeys, ...tagKeys];
+ if (allKeys.length > 0) {
+ await withTransientRetry(() => redisClient.del(...allKeys));
+ }
}
export async function createUpstashRedisPartnerSearchIndex({
@@ -638,7 +665,16 @@ export function createUpstashRedisPartnerSearchProvider({
}),
);
- return result.groups.buckets.flatMap(({ key, docCount }) => {
+ const { buckets, sumOtherDocCount } = result.groups;
+
+ if (buckets.length >= MAX_GROUPS) {
+ console.warn(
+ `[Partner Search] groupBy("${field}") returned ${buckets.length} buckets (limit: ${MAX_GROUPS}). ` +
+ `Results may be truncated (${sumOtherDocCount ?? "unknown"} docs in unlisted groups).`,
+ );
+ }
+
+ return buckets.flatMap(({ key, docCount }) => {
const value = mapGroupValue(key);
if (field === "referredByPartnerId" && value === null) {
return [];
@@ -648,7 +684,23 @@ export function createUpstashRedisPartnerSearchProvider({
},
async waitForIndexing() {
- const result = await withTransientRetry(() => writeIndex.waitIndexing());
+ const timeout = new Promise<never>((_, reject) =>
+ setTimeout(
+ () =>
+ reject(
+ new Error(
+ `Partner search waitForIndexing timed out after ${WAIT_FOR_INDEXING_TIMEOUT_MS}ms.`,
+ ),
+ ),
+ WAIT_FOR_INDEXING_TIMEOUT_MS,
+ ),
+ );
+
+ const result = await Promise.race([
+ withTransientRetry(() => writeIndex.waitIndexing()),
+ timeout,
+ ]);
+
if (result === 0) {
throw new Error(
`Partner search index ${resolvedIndexName} was not found.`,
diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
index 617f608345..35b0fe7694 100644
--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
@@ -53,6 +53,7 @@ const mocks = vi.hoisted(() => ({
jsonMget: vi.fn(),
jsonMset: vi.fn(),
query: vi.fn(),
+ scan: vi.fn(),
waitIndexing: vi.fn(),
}));
@@ -68,8 +69,9 @@ function createRedisMock(): Redis {
mocks.index.mockReturnValue(searchIndex);
mocks.createIndex.mockReturnValue(searchIndex);
- return {
+ const redisMock = {
del: mocks.del,
+ scan: mocks.scan,
json: {
mget: mocks.jsonMget,
mset: mocks.jsonMset,
@@ -78,7 +80,27 @@ function createRedisMock(): Redis {
createIndex: mocks.createIndex,
index: mocks.index,
},
+ pipeline() {
+ const commands: Array<() => void> = [];
+ return {
+ del: (...args: unknown[]) => {
+ commands.push(() => mocks.del(...args));
+ },
+ json: {
+ mset: (...args: unknown[]) => {
+ commands.push(() => mocks.jsonMset(...args));
+ },
+ },
+ async exec() {
+ for (const cmd of commands) {
+ await cmd();
+ }
+ },
+ };
+ },
} as unknown as Redis;
+
+ return redisMock;
}
describe("Upstash Redis partner search provider", () => {
@@ -98,6 +120,7 @@ describe("Upstash Redis partner search provider", () => {
mocks.jsonMget.mockResolvedValue([]);
mocks.jsonMset.mockResolvedValue("OK");
mocks.query.mockResolvedValue([]);
+ mocks.scan.mockResolvedValue([0, []]);
mocks.waitIndexing.mockResolvedValue(1);
});
@@ -222,7 +245,7 @@ describe("Upstash Redis partner search provider", () => {
documentType: "partner",
partnerTagIds: "ptag_test",
partnerTagIdsRaw: ["ptag_test"],
- tenantId: "__none__",
+ tenantId: expect.stringContaining("__null:"),
}),
}),
);
@@ -282,8 +305,12 @@ describe("Upstash Redis partner search provider", () => {
});
it("deletes the partner document and its tag shadow documents", async () => {
- mocks.jsonMget.mockResolvedValue([
- [{ partnerTagIdsRaw: ["ptag_one", "ptag_two"] }],
+ mocks.scan.mockResolvedValue([
+ 0,
+ [
+ "test-index:tag:pge_test:ptag_one",
+ "test-index:tag:pge_test:ptag_two",
+ ],
]);
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
@@ -292,6 +319,10 @@ describe("Upstash Redis partner search provider", () => {
await provider.delete([document.id]);
+ expect(mocks.scan).toHaveBeenCalledWith(0, {
+ match: "test-index:tag:pge_test:*",
+ count: 100,
+ });
expect(mocks.del).toHaveBeenCalledWith(
"test-index:partner:pge_test",
"test-index:tag:pge_test:ptag_one",