The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Use Redis transactions and document reads for safe cleanup
Rafi988b8b52340f
diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.tsindex 86fe07ca11..fc09963c1a 100644--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts@@ -186,8 +186,6 @@ function getQueryNgrams(query: string): string[] {);}--function serializeUpstashDocument(document: PartnerSearchDocument,documentType: typeof DOCUMENT_TYPE_PARTNER | typeof DOCUMENT_TYPE_TAG,@@ -298,10 +296,10 @@ function buildUpstashFilter("groupId" | "country" | "partnerTagIds",PartnerSearchListFilter | undefined,][] = [- ["groupId", filters?.groupIds],- ["country", filters?.countries],- ["partnerTagIds", filters?.partnerTagIds],- ];+ ["groupId", filters?.groupIds],+ ["country", filters?.countries],+ ["partnerTagIds", filters?.partnerTagIds],+ ];for (const [field, listFilter] of listFilters) {if (!listFilter) {@@ -503,61 +501,44 @@ async function upsertDocumentBatch();const upsertEntries = getUpsertEntries(indexName, documents);- // Pipeline the stale tag cleanup and document upsert into a single- // atomic call so a crash between the two cannot leave orphaned state.+ // Keep stale tag cleanup and document upserts in one transaction so another+ // synchronization cannot modify the same keys between these operationsawait withTransientRetry(async () => {- const pipeline = redisClient.pipeline();+ const transaction = redisClient.multi();if (staleTagDocumentKeys.length > 0) {- pipeline.del(...staleTagDocumentKeys);+ transaction.del(...staleTagDocumentKeys);}- pipeline.json.mset(...upsertEntries);- await pipeline.exec();+ transaction.json.mset(...upsertEntries);+ await transaction.exec();});}-async function scanTagKeys(- redisClient: Redis,- indexName: string,- documentId: string,-): Promise<string[]> {- const pattern = `${getDocumentPrefix(indexName)}tag:${documentId}:*`;- const keys: string[] = [];- let cursor = 0;-- do {- const [nextCursor, batch] = await withTransientRetry(() =>- redisClient.scan(cursor, { match: pattern, count: 100 }),- );- cursor = nextCursor;- keys.push(...batch);- } while (cursor !== 0);-- return keys;-}-async function deleteDocumentBatch(redisClient: Redis,indexName: string,documentIds: string[],) {- // Discover tag shadow keys via SCAN instead of reading document contents,- // saving a JSON.MGET round-trip per batch.- const tagKeys = (- await Promise.all(- documentIds.map((id) => scanTagKeys(redisClient, indexName, id)),- )- ).flat();-- const partnerKeys = documentIds.map((id) =>- getDocumentKey(indexName, id),+ // Read the batch once and derive exact tag keys rather than scanning the+ // entire Redis keyspace for each partner+ const storedDocuments = await getStoredDocuments(+ redisClient,+ indexName,+ documentIds,+ );+ const tagDocumentKeys = storedDocuments.flatMap((document, index) =>+ (document?.partnerTagIdsRaw ?? []).map((partnerTagId) =>+ getTagDocumentKey(indexName, documentIds[index]!, partnerTagId),+ ),);- const allKeys = [...partnerKeys, ...tagKeys];- if (allKeys.length > 0) {- await withTransientRetry(() => redisClient.del(...allKeys));- }+ await withTransientRetry(() =>+ redisClient.del(+ ...documentIds.map((documentId) => getDocumentKey(indexName, documentId)),+ ...tagDocumentKeys,+ ),+ );}export async function createUpstashRedisPartnerSearchIndex({@@ -617,8 +598,8 @@ export function createUpstashRedisPartnerSearchProvider({const offset = (query.page - 1) * query.pageSize;const orderBy = query.sort? ({- [query.sort.field]: query.sort.order.toUpperCase(),- } as Record<string, "ASC" | "DESC">)+ [query.sort.field]: query.sort.order.toUpperCase(),+ } as Record<string, "ASC" | "DESC">): undefined;const results = await withTransientRetry(() =>@@ -684,8 +665,9 @@ export function createUpstashRedisPartnerSearchProvider({},async waitForIndexing() {- const timeout = new Promise<never>((_, reject) =>- setTimeout(+ let timeoutId: ReturnType<typeof setTimeout> | undefined;+ const timeout = new Promise<never>((_, reject) => {+ timeoutId = setTimeout(() =>reject(new Error(@@ -693,13 +675,18 @@ export function createUpstashRedisPartnerSearchProvider({),),WAIT_FOR_INDEXING_TIMEOUT_MS,- ),- );+ );+ });- const result = await Promise.race([- withTransientRetry(() => writeIndex.waitIndexing()),- timeout,- ]);+ let result: number;+ try {+ result = await Promise.race([+ withTransientRetry(() => writeIndex.waitIndexing()),+ timeout,+ ]);+ } finally {+ clearTimeout(timeoutId);+ }if (result === 0) {throw new Error(diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.tsindex 35b0fe7694..766b00dd85 100644--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts@@ -52,8 +52,8 @@ const mocks = vi.hoisted(() => ({index: vi.fn(),jsonMget: vi.fn(),jsonMset: vi.fn(),+ multi: vi.fn(),query: vi.fn(),- scan: vi.fn(),waitIndexing: vi.fn(),}));@@ -71,7 +71,6 @@ function createRedisMock(): Redis {const redisMock = {del: mocks.del,- scan: mocks.scan,json: {mget: mocks.jsonMget,mset: mocks.jsonMset,@@ -80,7 +79,8 @@ function createRedisMock(): Redis {createIndex: mocks.createIndex,index: mocks.index,},- pipeline() {+ multi() {+ mocks.multi();const commands: Array<() => void> = [];return {del: (...args: unknown[]) => {@@ -120,7 +120,6 @@ describe("Upstash Redis partner search provider", () => {mocks.jsonMget.mockResolvedValue([]);mocks.jsonMset.mockResolvedValue("OK");mocks.query.mockResolvedValue([]);- mocks.scan.mockResolvedValue([0, []]);mocks.waitIndexing.mockResolvedValue(1);});@@ -276,6 +275,7 @@ describe("Upstash Redis partner search provider", () => {await provider.upsert([document]);+ expect(mocks.multi).toHaveBeenCalledOnce();expect(mocks.del).toHaveBeenCalledWith("test-index:tag:pge_test:ptag_old");expect(mocks.del.mock.invocationCallOrder[0]).toBeLessThan(mocks.jsonMset.mock.invocationCallOrder[0]!,@@ -305,12 +305,8 @@ describe("Upstash Redis partner search provider", () => {});it("deletes the partner document and its tag shadow documents", async () => {- mocks.scan.mockResolvedValue([- 0,- [- "test-index:tag:pge_test:ptag_one",- "test-index:tag:pge_test:ptag_two",- ],+ mocks.jsonMget.mockResolvedValue([+ [{ partnerTagIdsRaw: ["ptag_one", "ptag_two"] }],]);const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),@@ -319,10 +315,10 @@ describe("Upstash Redis partner search provider", () => {await provider.delete([document.id]);- expect(mocks.scan).toHaveBeenCalledWith(0, {- match: "test-index:tag:pge_test:*",- count: 100,- });+ expect(mocks.jsonMget).toHaveBeenCalledWith(+ ["test-index:partner:pge_test"],+ "$",+ );expect(mocks.del).toHaveBeenCalledWith("test-index:partner:pge_test","test-index:tag:pge_test:ptag_one",@@ -357,14 +353,20 @@ describe("Upstash Redis partner search provider", () => {});it("waits for pending index updates", async () => {+ vi.useFakeTimers();const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),indexName: "test-index",});- await provider.waitForIndexing();+ try {+ await provider.waitForIndexing();- expect(mocks.waitIndexing).toHaveBeenCalledOnce();+ expect(mocks.waitIndexing).toHaveBeenCalledOnce();+ expect(vi.getTimerCount()).toBe(0);+ } finally {+ vi.useRealTimers();+ }});it("creates the index explicitly with the partner-search prefix", async () => {