Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Use Redis transactions and document reads for safe cleanup

Rafi988b8b52340f
diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
index 86fe07ca11..fc09963c1a 100644
--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts
+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
@@ -186,8 +186,6 @@ function getQueryNgrams(query: string): string[] {
);
}
-
-
function serializeUpstashDocument(
document: PartnerSearchDocument,
documentType: typeof DOCUMENT_TYPE_PARTNER | typeof DOCUMENT_TYPE_TAG,
@@ -298,10 +296,10 @@ function buildUpstashFilter(
"groupId" | "country" | "partnerTagIds",
PartnerSearchListFilter | undefined,
][] = [
- ["groupId", filters?.groupIds],
- ["country", filters?.countries],
- ["partnerTagIds", filters?.partnerTagIds],
- ];
+ ["groupId", filters?.groupIds],
+ ["country", filters?.countries],
+ ["partnerTagIds", filters?.partnerTagIds],
+ ];
for (const [field, listFilter] of listFilters) {
if (!listFilter) {
@@ -503,61 +501,44 @@ async function upsertDocumentBatch(
);
const upsertEntries = getUpsertEntries(indexName, documents);
- // Pipeline the stale tag cleanup and document upsert into a single
- // atomic call so a crash between the two cannot leave orphaned state.
+ // Keep stale tag cleanup and document upserts in one transaction so another
+ // synchronization cannot modify the same keys between these operations
await withTransientRetry(async () => {
- const pipeline = redisClient.pipeline();
+ const transaction = redisClient.multi();
if (staleTagDocumentKeys.length > 0) {
- pipeline.del(...staleTagDocumentKeys);
+ transaction.del(...staleTagDocumentKeys);
}
- pipeline.json.mset(...upsertEntries);
- await pipeline.exec();
+ transaction.json.mset(...upsertEntries);
+ await transaction.exec();
});
}
-async function scanTagKeys(
- redisClient: Redis,
- indexName: string,
- documentId: string,
-): Promise<string[]> {
- const pattern = `${getDocumentPrefix(indexName)}tag:${documentId}:*`;
- const keys: string[] = [];
- let cursor = 0;
-
- do {
- const [nextCursor, batch] = await withTransientRetry(() =>
- redisClient.scan(cursor, { match: pattern, count: 100 }),
- );
- cursor = nextCursor;
- keys.push(...batch);
- } while (cursor !== 0);
-
- return keys;
-}
-
async function deleteDocumentBatch(
redisClient: Redis,
indexName: string,
documentIds: string[],
) {
- // Discover tag shadow keys via SCAN instead of reading document contents,
- // saving a JSON.MGET round-trip per batch.
- const tagKeys = (
- await Promise.all(
- documentIds.map((id) => scanTagKeys(redisClient, indexName, id)),
- )
- ).flat();
-
- const partnerKeys = documentIds.map((id) =>
- getDocumentKey(indexName, id),
+ // Read the batch once and derive exact tag keys rather than scanning the
+ // entire Redis keyspace for each partner
+ const storedDocuments = await getStoredDocuments(
+ redisClient,
+ indexName,
+ documentIds,
+ );
+ const tagDocumentKeys = storedDocuments.flatMap((document, index) =>
+ (document?.partnerTagIdsRaw ?? []).map((partnerTagId) =>
+ getTagDocumentKey(indexName, documentIds[index]!, partnerTagId),
+ ),
);
- const allKeys = [...partnerKeys, ...tagKeys];
- if (allKeys.length > 0) {
- await withTransientRetry(() => redisClient.del(...allKeys));
- }
+ await withTransientRetry(() =>
+ redisClient.del(
+ ...documentIds.map((documentId) => getDocumentKey(indexName, documentId)),
+ ...tagDocumentKeys,
+ ),
+ );
}
export async function createUpstashRedisPartnerSearchIndex({
@@ -617,8 +598,8 @@ export function createUpstashRedisPartnerSearchProvider({
const offset = (query.page - 1) * query.pageSize;
const orderBy = query.sort
? ({
- [query.sort.field]: query.sort.order.toUpperCase(),
- } as Record<string, "ASC" | "DESC">)
+ [query.sort.field]: query.sort.order.toUpperCase(),
+ } as Record<string, "ASC" | "DESC">)
: undefined;
const results = await withTransientRetry(() =>
@@ -684,8 +665,9 @@ export function createUpstashRedisPartnerSearchProvider({
},
async waitForIndexing() {
- const timeout = new Promise<never>((_, reject) =>
- setTimeout(
+ let timeoutId: ReturnType<typeof setTimeout> | undefined;
+ const timeout = new Promise<never>((_, reject) => {
+ timeoutId = setTimeout(
() =>
reject(
new Error(
@@ -693,13 +675,18 @@ export function createUpstashRedisPartnerSearchProvider({
),
),
WAIT_FOR_INDEXING_TIMEOUT_MS,
- ),
- );
+ );
+ });
- const result = await Promise.race([
- withTransientRetry(() => writeIndex.waitIndexing()),
- timeout,
- ]);
+ let result: number;
+ try {
+ result = await Promise.race([
+ withTransientRetry(() => writeIndex.waitIndexing()),
+ timeout,
+ ]);
+ } finally {
+ clearTimeout(timeoutId);
+ }
if (result === 0) {
throw new Error(
diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
index 35b0fe7694..766b00dd85 100644
--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
@@ -52,8 +52,8 @@ const mocks = vi.hoisted(() => ({
index: vi.fn(),
jsonMget: vi.fn(),
jsonMset: vi.fn(),
+ multi: vi.fn(),
query: vi.fn(),
- scan: vi.fn(),
waitIndexing: vi.fn(),
}));
@@ -71,7 +71,6 @@ function createRedisMock(): Redis {
const redisMock = {
del: mocks.del,
- scan: mocks.scan,
json: {
mget: mocks.jsonMget,
mset: mocks.jsonMset,
@@ -80,7 +79,8 @@ function createRedisMock(): Redis {
createIndex: mocks.createIndex,
index: mocks.index,
},
- pipeline() {
+ multi() {
+ mocks.multi();
const commands: Array<() => void> = [];
return {
del: (...args: unknown[]) => {
@@ -120,7 +120,6 @@ describe("Upstash Redis partner search provider", () => {
mocks.jsonMget.mockResolvedValue([]);
mocks.jsonMset.mockResolvedValue("OK");
mocks.query.mockResolvedValue([]);
- mocks.scan.mockResolvedValue([0, []]);
mocks.waitIndexing.mockResolvedValue(1);
});
@@ -276,6 +275,7 @@ describe("Upstash Redis partner search provider", () => {
await provider.upsert([document]);
+ expect(mocks.multi).toHaveBeenCalledOnce();
expect(mocks.del).toHaveBeenCalledWith("test-index:tag:pge_test:ptag_old");
expect(mocks.del.mock.invocationCallOrder[0]).toBeLessThan(
mocks.jsonMset.mock.invocationCallOrder[0]!,
@@ -305,12 +305,8 @@ describe("Upstash Redis partner search provider", () => {
});
it("deletes the partner document and its tag shadow documents", async () => {
- mocks.scan.mockResolvedValue([
- 0,
- [
- "test-index:tag:pge_test:ptag_one",
- "test-index:tag:pge_test:ptag_two",
- ],
+ mocks.jsonMget.mockResolvedValue([
+ [{ partnerTagIdsRaw: ["ptag_one", "ptag_two"] }],
]);
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
@@ -319,10 +315,10 @@ describe("Upstash Redis partner search provider", () => {
await provider.delete([document.id]);
- expect(mocks.scan).toHaveBeenCalledWith(0, {
- match: "test-index:tag:pge_test:*",
- count: 100,
- });
+ expect(mocks.jsonMget).toHaveBeenCalledWith(
+ ["test-index:partner:pge_test"],
+ "$",
+ );
expect(mocks.del).toHaveBeenCalledWith(
"test-index:partner:pge_test",
"test-index:tag:pge_test:ptag_one",
@@ -357,14 +353,20 @@ describe("Upstash Redis partner search provider", () => {
});
it("waits for pending index updates", async () => {
+ vi.useFakeTimers();
const provider = createUpstashRedisPartnerSearchProvider({
redisClient: createRedisMock(),
indexName: "test-index",
});
- await provider.waitForIndexing();
+ try {
+ await provider.waitForIndexing();
- expect(mocks.waitIndexing).toHaveBeenCalledOnce();
+ expect(mocks.waitIndexing).toHaveBeenCalledOnce();
+ expect(vi.getTimerCount()).toBe(0);
+ } finally {
+ vi.useRealTimers();
+ }
});
it("creates the index explicitly with the partner-search prefix", async () => {