Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Added reconciliation to backfilling

Rafi4f098deb32d4
diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.ts
index bf35146f04..6d600987ab 100644
--- a/apps/web/lib/api/partners/search/index.ts
+++ b/apps/web/lib/api/partners/search/index.ts
@@ -4,6 +4,7 @@ export * from "./order-search-results";
export * from "./provider";
export * from "./providers/upstash-redis";
export * from "./providers/upstash-search";
+export * from "./reconcile";
export * from "./searchable-values";
export * from "./serialize-document";
export * from "./sync-documents";
diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
index 4cb925b003..792583a0c7 100644
--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts
+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts
@@ -459,5 +459,22 @@ export function createUpstashRedisPartnerSearchProvider({
);
}
},
+
+ async listDocumentIds({ cursor, limit }) {
+ const documentPrefix = getDocumentPrefix(resolvedIndexName);
+ // COUNT is a hint, not a page size, so a page can come back empty with
+ // the cursor still live.
+ const [nextCursor, keys] = await withTransientRetry(() =>
+ resolvedRedisClient.scan(cursor ?? "0", {
+ match: `${documentPrefix}*`,
+ count: limit,
+ }),
+ );
+
+ return {
+ documentIds: keys.map((key) => key.slice(documentPrefix.length)),
+ cursor: nextCursor === "0" ? null : String(nextCursor),
+ };
+ },
};
}
diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.ts
index 0fac4baae5..6409315810 100644
--- a/apps/web/lib/api/partners/search/providers/upstash-search.ts
+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts
@@ -61,6 +61,10 @@ interface UpstashSearchIndexClient {
}[],
): Promise<string>;
delete(documentIds: string[]): Promise<{ deleted: number }>;
+ range(params: {
+ cursor: string;
+ limit: number;
+ }): Promise<{ nextCursor: string; documents: { id: string }[] }>;
info(): Promise<{ pendingDocumentCount: number; documentCount: number }>;
deleteIndex(): Promise<string>;
}
@@ -380,5 +384,18 @@ export function createUpstashSearchPartnerSearchProvider({
await withTransientRetry(() => index.delete(documentIdBatch));
}
},
+
+ async listDocumentIds({ cursor, limit }) {
+ const { nextCursor, documents } = await withTransientRetry(() =>
+ index.range({ cursor: cursor ?? "0", limit }),
+ );
+
+ // An empty nextCursor means the range is exhausted. The SDK types it as a
+ // plain string, so the sentinel is not visible from the signature.
+ return {
+ documentIds: documents.map(({ id }) => id),
+ cursor: nextCursor === "" ? null : nextCursor,
+ };
+ },
};
}
diff --git a/apps/web/lib/api/partners/search/reconcile.ts b/apps/web/lib/api/partners/search/reconcile.ts
new file mode 100644
index 0000000000..e0c6897112
--- /dev/null
+++ b/apps/web/lib/api/partners/search/reconcile.ts
@@ -0,0 +1,82 @@
+import { prisma } from "@/lib/prisma";
+import { getPartnerSearchProvider } from "./provider";
+import type { PartnerSearchProvider } from "./types";
+
+const DEFAULT_PAGE_SIZE = 500;
+
+export interface PartnerSearchReconcileProgress {
+ scanned: number;
+ deleted: number;
+ cursor: string | null;
+}
+
+interface ReconcilePartnerSearchIndexOptions {
+ pageSize?: number;
+ searchProvider?: PartnerSearchProvider | null;
+ onProgress?: (progress: PartnerSearchReconcileProgress) => void;
+}
+
+/**
+ * Deletes indexed documents whose program enrollment no longer exists.
+ *
+ * `backfillPartnerSearch` only upserts, so it cannot remove a document for a
+ * deleted enrollment — an index that has drifted stays drifted no matter how
+ * often it is rebuilt. Orphans are not harmless: they still match queries and
+ * still consume one of the PARTNER_SEARCH_CANDIDATE_LIMIT slots, and because
+ * the database join then drops them, the user silently sees fewer results than
+ * the limit with nothing reporting an error.
+ *
+ * This sweeps the whole index rather than one program, because neither provider
+ * can enumerate documents by program: Redis keys are prefixed by index name
+ * only, and Upstash Search ranges over document IDs. Scoping to a program would
+ * mean reading every document's body just to discard most of them.
+ */
+export async function reconcilePartnerSearchIndex({
+ pageSize = DEFAULT_PAGE_SIZE,
+ searchProvider = getPartnerSearchProvider(),
+ onProgress,
+}: ReconcilePartnerSearchIndexOptions = {}) {
+ if (!searchProvider) {
+ throw new Error("Partner search provider is not configured.");
+ }
+
+ if (!Number.isSafeInteger(pageSize) || pageSize <= 0) {
+ throw new Error("Page size must be a positive integer.");
+ }
+
+ let cursor: string | undefined;
+ let scanned = 0;
+ let deleted = 0;
+
+ do {
+ const page = await searchProvider.listDocumentIds({
+ cursor,
+ limit: pageSize,
+ });
+ scanned += page.documentIds.length;
+
+ if (page.documentIds.length > 0) {
+ const enrollments = await prisma.programEnrollment.findMany({
+ where: {
+ id: { in: page.documentIds },
+ },
+ select: { id: true },
+ });
+
+ const liveIds = new Set(enrollments.map(({ id }) => id));
+ const staleIds = page.documentIds.filter(
+ (documentId) => !liveIds.has(documentId),
+ );
+
+ if (staleIds.length > 0) {
+ await searchProvider.delete(staleIds);
+ deleted += staleIds.length;
+ }
+ }
+
+ cursor = page.cursor ?? undefined;
+ onProgress?.({ scanned, deleted, cursor: page.cursor });
+ } while (cursor);
+
+ return { scanned, deleted };
+}
diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.ts
index 2ed88da1e2..077d666725 100644
--- a/apps/web/lib/api/partners/search/types.ts
+++ b/apps/web/lib/api/partners/search/types.ts
@@ -55,6 +55,16 @@ export interface PartnerSearchCandidateQuery {
limit: number;
}
+export interface PartnerSearchDocumentPage {
+ documentIds: string[];
+ /**
+ * Null once enumeration is exhausted. An empty page with a non-null cursor is
+ * normal — Redis SCAN returns those routinely — so callers must loop until
+ * the cursor is null rather than stopping on the first empty page.
+ */
+ cursor: string | null;
+}
+
export interface PartnerSearchProvider {
searchCandidates(
query: PartnerSearchCandidateQuery,
@@ -62,4 +72,9 @@ export interface PartnerSearchProvider {
waitForIndexing(): Promise<void>;
upsert(documents: PartnerSearchDocument[]): Promise<void>;
delete(documentIds: string[]): Promise<void>;
+ /** Walks every indexed document so reconciliation can find orphans. */
+ listDocumentIds(options: {
+ cursor?: string;
+ limit: number;
+ }): Promise<PartnerSearchDocumentPage>;
}
diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.ts
index 5a06a9c216..84952dd69c 100644
--- a/apps/web/scripts/partners/backfill-partner-search.ts
+++ b/apps/web/scripts/partners/backfill-partner-search.ts
@@ -2,6 +2,7 @@ import {
backfillPartnerSearch,
createUpstashRedisPartnerSearchIndex,
getPartnerSearchProviderName,
+ reconcilePartnerSearchIndex,
type PartnerSearchBackfillProgress,
} from "@/lib/api/partners/search";
import { prisma } from "@/lib/prisma";
@@ -15,12 +16,14 @@ interface BackfillArguments {
programId: string;
batchSize: number;
after?: string;
+ reconcile: boolean;
}
function parseArguments(args: string[]): BackfillArguments {
let programId: string | undefined;
let batchSize = DEFAULT_BATCH_SIZE;
let after: string | undefined;
+ let reconcile = false;
for (const arg of args) {
if (arg.startsWith("--programId=")) {
@@ -32,6 +35,8 @@ function parseArguments(args: string[]): BackfillArguments {
);
} else if (arg.startsWith("--after=")) {
after = arg.slice("--after=".length);
+ } else if (arg === "--reconcile") {
+ reconcile = true;
} else {
throw new Error(`Unknown argument: ${arg}`);
}
@@ -49,7 +54,7 @@ function parseArguments(args: string[]): BackfillArguments {
throw new Error("--after cannot be empty.");
}
- return { programId, batchSize, after };
+ return { programId, batchSize, after, reconcile };
}
let resumeAfter: string | undefined;
@@ -66,7 +71,9 @@ function reportProgress({
}
async function main() {
- const { programId, batchSize, after } = parseArguments(process.argv.slice(2));
+ const { programId, batchSize, after, reconcile } = parseArguments(
+ process.argv.slice(2),
+ );
resumeAfter = after;
const providerName = getPartnerSearchProviderName();
if (!providerName) {
@@ -98,6 +105,29 @@ async function main() {
if (result.lastDocumentId) {
console.log(`Last document: ${result.lastDocumentId}`);
}
+
+ if (!reconcile) {
+ return;
+ }
+
+ // The backfill above only upserts, so it cannot remove documents for
+ // enrollments that were deleted. This sweep covers the whole index, not just
+ // --programId, because neither provider can enumerate documents by program.
+ console.log("Reconciling the index against the database (all programs)...");
+
+ const { scanned, deleted } = await reconcilePartnerSearchIndex({
+ onProgress: ({ scanned, deleted }) => {
+ if (scanned > 0 && scanned % 10_000 === 0) {
+ console.log(
+ `Scanned ${scanned.toLocaleString()} documents, removed ${deleted.toLocaleString()} orphans`,
+ );
+ }
+ },
+ });
+
+ console.log(
+ `Reconcile complete: scanned ${scanned.toLocaleString()} documents, removed ${deleted.toLocaleString()} orphans.`,
+ );
}
main()
diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.ts
index 31344499d9..1af823cb3c 100644
--- a/apps/web/tests/partners/get-partners-count-search.test.ts
+++ b/apps/web/tests/partners/get-partners-count-search.test.ts
@@ -31,6 +31,9 @@ function createSearchProvider(): PartnerSearchProvider {
waitForIndexing: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
+ listDocumentIds: vi
+ .fn()
+ .mockResolvedValue({ documentIds: [], cursor: null }),
};
}
diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.ts
index d6f8d3aa2f..db3205cf5c 100644
--- a/apps/web/tests/partners/get-partners-search.test.ts
+++ b/apps/web/tests/partners/get-partners-search.test.ts
@@ -40,6 +40,9 @@ function createSearchProvider(
waitForIndexing: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
+ listDocumentIds: vi
+ .fn()
+ .mockResolvedValue({ documentIds: [], cursor: null }),
};
}
diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.ts
index 732030193c..2a397b47bb 100644
--- a/apps/web/tests/partners/partner-search-backfill.test.ts
+++ b/apps/web/tests/partners/partner-search-backfill.test.ts
@@ -40,6 +40,9 @@ function createProvider(): PartnerSearchProvider {
waitForIndexing: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
+ listDocumentIds: vi
+ .fn()
+ .mockResolvedValue({ documentIds: [], cursor: null }),
};
}
diff --git a/apps/web/tests/partners/partner-search-query.test.ts b/apps/web/tests/partners/partner-search-query.test.ts
index 51d7f13c31..c295ed7c79 100644
--- a/apps/web/tests/partners/partner-search-query.test.ts
+++ b/apps/web/tests/partners/partner-search-query.test.ts
@@ -12,9 +12,7 @@ const defaultInput = {
describe("buildPartnerSearchCandidateQuery", () => {
it("builds a provider-neutral relevance candidate request", () => {
- expect(
- buildPartnerSearchCandidateQuery(defaultInput),
- ).toEqual({
+ expect(buildPartnerSearchCandidateQuery(defaultInput)).toEqual({
programId: "prog_test",
query: "examp",
limit: 100,
diff --git a/apps/web/tests/partners/partner-search-reconcile.test.ts b/apps/web/tests/partners/partner-search-reconcile.test.ts
new file mode 100644
index 0000000000..d6ea9b3312
--- /dev/null
+++ b/apps/web/tests/partners/partner-search-reconcile.test.ts
@@ -0,0 +1,147 @@
+import type { PartnerSearchProvider } from "@/lib/api/partners/search";
+import { reconcilePartnerSearchIndex } from "@/lib/api/partners/search";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ findMany: vi.fn(),
+}));
+
+vi.mock("@/lib/prisma", () => ({
+ sanitizeFullTextSearch: (value: string) => value,
+ prisma: {
+ programEnrollment: { findMany: mocks.findMany },
+ },
+}));
+
+function createProvider(
+ pages: { documentIds: string[]; cursor: string | null }[],
+): PartnerSearchProvider {
+ const listDocumentIds = vi.fn();
+ for (const page of pages) {
+ listDocumentIds.mockResolvedValueOnce(page);
+ }
+
+ return {
+ searchCandidates: vi.fn(),
+ waitForIndexing: vi.fn(),
+ upsert: vi.fn(),
+ delete: vi.fn().mockResolvedValue(undefined),
+ listDocumentIds,
+ };
+}
+
+/** Only the enrollment IDs listed here still exist in the database. */
+function withLiveEnrollments(liveIds: string[]) {
+ mocks.findMany.mockImplementation(async ({ where }) =>
+ where.id.in
+ .filter((id: string) => liveIds.includes(id))
+ .map((id: string) => ({ id })),
+ );
+}
+
+describe("reconcilePartnerSearchIndex", () => {
+ beforeEach(() => {
+ mocks.findMany.mockReset();
+ });
+
+ it("deletes documents whose enrollment no longer exists", async () => {
+ const provider = createProvider([
+ { documentIds: ["pge_live", "pge_orphan"], cursor: null },
+ ]);
+ withLiveEnrollments(["pge_live"]);
+
+ const result = await reconcilePartnerSearchIndex({
+ searchProvider: provider,
+ });
+
+ expect(provider.delete).toHaveBeenCalledWith(["pge_orphan"]);
+ expect(result).toEqual({ scanned: 2, deleted: 1 });
+ });
+
+ it("leaves a fully consistent index untouched", async () => {
+ const provider = createProvider([
+ { documentIds: ["pge_a", "pge_b"], cursor: null },
+ ]);
+ withLiveEnrollments(["pge_a", "pge_b"]);
+
+ const result = await reconcilePartnerSearchIndex({
+ searchProvider: provider,
+ });
+
+ expect(provider.delete).not.toHaveBeenCalled();
+ expect(result).toEqual({ scanned: 2, deleted: 0 });
+ });
+
+ it("keeps paging through empty pages until the cursor is null", async () => {
+ // Not contrived: Redis SCAN returns these, and stopping on the first one
+ // would skip the rest of the index.
+ const provider = createProvider([
+ { documentIds: [], cursor: "17" },
+ { documentIds: ["pge_orphan"], cursor: "42" },
+ { documentIds: [], cursor: null },
+ ]);
+ withLiveEnrollments([]);
+
+ const result = await reconcilePartnerSearchIndex({
+ searchProvider: provider,
+ pageSize: 2,
+ });
+
+ expect(provider.listDocumentIds).toHaveBeenCalledTimes(3);
+ expect(provider.listDocumentIds).toHaveBeenNthCalledWith(1, {
+ cursor: undefined,
+ limit: 2,
+ });
+ expect(provider.listDocumentIds).toHaveBeenNthCalledWith(2, {
+ cursor: "17",
+ limit: 2,
+ });
+ expect(result).toEqual({ scanned: 1, deleted: 1 });
+ });
+
+ it("does not query the database for an empty page", async () => {
+ const provider = createProvider([{ documentIds: [], cursor: null }]);
+
+ await reconcilePartnerSearchIndex({ searchProvider: provider });
+
+ expect(mocks.findMany).not.toHaveBeenCalled();
+ expect(provider.delete).not.toHaveBeenCalled();
+ });
+
+ it("reports progress per page", async () => {
+ const provider = createProvider([
+ { documentIds: ["pge_orphan"], cursor: "9" },
+ { documentIds: ["pge_live"], cursor: null },
+ ]);
+ withLiveEnrollments(["pge_live"]);
+ const onProgress = vi.fn();
+
+ await reconcilePartnerSearchIndex({ searchProvider: provider, onProgress });
+
+ expect(onProgress).toHaveBeenNthCalledWith(1, {
+ scanned: 1,
+ deleted: 1,
+ cursor: "9",
+ });
+ expect(onProgress).toHaveBeenNthCalledWith(2, {
+ scanned: 2,
+ deleted: 1,
+ cursor: null,
+ });
+ });
+
+ it("requires a configured provider", async () => {
+ await expect(
+ reconcilePartnerSearchIndex({ searchProvider: null }),
+ ).rejects.toThrow("Partner search provider is not configured.");
+ });
+
+ it("rejects a non-positive page size", async () => {
+ await expect(
+ reconcilePartnerSearchIndex({
+ searchProvider: createProvider([]),
+ pageSize: 0,
+ }),
+ ).rejects.toThrow("Page size must be a positive integer.");
+ });
+});
diff --git a/apps/web/tests/partners/partner-search-sync.test.ts b/apps/web/tests/partners/partner-search-sync.test.ts
index c59be90482..3e8ad2ac01 100644
--- a/apps/web/tests/partners/partner-search-sync.test.ts
+++ b/apps/web/tests/partners/partner-search-sync.test.ts
@@ -43,6 +43,9 @@ function createProvider(): PartnerSearchProvider {
waitForIndexing: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
+ listDocumentIds: vi
+ .fn()
+ .mockResolvedValue({ documentIds: [], cursor: null }),
};
}
diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
index 66fb0f0123..8d8763f71e 100644
--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts
@@ -31,6 +31,7 @@ const mocks = vi.hoisted(() => ({
index: vi.fn(),
jsonMset: vi.fn(),
query: vi.fn(),
+ scan: vi.fn(),
waitIndexing: vi.fn(),
}));
@@ -46,6 +47,7 @@ function createRedisMock(): Redis {
const redisMock = {
del: mocks.del,
+ scan: mocks.scan,
json: {
mset: mocks.jsonMset,
},
@@ -358,4 +360,39 @@ describe("Upstash Redis partner search provider", () => {
}),
).rejects.toThrow("Delete and recreate it before backfilling");
});
+
+ it("enumerates document IDs by stripping the key prefix", async () => {
+ const provider = createUpstashRedisPartnerSearchProvider({
+ redisClient: createRedisMock(),
+ indexName: "test-index",
+ });
+ mocks.scan.mockResolvedValue([
+ "42",
+ ["test-index:partner:pge_one", "test-index:partner:pge_two"],
+ ]);
+
+ const page = await provider.listDocumentIds({ limit: 500 });
+
+ expect(mocks.scan).toHaveBeenCalledWith("0", {
+ match: "test-index:partner:*",
+ count: 500,
+ });
+ expect(page).toEqual({
+ documentIds: ["pge_one", "pge_two"],
+ cursor: "42",
+ });
+ });
+
+ it("reports an exhausted SCAN cursor as null", async () => {
+ const provider = createUpstashRedisPartnerSearchProvider({
+ redisClient: createRedisMock(),
+ indexName: "test-index",
+ });
+ mocks.scan.mockResolvedValue(["0", []]);
+
+ await expect(
+ provider.listDocumentIds({ cursor: "42", limit: 10 }),
+ ).resolves.toEqual({ documentIds: [], cursor: null });
+ expect(mocks.scan).toHaveBeenCalledWith("42", expect.anything());
+ });
});
diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
index 553ba6fca7..0d5141ffd4 100644
--- a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
@@ -25,6 +25,7 @@ const mocks = vi.hoisted(() => ({
delete: vi.fn(),
deleteIndex: vi.fn(),
info: vi.fn(),
+ range: vi.fn(),
search: vi.fn(),
upsert: vi.fn(),
}));
@@ -34,6 +35,7 @@ function createSearchIndexMock() {
delete: mocks.delete,
deleteIndex: mocks.deleteIndex,
info: mocks.info,
+ range: mocks.range,
search: mocks.search,
upsert: mocks.upsert,
};
@@ -219,4 +221,39 @@ describe("Upstash Search partner search provider", () => {
});
expect(mocks.deleteIndex).toHaveBeenCalledOnce();
});
+
+ it("enumerates document IDs and reports an empty cursor as null", async () => {
+ const provider = createUpstashSearchPartnerSearchProvider({
+ searchIndex: createSearchIndexMock(),
+ indexName: "test-index",
+ });
+ mocks.range.mockResolvedValue({
+ nextCursor: "",
+ documents: [{ id: "pge_one" }, { id: "pge_two" }],
+ });
+
+ const page = await provider.listDocumentIds({ limit: 500 });
+
+ expect(mocks.range).toHaveBeenCalledWith({ cursor: "0", limit: 500 });
+ expect(page).toEqual({
+ documentIds: ["pge_one", "pge_two"],
+ cursor: null,
+ });
+ });
+
+ it("passes a live cursor through unchanged", async () => {
+ const provider = createUpstashSearchPartnerSearchProvider({
+ searchIndex: createSearchIndexMock(),
+ indexName: "test-index",
+ });
+ mocks.range.mockResolvedValue({
+ nextCursor: "next",
+ documents: [{ id: "pge_one" }],
+ });
+
+ await expect(
+ provider.listDocumentIds({ cursor: "prev", limit: 10 }),
+ ).resolves.toEqual({ documentIds: ["pge_one"], cursor: "next" });
+ expect(mocks.range).toHaveBeenCalledWith({ cursor: "prev", limit: 10 });
+ });
});