The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Added reconciliation to backfilling
Rafi4f098deb32d4
diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.tsindex bf35146f04..6d600987ab 100644--- a/apps/web/lib/api/partners/search/index.ts+++ b/apps/web/lib/api/partners/search/index.ts@@ -4,6 +4,7 @@ export * from "./order-search-results";export * from "./provider";export * from "./providers/upstash-redis";export * from "./providers/upstash-search";+export * from "./reconcile";export * from "./searchable-values";export * from "./serialize-document";export * from "./sync-documents";diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.tsindex 4cb925b003..792583a0c7 100644--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts@@ -459,5 +459,22 @@ export function createUpstashRedisPartnerSearchProvider({);}},++ async listDocumentIds({ cursor, limit }) {+ const documentPrefix = getDocumentPrefix(resolvedIndexName);+ // COUNT is a hint, not a page size, so a page can come back empty with+ // the cursor still live.+ const [nextCursor, keys] = await withTransientRetry(() =>+ resolvedRedisClient.scan(cursor ?? "0", {+ match: `${documentPrefix}*`,+ count: limit,+ }),+ );++ return {+ documentIds: keys.map((key) => key.slice(documentPrefix.length)),+ cursor: nextCursor === "0" ? null : String(nextCursor),+ };+ },};}diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.tsindex 0fac4baae5..6409315810 100644--- a/apps/web/lib/api/partners/search/providers/upstash-search.ts+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts@@ -61,6 +61,10 @@ interface UpstashSearchIndexClient {}[],): Promise<string>;delete(documentIds: string[]): Promise<{ deleted: number }>;+ range(params: {+ cursor: string;+ limit: number;+ }): Promise<{ nextCursor: string; documents: { id: string }[] }>;info(): Promise<{ pendingDocumentCount: number; documentCount: number }>;deleteIndex(): Promise<string>;}@@ -380,5 +384,18 @@ export function createUpstashSearchPartnerSearchProvider({await withTransientRetry(() => index.delete(documentIdBatch));}},++ async listDocumentIds({ cursor, limit }) {+ const { nextCursor, documents } = await withTransientRetry(() =>+ index.range({ cursor: cursor ?? "0", limit }),+ );++ // An empty nextCursor means the range is exhausted. The SDK types it as a+ // plain string, so the sentinel is not visible from the signature.+ return {+ documentIds: documents.map(({ id }) => id),+ cursor: nextCursor === "" ? null : nextCursor,+ };+ },};}diff --git a/apps/web/lib/api/partners/search/reconcile.ts b/apps/web/lib/api/partners/search/reconcile.tsnew file mode 100644index 0000000000..e0c6897112--- /dev/null+++ b/apps/web/lib/api/partners/search/reconcile.ts@@ -0,0 +1,82 @@+import { prisma } from "@/lib/prisma";+import { getPartnerSearchProvider } from "./provider";+import type { PartnerSearchProvider } from "./types";++const DEFAULT_PAGE_SIZE = 500;++export interface PartnerSearchReconcileProgress {+ scanned: number;+ deleted: number;+ cursor: string | null;+}++interface ReconcilePartnerSearchIndexOptions {+ pageSize?: number;+ searchProvider?: PartnerSearchProvider | null;+ onProgress?: (progress: PartnerSearchReconcileProgress) => void;+}++/**+ * Deletes indexed documents whose program enrollment no longer exists.+ *+ * `backfillPartnerSearch` only upserts, so it cannot remove a document for a+ * deleted enrollment — an index that has drifted stays drifted no matter how+ * often it is rebuilt. Orphans are not harmless: they still match queries and+ * still consume one of the PARTNER_SEARCH_CANDIDATE_LIMIT slots, and because+ * the database join then drops them, the user silently sees fewer results than+ * the limit with nothing reporting an error.+ *+ * This sweeps the whole index rather than one program, because neither provider+ * can enumerate documents by program: Redis keys are prefixed by index name+ * only, and Upstash Search ranges over document IDs. Scoping to a program would+ * mean reading every document's body just to discard most of them.+ */+export async function reconcilePartnerSearchIndex({+ pageSize = DEFAULT_PAGE_SIZE,+ searchProvider = getPartnerSearchProvider(),+ onProgress,+}: ReconcilePartnerSearchIndexOptions = {}) {+ if (!searchProvider) {+ throw new Error("Partner search provider is not configured.");+ }++ if (!Number.isSafeInteger(pageSize) || pageSize <= 0) {+ throw new Error("Page size must be a positive integer.");+ }++ let cursor: string | undefined;+ let scanned = 0;+ let deleted = 0;++ do {+ const page = await searchProvider.listDocumentIds({+ cursor,+ limit: pageSize,+ });+ scanned += page.documentIds.length;++ if (page.documentIds.length > 0) {+ const enrollments = await prisma.programEnrollment.findMany({+ where: {+ id: { in: page.documentIds },+ },+ select: { id: true },+ });++ const liveIds = new Set(enrollments.map(({ id }) => id));+ const staleIds = page.documentIds.filter(+ (documentId) => !liveIds.has(documentId),+ );++ if (staleIds.length > 0) {+ await searchProvider.delete(staleIds);+ deleted += staleIds.length;+ }+ }++ cursor = page.cursor ?? undefined;+ onProgress?.({ scanned, deleted, cursor: page.cursor });+ } while (cursor);++ return { scanned, deleted };+}diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.tsindex 2ed88da1e2..077d666725 100644--- a/apps/web/lib/api/partners/search/types.ts+++ b/apps/web/lib/api/partners/search/types.ts@@ -55,6 +55,16 @@ export interface PartnerSearchCandidateQuery {limit: number;}+export interface PartnerSearchDocumentPage {+ documentIds: string[];+ /**+ * Null once enumeration is exhausted. An empty page with a non-null cursor is+ * normal — Redis SCAN returns those routinely — so callers must loop until+ * the cursor is null rather than stopping on the first empty page.+ */+ cursor: string | null;+}+export interface PartnerSearchProvider {searchCandidates(query: PartnerSearchCandidateQuery,@@ -62,4 +72,9 @@ export interface PartnerSearchProvider {waitForIndexing(): Promise<void>;upsert(documents: PartnerSearchDocument[]): Promise<void>;delete(documentIds: string[]): Promise<void>;+ /** Walks every indexed document so reconciliation can find orphans. */+ listDocumentIds(options: {+ cursor?: string;+ limit: number;+ }): Promise<PartnerSearchDocumentPage>;}diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.tsindex 5a06a9c216..84952dd69c 100644--- a/apps/web/scripts/partners/backfill-partner-search.ts+++ b/apps/web/scripts/partners/backfill-partner-search.ts@@ -2,6 +2,7 @@ import {backfillPartnerSearch,createUpstashRedisPartnerSearchIndex,getPartnerSearchProviderName,+ reconcilePartnerSearchIndex,type PartnerSearchBackfillProgress,} from "@/lib/api/partners/search";import { prisma } from "@/lib/prisma";@@ -15,12 +16,14 @@ interface BackfillArguments {programId: string;batchSize: number;after?: string;+ reconcile: boolean;}function parseArguments(args: string[]): BackfillArguments {let programId: string | undefined;let batchSize = DEFAULT_BATCH_SIZE;let after: string | undefined;+ let reconcile = false;for (const arg of args) {if (arg.startsWith("--programId=")) {@@ -32,6 +35,8 @@ function parseArguments(args: string[]): BackfillArguments {);} else if (arg.startsWith("--after=")) {after = arg.slice("--after=".length);+ } else if (arg === "--reconcile") {+ reconcile = true;} else {throw new Error(`Unknown argument: ${arg}`);}@@ -49,7 +54,7 @@ function parseArguments(args: string[]): BackfillArguments {throw new Error("--after cannot be empty.");}- return { programId, batchSize, after };+ return { programId, batchSize, after, reconcile };}let resumeAfter: string | undefined;@@ -66,7 +71,9 @@ function reportProgress({}async function main() {- const { programId, batchSize, after } = parseArguments(process.argv.slice(2));+ const { programId, batchSize, after, reconcile } = parseArguments(+ process.argv.slice(2),+ );resumeAfter = after;const providerName = getPartnerSearchProviderName();if (!providerName) {@@ -98,6 +105,29 @@ async function main() {if (result.lastDocumentId) {console.log(`Last document: ${result.lastDocumentId}`);}++ if (!reconcile) {+ return;+ }++ // The backfill above only upserts, so it cannot remove documents for+ // enrollments that were deleted. This sweep covers the whole index, not just+ // --programId, because neither provider can enumerate documents by program.+ console.log("Reconciling the index against the database (all programs)...");++ const { scanned, deleted } = await reconcilePartnerSearchIndex({+ onProgress: ({ scanned, deleted }) => {+ if (scanned > 0 && scanned % 10_000 === 0) {+ console.log(+ `Scanned ${scanned.toLocaleString()} documents, removed ${deleted.toLocaleString()} orphans`,+ );+ }+ },+ });++ console.log(+ `Reconcile complete: scanned ${scanned.toLocaleString()} documents, removed ${deleted.toLocaleString()} orphans.`,+ );}main()diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.tsindex 31344499d9..1af823cb3c 100644--- a/apps/web/tests/partners/get-partners-count-search.test.ts+++ b/apps/web/tests/partners/get-partners-count-search.test.ts@@ -31,6 +31,9 @@ function createSearchProvider(): PartnerSearchProvider {waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),+ listDocumentIds: vi+ .fn()+ .mockResolvedValue({ documentIds: [], cursor: null }),};}diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.tsindex d6f8d3aa2f..db3205cf5c 100644--- a/apps/web/tests/partners/get-partners-search.test.ts+++ b/apps/web/tests/partners/get-partners-search.test.ts@@ -40,6 +40,9 @@ function createSearchProvider(waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),+ listDocumentIds: vi+ .fn()+ .mockResolvedValue({ documentIds: [], cursor: null }),};}diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.tsindex 732030193c..2a397b47bb 100644--- a/apps/web/tests/partners/partner-search-backfill.test.ts+++ b/apps/web/tests/partners/partner-search-backfill.test.ts@@ -40,6 +40,9 @@ function createProvider(): PartnerSearchProvider {waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),+ listDocumentIds: vi+ .fn()+ .mockResolvedValue({ documentIds: [], cursor: null }),};}diff --git a/apps/web/tests/partners/partner-search-query.test.ts b/apps/web/tests/partners/partner-search-query.test.tsindex 51d7f13c31..c295ed7c79 100644--- a/apps/web/tests/partners/partner-search-query.test.ts+++ b/apps/web/tests/partners/partner-search-query.test.ts@@ -12,9 +12,7 @@ const defaultInput = {describe("buildPartnerSearchCandidateQuery", () => {it("builds a provider-neutral relevance candidate request", () => {- expect(- buildPartnerSearchCandidateQuery(defaultInput),- ).toEqual({+ expect(buildPartnerSearchCandidateQuery(defaultInput)).toEqual({programId: "prog_test",query: "examp",limit: 100,diff --git a/apps/web/tests/partners/partner-search-reconcile.test.ts b/apps/web/tests/partners/partner-search-reconcile.test.tsnew file mode 100644index 0000000000..d6ea9b3312--- /dev/null+++ b/apps/web/tests/partners/partner-search-reconcile.test.ts@@ -0,0 +1,147 @@+import type { PartnerSearchProvider } from "@/lib/api/partners/search";+import { reconcilePartnerSearchIndex } from "@/lib/api/partners/search";+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ findMany: vi.fn(),+}));++vi.mock("@/lib/prisma", () => ({+ sanitizeFullTextSearch: (value: string) => value,+ prisma: {+ programEnrollment: { findMany: mocks.findMany },+ },+}));++function createProvider(+ pages: { documentIds: string[]; cursor: string | null }[],+): PartnerSearchProvider {+ const listDocumentIds = vi.fn();+ for (const page of pages) {+ listDocumentIds.mockResolvedValueOnce(page);+ }++ return {+ searchCandidates: vi.fn(),+ waitForIndexing: vi.fn(),+ upsert: vi.fn(),+ delete: vi.fn().mockResolvedValue(undefined),+ listDocumentIds,+ };+}++/** Only the enrollment IDs listed here still exist in the database. */+function withLiveEnrollments(liveIds: string[]) {+ mocks.findMany.mockImplementation(async ({ where }) =>+ where.id.in+ .filter((id: string) => liveIds.includes(id))+ .map((id: string) => ({ id })),+ );+}++describe("reconcilePartnerSearchIndex", () => {+ beforeEach(() => {+ mocks.findMany.mockReset();+ });++ it("deletes documents whose enrollment no longer exists", async () => {+ const provider = createProvider([+ { documentIds: ["pge_live", "pge_orphan"], cursor: null },+ ]);+ withLiveEnrollments(["pge_live"]);++ const result = await reconcilePartnerSearchIndex({+ searchProvider: provider,+ });++ expect(provider.delete).toHaveBeenCalledWith(["pge_orphan"]);+ expect(result).toEqual({ scanned: 2, deleted: 1 });+ });++ it("leaves a fully consistent index untouched", async () => {+ const provider = createProvider([+ { documentIds: ["pge_a", "pge_b"], cursor: null },+ ]);+ withLiveEnrollments(["pge_a", "pge_b"]);++ const result = await reconcilePartnerSearchIndex({+ searchProvider: provider,+ });++ expect(provider.delete).not.toHaveBeenCalled();+ expect(result).toEqual({ scanned: 2, deleted: 0 });+ });++ it("keeps paging through empty pages until the cursor is null", async () => {+ // Not contrived: Redis SCAN returns these, and stopping on the first one+ // would skip the rest of the index.+ const provider = createProvider([+ { documentIds: [], cursor: "17" },+ { documentIds: ["pge_orphan"], cursor: "42" },+ { documentIds: [], cursor: null },+ ]);+ withLiveEnrollments([]);++ const result = await reconcilePartnerSearchIndex({+ searchProvider: provider,+ pageSize: 2,+ });++ expect(provider.listDocumentIds).toHaveBeenCalledTimes(3);+ expect(provider.listDocumentIds).toHaveBeenNthCalledWith(1, {+ cursor: undefined,+ limit: 2,+ });+ expect(provider.listDocumentIds).toHaveBeenNthCalledWith(2, {+ cursor: "17",+ limit: 2,+ });+ expect(result).toEqual({ scanned: 1, deleted: 1 });+ });++ it("does not query the database for an empty page", async () => {+ const provider = createProvider([{ documentIds: [], cursor: null }]);++ await reconcilePartnerSearchIndex({ searchProvider: provider });++ expect(mocks.findMany).not.toHaveBeenCalled();+ expect(provider.delete).not.toHaveBeenCalled();+ });++ it("reports progress per page", async () => {+ const provider = createProvider([+ { documentIds: ["pge_orphan"], cursor: "9" },+ { documentIds: ["pge_live"], cursor: null },+ ]);+ withLiveEnrollments(["pge_live"]);+ const onProgress = vi.fn();++ await reconcilePartnerSearchIndex({ searchProvider: provider, onProgress });++ expect(onProgress).toHaveBeenNthCalledWith(1, {+ scanned: 1,+ deleted: 1,+ cursor: "9",+ });+ expect(onProgress).toHaveBeenNthCalledWith(2, {+ scanned: 2,+ deleted: 1,+ cursor: null,+ });+ });++ it("requires a configured provider", async () => {+ await expect(+ reconcilePartnerSearchIndex({ searchProvider: null }),+ ).rejects.toThrow("Partner search provider is not configured.");+ });++ it("rejects a non-positive page size", async () => {+ await expect(+ reconcilePartnerSearchIndex({+ searchProvider: createProvider([]),+ pageSize: 0,+ }),+ ).rejects.toThrow("Page size must be a positive integer.");+ });+});diff --git a/apps/web/tests/partners/partner-search-sync.test.ts b/apps/web/tests/partners/partner-search-sync.test.tsindex c59be90482..3e8ad2ac01 100644--- a/apps/web/tests/partners/partner-search-sync.test.ts+++ b/apps/web/tests/partners/partner-search-sync.test.ts@@ -43,6 +43,9 @@ function createProvider(): PartnerSearchProvider {waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),+ listDocumentIds: vi+ .fn()+ .mockResolvedValue({ documentIds: [], cursor: null }),};}diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.tsindex 66fb0f0123..8d8763f71e 100644--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts@@ -31,6 +31,7 @@ const mocks = vi.hoisted(() => ({index: vi.fn(),jsonMset: vi.fn(),query: vi.fn(),+ scan: vi.fn(),waitIndexing: vi.fn(),}));@@ -46,6 +47,7 @@ function createRedisMock(): Redis {const redisMock = {del: mocks.del,+ scan: mocks.scan,json: {mset: mocks.jsonMset,},@@ -358,4 +360,39 @@ describe("Upstash Redis partner search provider", () => {}),).rejects.toThrow("Delete and recreate it before backfilling");});++ it("enumerates document IDs by stripping the key prefix", async () => {+ const provider = createUpstashRedisPartnerSearchProvider({+ redisClient: createRedisMock(),+ indexName: "test-index",+ });+ mocks.scan.mockResolvedValue([+ "42",+ ["test-index:partner:pge_one", "test-index:partner:pge_two"],+ ]);++ const page = await provider.listDocumentIds({ limit: 500 });++ expect(mocks.scan).toHaveBeenCalledWith("0", {+ match: "test-index:partner:*",+ count: 500,+ });+ expect(page).toEqual({+ documentIds: ["pge_one", "pge_two"],+ cursor: "42",+ });+ });++ it("reports an exhausted SCAN cursor as null", async () => {+ const provider = createUpstashRedisPartnerSearchProvider({+ redisClient: createRedisMock(),+ indexName: "test-index",+ });+ mocks.scan.mockResolvedValue(["0", []]);++ await expect(+ provider.listDocumentIds({ cursor: "42", limit: 10 }),+ ).resolves.toEqual({ documentIds: [], cursor: null });+ expect(mocks.scan).toHaveBeenCalledWith("42", expect.anything());+ });});diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.tsindex 553ba6fca7..0d5141ffd4 100644--- a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts@@ -25,6 +25,7 @@ const mocks = vi.hoisted(() => ({delete: vi.fn(),deleteIndex: vi.fn(),info: vi.fn(),+ range: vi.fn(),search: vi.fn(),upsert: vi.fn(),}));@@ -34,6 +35,7 @@ function createSearchIndexMock() {delete: mocks.delete,deleteIndex: mocks.deleteIndex,info: mocks.info,+ range: mocks.range,search: mocks.search,upsert: mocks.upsert,};@@ -219,4 +221,39 @@ describe("Upstash Search partner search provider", () => {});expect(mocks.deleteIndex).toHaveBeenCalledOnce();});++ it("enumerates document IDs and reports an empty cursor as null", async () => {+ const provider = createUpstashSearchPartnerSearchProvider({+ searchIndex: createSearchIndexMock(),+ indexName: "test-index",+ });+ mocks.range.mockResolvedValue({+ nextCursor: "",+ documents: [{ id: "pge_one" }, { id: "pge_two" }],+ });++ const page = await provider.listDocumentIds({ limit: 500 });++ expect(mocks.range).toHaveBeenCalledWith({ cursor: "0", limit: 500 });+ expect(page).toEqual({+ documentIds: ["pge_one", "pge_two"],+ cursor: null,+ });+ });++ it("passes a live cursor through unchanged", async () => {+ const provider = createUpstashSearchPartnerSearchProvider({+ searchIndex: createSearchIndexMock(),+ indexName: "test-index",+ });+ mocks.range.mockResolvedValue({+ nextCursor: "next",+ documents: [{ id: "pge_one" }],+ });++ await expect(+ provider.listDocumentIds({ cursor: "prev", limit: 10 }),+ ).resolves.toEqual({ documentIds: ["pge_one"], cursor: "next" });+ expect(mocks.range).toHaveBeenCalledWith({ cursor: "prev", limit: 10 });+ });});