The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Remove reconciliation step
Rafi927d8ef9a964
diff --git a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/partners-table.tsx b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/partners-table.tsxindex d4dd3bc35d..f89e3190fc 100644--- a/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/partners-table.tsx+++ b/apps/web/app/app.dub.co/(dashboard)/[slug]/(ee)/program/partners/partners-table.tsx@@ -117,9 +117,9 @@ const partnersColumns = {],};-// Searching must fall back to relevance ordering. A column sort still applies to-// the same relevance-ranked candidates, but it reorders them, so the best match-// lands somewhere down the list instead of first — which reads as broken search.+// Searching sorts by relevance so the closest match to the query is the first+// row. A column sort still applies to the same relevance-ranked candidates, but+// it reorders them, so the best match moves down the list.const PARTNER_SEARCH_RESET_PARAMS = ["sortBy", "sortOrder"];const getPartnerUrl = ({diff --git a/apps/web/lib/api/partners/search/index.ts b/apps/web/lib/api/partners/search/index.tsindex 4b77ca6093..05c9d4faf3 100644--- a/apps/web/lib/api/partners/search/index.ts+++ b/apps/web/lib/api/partners/search/index.ts@@ -3,7 +3,6 @@ export * from "./build-search-query";export * from "./find-candidates";export * from "./order-search-results";export * from "./provider";-export * from "./reconcile";export * from "./searchable-values";export * from "./serialize-document";export * from "./types";diff --git a/apps/web/lib/api/partners/search/provider.ts b/apps/web/lib/api/partners/search/provider.tsindex ee0ae81684..f1659f6049 100644--- a/apps/web/lib/api/partners/search/provider.ts+++ b/apps/web/lib/api/partners/search/provider.ts@@ -2,10 +2,7 @@ import { createTurbopufferPartnerSearchProvider } from "./providers/turbopuffer"import { createUpstashRedisPartnerSearchProvider } from "./providers/upstash-redis";import type { PartnerSearchProvider } from "./types";-const PARTNER_SEARCH_PROVIDER_NAMES = [- "upstash-redis",- "turbopuffer",-] as const;+const PARTNER_SEARCH_PROVIDER_NAMES = ["upstash-redis", "turbopuffer"] as const;export type PartnerSearchProviderName =(typeof PARTNER_SEARCH_PROVIDER_NAMES)[number];diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.tsindex 933744c97f..6655cfa609 100644--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts@@ -72,9 +72,6 @@ interface CreateTurbopufferPartnerSearchProviderOptions {/** The subset of the SDK namespace this provider uses, so tests can fake it. */export interface TurbopufferNamespace {write(params: Record<string, unknown>): Promise<{ rows_affected: number }>;- query(params: Record<string, unknown>): Promise<{- rows?: { id: string | number; $dist?: number }[];- }>;multiQuery(params: Record<string, unknown>): Promise<{results?: { rows?: { id: string | number; $dist?: number }[] }[];}>;@@ -245,30 +242,5 @@ export function createTurbopufferPartnerSearchProvider({);}},-- async listDocumentIds({ cursor, limit }) {- // Keyset pagination over the ID order, the same shape the backfill uses- // against the database. Turbopuffer has no scan cursor of its own.- const { rows } = await withTransientRetry(() =>- resolvedNamespace.query({- rank_by: ["id", "asc"],- top_k: limit,- include_attributes: false,- ...(cursor ? { filters: ["id", "Gt", cursor] } : {}),- }),- );-- const documentIds = (rows ?? []).map(({ id }) => String(id));-- return {- documentIds,- // A short page means the namespace is exhausted; otherwise resume from- // the last ID.- cursor:- documentIds.length < limit- ? null- : documentIds[documentIds.length - 1],- };- },};}diff --git a/apps/web/lib/api/partners/search/providers/upstash-redis.ts b/apps/web/lib/api/partners/search/providers/upstash-redis.tsindex dae4bec770..53f398e4d2 100644--- a/apps/web/lib/api/partners/search/providers/upstash-redis.ts+++ b/apps/web/lib/api/partners/search/providers/upstash-redis.ts@@ -343,22 +343,5 @@ export function createUpstashRedisPartnerSearchProvider({);}},-- async listDocumentIds({ cursor, limit }) {- const documentPrefix = getDocumentPrefix(resolvedIndexName);- // COUNT is a hint, not a page size, so a page can come back empty with- // the cursor still live.- const [nextCursor, keys] = await withTransientRetry(() =>- resolvedRedisClient.scan(cursor ?? "0", {- match: `${documentPrefix}*`,- count: limit,- }),- );-- return {- documentIds: keys.map((key) => key.slice(documentPrefix.length)),- cursor: nextCursor === "0" ? null : String(nextCursor),- };- },};}diff --git a/apps/web/lib/api/partners/search/reconcile.ts b/apps/web/lib/api/partners/search/reconcile.tsdeleted file mode 100644index e2c6591b69..0000000000--- a/apps/web/lib/api/partners/search/reconcile.ts+++ /dev/null@@ -1,82 +0,0 @@-import { prisma } from "@/lib/prisma";-import { getPartnerSearchProvider } from "./provider";-import type { PartnerSearchProvider } from "./types";--const DEFAULT_PAGE_SIZE = 500;--export interface PartnerSearchReconcileProgress {- scanned: number;- deleted: number;- cursor: string | null;-}--interface ReconcilePartnerSearchIndexOptions {- pageSize?: number;- searchProvider?: PartnerSearchProvider | null;- onProgress?: (progress: PartnerSearchReconcileProgress) => void;-}--/**- * Deletes indexed documents whose program enrollment no longer exists.- *- * `backfillPartnerSearch` only upserts, so it cannot remove a document for a- * deleted enrollment — an index that has drifted stays drifted no matter how- * often it is rebuilt. Orphans are not harmless: they still match queries and- * still consume one of the PARTNER_SEARCH_CANDIDATE_LIMIT slots, and because- * the database join then drops them, the user silently sees fewer results than- * the limit with nothing reporting an error.- *- * This sweeps the whole index rather than one program, because neither provider- * can enumerate documents by program: Redis keys are prefixed by index name- * only, and Turbopuffer pages over document IDs. Scoping to a program would- * mean reading every document's body just to discard most of them.- */-export async function reconcilePartnerSearchIndex({- pageSize = DEFAULT_PAGE_SIZE,- searchProvider = getPartnerSearchProvider(),- onProgress,-}: ReconcilePartnerSearchIndexOptions = {}) {- if (!searchProvider) {- throw new Error("Partner search provider is not configured.");- }-- if (!Number.isSafeInteger(pageSize) || pageSize <= 0) {- throw new Error("Page size must be a positive integer.");- }-- let cursor: string | undefined;- let scanned = 0;- let deleted = 0;-- do {- const page = await searchProvider.listDocumentIds({- cursor,- limit: pageSize,- });- scanned += page.documentIds.length;-- if (page.documentIds.length > 0) {- const enrollments = await prisma.programEnrollment.findMany({- where: {- id: { in: page.documentIds },- },- select: { id: true },- });-- const liveIds = new Set(enrollments.map(({ id }) => id));- const staleIds = page.documentIds.filter(- (documentId) => !liveIds.has(documentId),- );-- if (staleIds.length > 0) {- await searchProvider.delete(staleIds);- deleted += staleIds.length;- }- }-- cursor = page.cursor ?? undefined;- onProgress?.({ scanned, deleted, cursor: page.cursor });- } while (cursor);-- return { scanned, deleted };-}diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.tsindex e97190579c..98ce65ef94 100644--- a/apps/web/lib/api/partners/search/types.ts+++ b/apps/web/lib/api/partners/search/types.ts@@ -73,16 +73,6 @@ export interface PartnerSearchCandidateQuery {limit: number;}-export interface PartnerSearchDocumentPage {- documentIds: string[];- /**- * Null once enumeration is exhausted. An empty page with a non-null cursor is- * normal — Redis SCAN returns those routinely — so callers must loop until- * the cursor is null rather than stopping on the first empty page.- */- cursor: string | null;-}-export interface PartnerSearchProvider {searchCandidates(query: PartnerSearchCandidateQuery,@@ -90,9 +80,4 @@ export interface PartnerSearchProvider {waitForIndexing(): Promise<void>;upsert(documents: PartnerSearchDocument[]): Promise<void>;delete(documentIds: string[]): Promise<void>;- /** Walks every indexed document so reconciliation can find orphans. */- listDocumentIds(options: {- cursor?: string;- limit: number;- }): Promise<PartnerSearchDocumentPage>;}diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.tsindex 04ac9e9424..caca3938fd 100644--- a/apps/web/scripts/partners/backfill-partner-search.ts+++ b/apps/web/scripts/partners/backfill-partner-search.ts@@ -5,13 +5,13 @@* a 100K-partner program. A failed run prints the command to resume from the* last indexed document, so a partial backfill is never redone from zero.*- * The backfill only upserts, so it cannot remove documents whose enrollment was- * deleted. --reconcile sweeps up those orphans afterwards — note that sweep- * covers the entire index, not only --programId.+ * The backfill only upserts. Documents whose enrollment was deleted are removed+ * by rebuilding the index wholesale, not incrementally — delete it and backfill+ * again.** cd apps/web* pnpm run script partners/backfill-partner-search --programId=prog_123- * [--batchSize=500] [--after=pge_123] [--reconcile]+ * [--batchSize=500] [--after=pge_123]** Requires PARTNER_SEARCH_PROVIDER to be configured.*/@@ -19,7 +19,6 @@import {backfillPartnerSearch,getPartnerSearchProviderName,- reconcilePartnerSearchIndex,type PartnerSearchBackfillProgress,} from "@/lib/api/partners/search";import { createUpstashRedisPartnerSearchIndex } from "@/lib/api/partners/search/providers/upstash-redis";@@ -34,14 +33,12 @@ interface BackfillArguments {programId: string;batchSize: number;after?: string;- reconcile: boolean;}function parseArguments(args: string[]): BackfillArguments {let programId: string | undefined;let batchSize = DEFAULT_BATCH_SIZE;let after: string | undefined;- let reconcile = false;for (const arg of args) {if (arg.startsWith("--programId=")) {@@ -53,8 +50,6 @@ function parseArguments(args: string[]): BackfillArguments {);} else if (arg.startsWith("--after=")) {after = arg.slice("--after=".length);- } else if (arg === "--reconcile") {- reconcile = true;} else {throw new Error(`Unknown argument: ${arg}`);}@@ -72,7 +67,7 @@ function parseArguments(args: string[]): BackfillArguments {throw new Error("--after cannot be empty.");}- return { programId, batchSize, after, reconcile };+ return { programId, batchSize, after };}let resumeAfter: string | undefined;@@ -105,9 +100,7 @@ function createProgressReporter(totalDocuments: number, batchSize: number) {}async function main() {- const { programId, batchSize, after, reconcile } = parseArguments(- process.argv.slice(2),- );+ const { programId, batchSize, after } = parseArguments(process.argv.slice(2));resumeAfter = after;const providerName = getPartnerSearchProviderName();if (!providerName) {@@ -148,27 +141,6 @@ async function main() {if (result.lastDocumentId) {console.log(`Last document: ${result.lastDocumentId}`);}-- if (!reconcile) {- return;- }-- // Index-wide because neither provider can enumerate documents by program.- console.log("Reconciling the index against the database (all programs)...");-- const { scanned, deleted } = await reconcilePartnerSearchIndex({- onProgress: ({ scanned, deleted }) => {- if (scanned > 0 && scanned % 10_000 === 0) {- console.log(- `Scanned ${scanned.toLocaleString()} documents, removed ${deleted.toLocaleString()} orphans`,- );- }- },- });-- console.log(- `Reconcile complete: scanned ${scanned.toLocaleString()} documents, removed ${deleted.toLocaleString()} orphans.`,- );}main()diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.tsindex b5be43b324..4b6e78b99d 100644--- a/apps/web/tests/partners/get-partners-count-search.test.ts+++ b/apps/web/tests/partners/get-partners-count-search.test.ts@@ -31,9 +31,6 @@ function createSearchProvider(): PartnerSearchProvider {waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),- listDocumentIds: vi- .fn()- .mockResolvedValue({ documentIds: [], cursor: null }),};}diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.tsindex 86a50de135..dc164e0a51 100644--- a/apps/web/tests/partners/get-partners-search.test.ts+++ b/apps/web/tests/partners/get-partners-search.test.ts@@ -40,9 +40,6 @@ function createSearchProvider(waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),- listDocumentIds: vi- .fn()- .mockResolvedValue({ documentIds: [], cursor: null }),};}diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.tsindex 2a397b47bb..732030193c 100644--- a/apps/web/tests/partners/partner-search-backfill.test.ts+++ b/apps/web/tests/partners/partner-search-backfill.test.ts@@ -40,9 +40,6 @@ function createProvider(): PartnerSearchProvider {waitForIndexing: vi.fn(),upsert: vi.fn(),delete: vi.fn(),- listDocumentIds: vi- .fn()- .mockResolvedValue({ documentIds: [], cursor: null }),};}diff --git a/apps/web/tests/partners/partner-search-reconcile.test.ts b/apps/web/tests/partners/partner-search-reconcile.test.tsdeleted file mode 100644index d6ea9b3312..0000000000--- a/apps/web/tests/partners/partner-search-reconcile.test.ts+++ /dev/null@@ -1,147 +0,0 @@-import type { PartnerSearchProvider } from "@/lib/api/partners/search";-import { reconcilePartnerSearchIndex } from "@/lib/api/partners/search";-import { beforeEach, describe, expect, it, vi } from "vitest";--const mocks = vi.hoisted(() => ({- findMany: vi.fn(),-}));--vi.mock("@/lib/prisma", () => ({- sanitizeFullTextSearch: (value: string) => value,- prisma: {- programEnrollment: { findMany: mocks.findMany },- },-}));--function createProvider(- pages: { documentIds: string[]; cursor: string | null }[],-): PartnerSearchProvider {- const listDocumentIds = vi.fn();- for (const page of pages) {- listDocumentIds.mockResolvedValueOnce(page);- }-- return {- searchCandidates: vi.fn(),- waitForIndexing: vi.fn(),- upsert: vi.fn(),- delete: vi.fn().mockResolvedValue(undefined),- listDocumentIds,- };-}--/** Only the enrollment IDs listed here still exist in the database. */-function withLiveEnrollments(liveIds: string[]) {- mocks.findMany.mockImplementation(async ({ where }) =>- where.id.in- .filter((id: string) => liveIds.includes(id))- .map((id: string) => ({ id })),- );-}--describe("reconcilePartnerSearchIndex", () => {- beforeEach(() => {- mocks.findMany.mockReset();- });-- it("deletes documents whose enrollment no longer exists", async () => {- const provider = createProvider([- { documentIds: ["pge_live", "pge_orphan"], cursor: null },- ]);- withLiveEnrollments(["pge_live"]);-- const result = await reconcilePartnerSearchIndex({- searchProvider: provider,- });-- expect(provider.delete).toHaveBeenCalledWith(["pge_orphan"]);- expect(result).toEqual({ scanned: 2, deleted: 1 });- });-- it("leaves a fully consistent index untouched", async () => {- const provider = createProvider([- { documentIds: ["pge_a", "pge_b"], cursor: null },- ]);- withLiveEnrollments(["pge_a", "pge_b"]);-- const result = await reconcilePartnerSearchIndex({- searchProvider: provider,- });-- expect(provider.delete).not.toHaveBeenCalled();- expect(result).toEqual({ scanned: 2, deleted: 0 });- });-- it("keeps paging through empty pages until the cursor is null", async () => {- // Not contrived: Redis SCAN returns these, and stopping on the first one- // would skip the rest of the index.- const provider = createProvider([- { documentIds: [], cursor: "17" },- { documentIds: ["pge_orphan"], cursor: "42" },- { documentIds: [], cursor: null },- ]);- withLiveEnrollments([]);-- const result = await reconcilePartnerSearchIndex({- searchProvider: provider,- pageSize: 2,- });-- expect(provider.listDocumentIds).toHaveBeenCalledTimes(3);- expect(provider.listDocumentIds).toHaveBeenNthCalledWith(1, {- cursor: undefined,- limit: 2,- });- expect(provider.listDocumentIds).toHaveBeenNthCalledWith(2, {- cursor: "17",- limit: 2,- });- expect(result).toEqual({ scanned: 1, deleted: 1 });- });-- it("does not query the database for an empty page", async () => {- const provider = createProvider([{ documentIds: [], cursor: null }]);-- await reconcilePartnerSearchIndex({ searchProvider: provider });-- expect(mocks.findMany).not.toHaveBeenCalled();- expect(provider.delete).not.toHaveBeenCalled();- });-- it("reports progress per page", async () => {- const provider = createProvider([- { documentIds: ["pge_orphan"], cursor: "9" },- { documentIds: ["pge_live"], cursor: null },- ]);- withLiveEnrollments(["pge_live"]);- const onProgress = vi.fn();-- await reconcilePartnerSearchIndex({ searchProvider: provider, onProgress });-- expect(onProgress).toHaveBeenNthCalledWith(1, {- scanned: 1,- deleted: 1,- cursor: "9",- });- expect(onProgress).toHaveBeenNthCalledWith(2, {- scanned: 2,- deleted: 1,- cursor: null,- });- });-- it("requires a configured provider", async () => {- await expect(- reconcilePartnerSearchIndex({ searchProvider: null }),- ).rejects.toThrow("Partner search provider is not configured.");- });-- it("rejects a non-positive page size", async () => {- await expect(- reconcilePartnerSearchIndex({- searchProvider: createProvider([]),- pageSize: 0,- }),- ).rejects.toThrow("Page size must be a positive integer.");- });-});diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.tsindex 475f71abef..312a550d6b 100644--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts@@ -27,7 +27,6 @@ const document: PartnerSearchDocument = {const mocks = vi.hoisted(() => ({write: vi.fn(),- query: vi.fn(),multiQuery: vi.fn(),deleteAll: vi.fn(),}));@@ -35,7 +34,6 @@ const mocks = vi.hoisted(() => ({function createNamespaceMock(): TurbopufferNamespace {return {write: mocks.write,- query: mocks.query,multiQuery: mocks.multiQuery,deleteAll: mocks.deleteAll,} as unknown as TurbopufferNamespace;@@ -55,7 +53,6 @@ describe("Turbopuffer partner search provider", () => {}mocks.write.mockResolvedValue({ rows_affected: 1 });mocks.multiQuery.mockResolvedValue({ results: [] });- mocks.query.mockResolvedValue({ rows: [] });});it("flattens every searchable field into one BM25 attribute", async () => {@@ -220,40 +217,8 @@ describe("Turbopuffer partner search provider", () => {});});- it("enumerates document IDs by keyset over the ID order", async () => {- mocks.query.mockResolvedValue({- rows: [{ id: "pge_1" }, { id: "pge_2" }],- });-- const page = await createProvider().listDocumentIds({ limit: 2 });-- expect(mocks.query).toHaveBeenCalledWith(- expect.objectContaining({ rank_by: ["id", "asc"], top_k: 2 }),- );- // A full page means there may be more, so the cursor is the last ID.- expect(page).toEqual({- documentIds: ["pge_1", "pge_2"],- cursor: "pge_2",- });- });-- it("resumes after the cursor and reports exhaustion on a short page", async () => {- mocks.query.mockResolvedValue({ rows: [{ id: "pge_9" }] });-- const page = await createProvider().listDocumentIds({- cursor: "pge_8",- limit: 2,- });-- expect(mocks.query).toHaveBeenCalledWith(- expect.objectContaining({ filters: ["id", "Gt", "pge_8"] }),- );- expect(page).toEqual({ documentIds: ["pge_9"], cursor: null });- });-it("does not wait for indexing, which turbopuffer does not need", async () => {await expect(createProvider().waitForIndexing()).resolves.toBeUndefined();- expect(mocks.query).not.toHaveBeenCalled();expect(mocks.write).not.toHaveBeenCalled();});diff --git a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.tsindex c4248db554..14ca8cbaf5 100644--- a/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts+++ b/apps/web/tests/partners/upstash-redis-partner-search-provider.test.ts@@ -33,7 +33,6 @@ const mocks = vi.hoisted(() => ({index: vi.fn(),jsonMset: vi.fn(),query: vi.fn(),- scan: vi.fn(),waitIndexing: vi.fn(),}));@@ -49,7 +48,6 @@ function createRedisMock(): Redis {const redisMock = {del: mocks.del,- scan: mocks.scan,json: {mset: mocks.jsonMset,},@@ -362,41 +360,6 @@ describe("Upstash Redis partner search provider", () => {).rejects.toThrow("Delete and recreate it before backfilling");});- it("enumerates document IDs by stripping the key prefix", async () => {- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });- mocks.scan.mockResolvedValue([- "42",- ["test-index:partner:pge_one", "test-index:partner:pge_two"],- ]);-- const page = await provider.listDocumentIds({ limit: 500 });-- expect(mocks.scan).toHaveBeenCalledWith("0", {- match: "test-index:partner:*",- count: 500,- });- expect(page).toEqual({- documentIds: ["pge_one", "pge_two"],- cursor: "42",- });- });-- it("reports an exhausted SCAN cursor as null", async () => {- const provider = createUpstashRedisPartnerSearchProvider({- redisClient: createRedisMock(),- indexName: "test-index",- });- mocks.scan.mockResolvedValue(["0", []]);-- await expect(- provider.listDocumentIds({ cursor: "42", limit: 10 }),- ).resolves.toEqual({ documentIds: [], cursor: null });- expect(mocks.scan).toHaveBeenCalledWith("42", expect.anything());- });-it.each([["blank", ""],["whitespace", " "],@@ -407,13 +370,11 @@ describe("Upstash Redis partner search provider", () => {const provider = createUpstashRedisPartnerSearchProvider({redisClient: createRedisMock(),});- mocks.scan.mockResolvedValue(["0", []]);- await provider.listDocumentIds({ limit: 10 });+ await provider.upsert([document]);- expect(mocks.scan).toHaveBeenCalledWith(- "0",- expect.objectContaining({ match: "partner-search-v1:partner:*" }),+ expect(mocks.jsonMset).toHaveBeenCalledWith(+ expect.objectContaining({ key: "partner-search-v1:partner:pge_test" }),);vi.unstubAllEnvs();},