The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Count is an aggregate filter now
Rafi3bb753b2819b
diff --git a/apps/web/lib/api/partners/get-partners-count.ts b/apps/web/lib/api/partners/get-partners-count.tsindex 03a9d4546f..cc6304cb31 100644--- a/apps/web/lib/api/partners/get-partners-count.ts+++ b/apps/web/lib/api/partners/get-partners-count.ts@@ -260,6 +260,32 @@ export async function getPartnersCount<T>(})) as T;}+ // Counting the candidate IDs reports a floor, because the provider truncated+ // them to the candidate ceiling — "999 partners" when there are twelve+ // thousand. Ask the provider for the real total instead, but only when it can+ // see every filter that applies: `partnerIds`, `tenantId`, and the metric+ // ranges are database-only, so leaving them out would over-count.+ const databaseOnlyFilters =+ Boolean(tenantId) ||+ Boolean(partnerIds?.length) ||+ Object.keys(buildMetricRangeWhere(enrollmentBase)).length > 0;++ if (+ searchProvider &&+ candidateQuery &&+ candidateResult &&+ !databaseOnlyFilters+ ) {+ try {+ return (await searchProvider.countCandidates(candidateQuery)) as T;+ } catch (error) {+ console.error(+ "[Partner Search] Count aggregation failed, falling back to counting candidates.",+ error,+ );+ }+ }+// Get absolute count of partnersconst count = await prisma.programEnrollment.count({where: {diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.tsindex 15b2a03a17..53a48e0f1a 100644--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts@@ -67,9 +67,9 @@ const SEARCH_TEXT_TOKENIZER = "word_v2";// marking them filterable would buy nothing and cost plenty: turbopuffer bills// an attribute per enabled index, so FTS + filterable is 200% of logical size.//-// The scalar attributes below are filterable because they narrow *before* the-// ranking truncates. Applying them afterwards is what made a broad query with-// `country=US` return 85 rows out of 7,698 real matches.+// The scalar attributes are filterable so the provider narrows before it+// truncates to `limit`, rather than leaving filters to run over an already+// truncated list.const TURBOPUFFER_SCHEMA = {programId: { type: "string", filterable: true },status: { type: "string", filterable: true },@@ -101,6 +101,9 @@ export interface TurbopufferNamespace {multiQuery(params: Record<string, unknown>): Promise<{results?: { rows?: { id: string | number; $dist?: number }[] }[];}>;+ query(params: Record<string, unknown>): Promise<{+ aggregations?: Record<string, unknown>;+ }>;deleteAll(): Promise<unknown>;}@@ -165,12 +168,7 @@ function serializeTurbopufferRow(* shares just one, which is how an unrelated address looks like a partial-email* match.*/-/**- * Turns the discrete filters into turbopuffer clauses. Exclusion uses NotIn and- * NotContainsAny, which match documents that omit the attribute entirely — the- * partners with no group or no country, which the database also includes when- * it negates.- */+/** Turns the discrete filters into turbopuffer clauses. */function buildFilterClauses(filters: PartnerSearchCandidateQuery["filters"],): unknown[] {@@ -256,6 +254,38 @@ function buildQueryBranches(return branches;}+/**+ * The same documents the ranked branches would return, as one filter.+ *+ * `identityText` holds a subset of what `searchText` holds, so a single+ * ContainsAnyToken over `searchText` covers both text branches and the all-terms+ * branch that narrows them. BM25 matches a document sharing any query token, and+ * ContainsAnyToken is that same test without the scoring.+ */+function buildCountFilter(+ programId: string,+ query: string,+ filters: PartnerSearchCandidateQuery["filters"],+) {+ const matchClauses: unknown[] = [+ ["searchText", "ContainsAnyToken", query, { last_as_prefix: true }],+ ];++ const ngrams = getQueryNgrams(query).join(" ");+ if (ngrams) {+ matchClauses.push(["emailNgrams", "ContainsAllTokens", ngrams]);+ }++ return [+ "And",+ [+ ["programId", "Eq", programId],+ ...buildFilterClauses(filters),+ ["Or", matchClauses],+ ],+ ];+}+// The standard reciprocal-rank-fusion constant: dampens the gap between// neighboring ranks so one branch's top result cannot drown out the other's.const RRF_RANK_CONSTANT = 60;@@ -336,6 +366,22 @@ export function createTurbopufferPartnerSearchProvider({return { hits };},+ async countCandidates({ programId, query, filters }) {+ const normalizedQuery = normalizePartnerSearchQuery(query);+ const response = await withQueryDeadline(+ () =>+ resolvedNamespace.query({+ filters: buildCountFilter(programId, normalizedQuery, filters),+ aggregate_by: { total: ["Count"] },+ }),+ QUERY_OPERATION_TIMEOUT_MS,+ );++ const total = response.aggregations?.total;++ return typeof total === "number" ? total : 0;+ },+async upsert(documents) {for (const documentBatch of chunk(documents, WRITE_BATCH_SIZE)) {await withTransientRetry(() =>diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.tsindex 2ebcac9371..b5d0732fc4 100644--- a/apps/web/lib/api/partners/search/types.ts+++ b/apps/web/lib/api/partners/search/types.ts@@ -57,11 +57,8 @@ export interface PartnerSearchDocument {shortLinks: string[];destinationUrls: string[];- // Filterable fields, chosen because they change when someone acts on a- // partner rather than continuously. Metrics are deliberately absent: they move- // on every click and conversion, so indexing them would make the document- // churn constantly and turn the index into a second source of truth for- // numbers the table displays.+ // Filterable fields. Metrics are deliberately absent: they move on every click+ // and conversion, so indexing them would make the document churn continuously.status: ProgramEnrollmentStatus;groupId: string | null;country: string | null;@@ -108,6 +105,8 @@ export interface PartnerSearchProvider {searchCandidates(query: PartnerSearchCandidateQuery,): Promise<PartnerSearchResult>;+ /** How many documents match, ignoring `limit`. */+ countCandidates(query: PartnerSearchCandidateQuery): Promise<number>;upsert(documents: PartnerSearchDocument[]): Promise<void>;delete(documentIds: string[]): Promise<void>;}diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.tsindex aa088921bc..ef852b5948 100644--- a/apps/web/tests/partners/get-partners-count-search.test.ts+++ b/apps/web/tests/partners/get-partners-count-search.test.ts@@ -23,11 +23,12 @@ vi.mock("@/lib/prisma", () => ({},}));-function createSearchProvider(): PartnerSearchProvider {+function createSearchProvider(total = 12_000): PartnerSearchProvider {return {searchCandidates: vi.fn().mockResolvedValue({hits: [{ id: "pge_2" }, { id: "pge_1" }],}),+ countCandidates: vi.fn().mockResolvedValue(total),upsert: vi.fn(),delete: vi.fn(),};@@ -40,9 +41,10 @@ describe("getPartnersCount search", () => {}});- it("counts database-filtered relevance candidates", async () => {- const searchProvider = createSearchProvider();- mocks.count.mockResolvedValue(1);+ it("reports the provider's real total, not the truncated candidate count", async () => {+ // Counting the hits would report the candidate ceiling as the answer.+ const searchProvider = createSearchProvider(12_000);+ mocks.count.mockResolvedValue(999);const count = await getPartnersCount<number>({@@ -54,8 +56,8 @@ describe("getPartnersCount search", () => {{ searchProvider },);- expect(count).toBe(1);- expect(searchProvider.searchCandidates).toHaveBeenCalledWith({+ expect(count).toBe(12_000);+ expect(searchProvider.countCandidates).toHaveBeenCalledWith({programId: "prog_test",query: "examp",limit: 999,@@ -66,14 +68,46 @@ describe("getPartnersCount search", () => {partnerTagIds: undefined,},});- expect(mocks.count).toHaveBeenCalledWith({- where: expect.objectContaining({- programId: "prog_test",- status: "approved",- id: { in: ["pge_2", "pge_1"] },- partner: { country: "CA" },- }),- });+ expect(mocks.count).not.toHaveBeenCalled();+ });++ it.each([+ ["a tenant ID", { tenantId: "tenant_1" }],+ ["explicit partner IDs", { partnerIds: ["pn_1"] }],+ ["a metric range", { totalClicksMin: 10 }],+ ])(+ "falls back to the database count when the provider cannot see %s",+ async (_label, extra) => {+ // The aggregation only knows the filters it was given, so counting with+ // one of these applied would over-count.+ const searchProvider = createSearchProvider(12_000);+ mocks.count.mockResolvedValue(3);++ const count = await getPartnersCount<number>(+ { programId: "prog_test", search: "examp", ...extra },+ { searchProvider },+ );++ expect(count).toBe(3);+ expect(searchProvider.countCandidates).not.toHaveBeenCalled();+ expect(mocks.count).toHaveBeenCalled();+ },+ );++ it("falls back to the database count when the aggregation fails", async () => {+ const searchProvider = createSearchProvider();+ (+ searchProvider.countCandidates as ReturnType<typeof vi.fn>+ ).mockRejectedValue(new Error("aggregation down"));+ mocks.count.mockResolvedValue(7);+ vi.spyOn(console, "error").mockImplementation(() => {});++ const count = await getPartnersCount<number>(+ { programId: "prog_test", search: "examp" },+ { searchProvider },+ );++ expect(count).toBe(7);});it("groups database-filtered relevance candidates", async () => {@@ -178,21 +212,28 @@ describe("getPartnersCount search", () => {mocks.count.mockResolvedValue(0);const groupings = [- { groupBy: "status", mock: mocks.enrollmentGroupBy },- { groupBy: "groupId", mock: mocks.enrollmentGroupBy },- { groupBy: "country", mock: mocks.partnerGroupBy },- { groupBy: "partnerTagId", mock: mocks.partnerTagGroupBy },- { groupBy: "referredByPartnerId", mock: mocks.applicationEventGroupBy },- { groupBy: undefined, mock: mocks.count },+ { groupBy: "status", mock: mocks.enrollmentGroupBy, extra: {} },+ { groupBy: "groupId", mock: mocks.enrollmentGroupBy, extra: {} },+ { groupBy: "country", mock: mocks.partnerGroupBy, extra: {} },+ { groupBy: "partnerTagId", mock: mocks.partnerTagGroupBy, extra: {} },+ {+ groupBy: "referredByPartnerId",+ mock: mocks.applicationEventGroupBy,+ extra: {},+ },+ // The ungrouped count only reaches the database when a filter the provider+ // cannot see forces it to, so give it one.+ { groupBy: undefined, mock: mocks.count, extra: { totalClicksMin: 1 } },] as const;- for (const { groupBy, mock } of groupings) {+ for (const { groupBy, mock, extra } of groupings) {mock.mockClear();await getPartnersCount({programId: "prog_test",search: "examp",+ ...extra,...(groupBy && { groupBy }),},{ searchProvider: createSearchProvider() },diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.tsindex cf49356cee..f932ba2f6a 100644--- a/apps/web/tests/partners/get-partners-search.test.ts+++ b/apps/web/tests/partners/get-partners-search.test.ts@@ -37,6 +37,7 @@ function createSearchProvider(): PartnerSearchProvider {return {searchCandidates: vi.fn().mockResolvedValue({ hits }),+ countCandidates: vi.fn(),upsert: vi.fn(),delete: vi.fn(),};diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.tsindex da5b0a5157..62730b1294 100644--- a/apps/web/tests/partners/partner-search-backfill.test.ts+++ b/apps/web/tests/partners/partner-search-backfill.test.ts@@ -41,6 +41,7 @@ function createSource(id: string): PartnerSearchDocumentSource {function createProvider(): PartnerSearchProvider {return {searchCandidates: vi.fn(),+ countCandidates: vi.fn(),upsert: vi.fn(),delete: vi.fn(),};diff --git a/apps/web/tests/partners/partner-search-find-candidates.test.ts b/apps/web/tests/partners/partner-search-find-candidates.test.tsindex 05668f567f..e06bc59b64 100644--- a/apps/web/tests/partners/partner-search-find-candidates.test.ts+++ b/apps/web/tests/partners/partner-search-find-candidates.test.ts@@ -13,6 +13,7 @@ function createProvider() {return {searchCandidates,+ countCandidates: vi.fn(),upsert: vi.fn(),delete: vi.fn(),} as unknown as PartnerSearchProvider & {