Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Count is an aggregate filter now

Rafi3bb753b2819b
diff --git a/apps/web/lib/api/partners/get-partners-count.ts b/apps/web/lib/api/partners/get-partners-count.ts
index 03a9d4546f..cc6304cb31 100644
--- a/apps/web/lib/api/partners/get-partners-count.ts
+++ b/apps/web/lib/api/partners/get-partners-count.ts
@@ -260,6 +260,32 @@ export async function getPartnersCount<T>(
})) as T;
}
+ // Counting the candidate IDs reports a floor, because the provider truncated
+ // them to the candidate ceiling — "999 partners" when there are twelve
+ // thousand. Ask the provider for the real total instead, but only when it can
+ // see every filter that applies: `partnerIds`, `tenantId`, and the metric
+ // ranges are database-only, so leaving them out would over-count.
+ const databaseOnlyFilters =
+ Boolean(tenantId) ||
+ Boolean(partnerIds?.length) ||
+ Object.keys(buildMetricRangeWhere(enrollmentBase)).length > 0;
+
+ if (
+ searchProvider &&
+ candidateQuery &&
+ candidateResult &&
+ !databaseOnlyFilters
+ ) {
+ try {
+ return (await searchProvider.countCandidates(candidateQuery)) as T;
+ } catch (error) {
+ console.error(
+ "[Partner Search] Count aggregation failed, falling back to counting candidates.",
+ error,
+ );
+ }
+ }
+
// Get absolute count of partners
const count = await prisma.programEnrollment.count({
where: {
diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
index 15b2a03a17..53a48e0f1a 100644
--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts
+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
@@ -67,9 +67,9 @@ const SEARCH_TEXT_TOKENIZER = "word_v2";
// marking them filterable would buy nothing and cost plenty: turbopuffer bills
// an attribute per enabled index, so FTS + filterable is 200% of logical size.
//
-// The scalar attributes below are filterable because they narrow *before* the
-// ranking truncates. Applying them afterwards is what made a broad query with
-// `country=US` return 85 rows out of 7,698 real matches.
+// The scalar attributes are filterable so the provider narrows before it
+// truncates to `limit`, rather than leaving filters to run over an already
+// truncated list.
const TURBOPUFFER_SCHEMA = {
programId: { type: "string", filterable: true },
status: { type: "string", filterable: true },
@@ -101,6 +101,9 @@ export interface TurbopufferNamespace {
multiQuery(params: Record<string, unknown>): Promise<{
results?: { rows?: { id: string | number; $dist?: number }[] }[];
}>;
+ query(params: Record<string, unknown>): Promise<{
+ aggregations?: Record<string, unknown>;
+ }>;
deleteAll(): Promise<unknown>;
}
@@ -165,12 +168,7 @@ function serializeTurbopufferRow(
* shares just one, which is how an unrelated address looks like a partial-email
* match.
*/
-/**
- * Turns the discrete filters into turbopuffer clauses. Exclusion uses NotIn and
- * NotContainsAny, which match documents that omit the attribute entirely — the
- * partners with no group or no country, which the database also includes when
- * it negates.
- */
+/** Turns the discrete filters into turbopuffer clauses. */
function buildFilterClauses(
filters: PartnerSearchCandidateQuery["filters"],
): unknown[] {
@@ -256,6 +254,38 @@ function buildQueryBranches(
return branches;
}
+/**
+ * The same documents the ranked branches would return, as one filter.
+ *
+ * `identityText` holds a subset of what `searchText` holds, so a single
+ * ContainsAnyToken over `searchText` covers both text branches and the all-terms
+ * branch that narrows them. BM25 matches a document sharing any query token, and
+ * ContainsAnyToken is that same test without the scoring.
+ */
+function buildCountFilter(
+ programId: string,
+ query: string,
+ filters: PartnerSearchCandidateQuery["filters"],
+) {
+ const matchClauses: unknown[] = [
+ ["searchText", "ContainsAnyToken", query, { last_as_prefix: true }],
+ ];
+
+ const ngrams = getQueryNgrams(query).join(" ");
+ if (ngrams) {
+ matchClauses.push(["emailNgrams", "ContainsAllTokens", ngrams]);
+ }
+
+ return [
+ "And",
+ [
+ ["programId", "Eq", programId],
+ ...buildFilterClauses(filters),
+ ["Or", matchClauses],
+ ],
+ ];
+}
+
// The standard reciprocal-rank-fusion constant: dampens the gap between
// neighboring ranks so one branch's top result cannot drown out the other's.
const RRF_RANK_CONSTANT = 60;
@@ -336,6 +366,22 @@ export function createTurbopufferPartnerSearchProvider({
return { hits };
},
+ async countCandidates({ programId, query, filters }) {
+ const normalizedQuery = normalizePartnerSearchQuery(query);
+ const response = await withQueryDeadline(
+ () =>
+ resolvedNamespace.query({
+ filters: buildCountFilter(programId, normalizedQuery, filters),
+ aggregate_by: { total: ["Count"] },
+ }),
+ QUERY_OPERATION_TIMEOUT_MS,
+ );
+
+ const total = response.aggregations?.total;
+
+ return typeof total === "number" ? total : 0;
+ },
+
async upsert(documents) {
for (const documentBatch of chunk(documents, WRITE_BATCH_SIZE)) {
await withTransientRetry(() =>
diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.ts
index 2ebcac9371..b5d0732fc4 100644
--- a/apps/web/lib/api/partners/search/types.ts
+++ b/apps/web/lib/api/partners/search/types.ts
@@ -57,11 +57,8 @@ export interface PartnerSearchDocument {
shortLinks: string[];
destinationUrls: string[];
- // Filterable fields, chosen because they change when someone acts on a
- // partner rather than continuously. Metrics are deliberately absent: they move
- // on every click and conversion, so indexing them would make the document
- // churn constantly and turn the index into a second source of truth for
- // numbers the table displays.
+ // Filterable fields. Metrics are deliberately absent: they move on every click
+ // and conversion, so indexing them would make the document churn continuously.
status: ProgramEnrollmentStatus;
groupId: string | null;
country: string | null;
@@ -108,6 +105,8 @@ export interface PartnerSearchProvider {
searchCandidates(
query: PartnerSearchCandidateQuery,
): Promise<PartnerSearchResult>;
+ /** How many documents match, ignoring `limit`. */
+ countCandidates(query: PartnerSearchCandidateQuery): Promise<number>;
upsert(documents: PartnerSearchDocument[]): Promise<void>;
delete(documentIds: string[]): Promise<void>;
}
diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.ts
index aa088921bc..ef852b5948 100644
--- a/apps/web/tests/partners/get-partners-count-search.test.ts
+++ b/apps/web/tests/partners/get-partners-count-search.test.ts
@@ -23,11 +23,12 @@ vi.mock("@/lib/prisma", () => ({
},
}));
-function createSearchProvider(): PartnerSearchProvider {
+function createSearchProvider(total = 12_000): PartnerSearchProvider {
return {
searchCandidates: vi.fn().mockResolvedValue({
hits: [{ id: "pge_2" }, { id: "pge_1" }],
}),
+ countCandidates: vi.fn().mockResolvedValue(total),
upsert: vi.fn(),
delete: vi.fn(),
};
@@ -40,9 +41,10 @@ describe("getPartnersCount search", () => {
}
});
- it("counts database-filtered relevance candidates", async () => {
- const searchProvider = createSearchProvider();
- mocks.count.mockResolvedValue(1);
+ it("reports the provider's real total, not the truncated candidate count", async () => {
+ // Counting the hits would report the candidate ceiling as the answer.
+ const searchProvider = createSearchProvider(12_000);
+ mocks.count.mockResolvedValue(999);
const count = await getPartnersCount<number>(
{
@@ -54,8 +56,8 @@ describe("getPartnersCount search", () => {
{ searchProvider },
);
- expect(count).toBe(1);
- expect(searchProvider.searchCandidates).toHaveBeenCalledWith({
+ expect(count).toBe(12_000);
+ expect(searchProvider.countCandidates).toHaveBeenCalledWith({
programId: "prog_test",
query: "examp",
limit: 999,
@@ -66,14 +68,46 @@ describe("getPartnersCount search", () => {
partnerTagIds: undefined,
},
});
- expect(mocks.count).toHaveBeenCalledWith({
- where: expect.objectContaining({
- programId: "prog_test",
- status: "approved",
- id: { in: ["pge_2", "pge_1"] },
- partner: { country: "CA" },
- }),
- });
+ expect(mocks.count).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ["a tenant ID", { tenantId: "tenant_1" }],
+ ["explicit partner IDs", { partnerIds: ["pn_1"] }],
+ ["a metric range", { totalClicksMin: 10 }],
+ ])(
+ "falls back to the database count when the provider cannot see %s",
+ async (_label, extra) => {
+ // The aggregation only knows the filters it was given, so counting with
+ // one of these applied would over-count.
+ const searchProvider = createSearchProvider(12_000);
+ mocks.count.mockResolvedValue(3);
+
+ const count = await getPartnersCount<number>(
+ { programId: "prog_test", search: "examp", ...extra },
+ { searchProvider },
+ );
+
+ expect(count).toBe(3);
+ expect(searchProvider.countCandidates).not.toHaveBeenCalled();
+ expect(mocks.count).toHaveBeenCalled();
+ },
+ );
+
+ it("falls back to the database count when the aggregation fails", async () => {
+ const searchProvider = createSearchProvider();
+ (
+ searchProvider.countCandidates as ReturnType<typeof vi.fn>
+ ).mockRejectedValue(new Error("aggregation down"));
+ mocks.count.mockResolvedValue(7);
+ vi.spyOn(console, "error").mockImplementation(() => {});
+
+ const count = await getPartnersCount<number>(
+ { programId: "prog_test", search: "examp" },
+ { searchProvider },
+ );
+
+ expect(count).toBe(7);
});
it("groups database-filtered relevance candidates", async () => {
@@ -178,21 +212,28 @@ describe("getPartnersCount search", () => {
mocks.count.mockResolvedValue(0);
const groupings = [
- { groupBy: "status", mock: mocks.enrollmentGroupBy },
- { groupBy: "groupId", mock: mocks.enrollmentGroupBy },
- { groupBy: "country", mock: mocks.partnerGroupBy },
- { groupBy: "partnerTagId", mock: mocks.partnerTagGroupBy },
- { groupBy: "referredByPartnerId", mock: mocks.applicationEventGroupBy },
- { groupBy: undefined, mock: mocks.count },
+ { groupBy: "status", mock: mocks.enrollmentGroupBy, extra: {} },
+ { groupBy: "groupId", mock: mocks.enrollmentGroupBy, extra: {} },
+ { groupBy: "country", mock: mocks.partnerGroupBy, extra: {} },
+ { groupBy: "partnerTagId", mock: mocks.partnerTagGroupBy, extra: {} },
+ {
+ groupBy: "referredByPartnerId",
+ mock: mocks.applicationEventGroupBy,
+ extra: {},
+ },
+ // The ungrouped count only reaches the database when a filter the provider
+ // cannot see forces it to, so give it one.
+ { groupBy: undefined, mock: mocks.count, extra: { totalClicksMin: 1 } },
] as const;
- for (const { groupBy, mock } of groupings) {
+ for (const { groupBy, mock, extra } of groupings) {
mock.mockClear();
await getPartnersCount(
{
programId: "prog_test",
search: "examp",
+ ...extra,
...(groupBy && { groupBy }),
},
{ searchProvider: createSearchProvider() },
diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.ts
index cf49356cee..f932ba2f6a 100644
--- a/apps/web/tests/partners/get-partners-search.test.ts
+++ b/apps/web/tests/partners/get-partners-search.test.ts
@@ -37,6 +37,7 @@ function createSearchProvider(
): PartnerSearchProvider {
return {
searchCandidates: vi.fn().mockResolvedValue({ hits }),
+ countCandidates: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
};
diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.ts
index da5b0a5157..62730b1294 100644
--- a/apps/web/tests/partners/partner-search-backfill.test.ts
+++ b/apps/web/tests/partners/partner-search-backfill.test.ts
@@ -41,6 +41,7 @@ function createSource(id: string): PartnerSearchDocumentSource {
function createProvider(): PartnerSearchProvider {
return {
searchCandidates: vi.fn(),
+ countCandidates: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
};
diff --git a/apps/web/tests/partners/partner-search-find-candidates.test.ts b/apps/web/tests/partners/partner-search-find-candidates.test.ts
index 05668f567f..e06bc59b64 100644
--- a/apps/web/tests/partners/partner-search-find-candidates.test.ts
+++ b/apps/web/tests/partners/partner-search-find-candidates.test.ts
@@ -13,6 +13,7 @@ function createProvider(
) {
return {
searchCandidates,
+ countCandidates: vi.fn(),
upsert: vi.fn(),
delete: vi.fn(),
} as unknown as PartnerSearchProvider & {