The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Filter status, group, country, and tags before ranking
Rafi37faaf2d44cb
diff --git a/apps/web/lib/api/partners/search/build-search-query.ts b/apps/web/lib/api/partners/search/build-search-query.tsindex f0951fbeb0..af7894ad3a 100644--- a/apps/web/lib/api/partners/search/build-search-query.ts+++ b/apps/web/lib/api/partners/search/build-search-query.ts@@ -3,6 +3,7 @@ import { getPartnersQuerySchemaExtended } from "@/lib/zod/schemas/partners";import * as z from "zod/v4";import {PARTNER_SEARCH_CANDIDATE_LIMIT,+ PartnerSearchCandidateFilter,PartnerSearchCandidateQuery,} from "./types";@@ -17,14 +18,44 @@ export type PartnerSearchQueryInput = z.infer<type PartnerSearchCandidateQueryInput = Pick<PartnerSearchQueryInput,- "programId" | "search" | "email" | "tenantId"+ | "programId"+ | "search"+ | "email"+ | "tenantId"+ | "status"+ | "groupId"+ | "country"+ | "partnerTagId"+ | "groupIdOperator"+ | "countryOperator"+ | "partnerTagIdOperator">;+function toFilter(+ value: string | string[] | undefined | null,+ operator?: "IN" | "NOT IN",+): PartnerSearchCandidateFilter | undefined {+ const values = (Array.isArray(value) ? value : [value]).filter(+ (entry): entry is string => Boolean(entry),+ );++ return values.length > 0+ ? { values, exclude: operator === "NOT IN" }+ : undefined;+}+export function buildPartnerSearchCandidateQuery({programId,search,email,tenantId,+ status,+ groupId,+ country,+ partnerTagId,+ groupIdOperator,+ countryOperator,+ partnerTagIdOperator,}: PartnerSearchCandidateQueryInput): PartnerSearchCandidateQuery | null {const query = search?.trim();@@ -42,5 +73,11 @@ export function buildPartnerSearchCandidateQuery({programId,query,limit: PARTNER_SEARCH_CANDIDATE_LIMIT,+ filters: {+ status: toFilter(status),+ groupId: toFilter(groupId, groupIdOperator),+ country: toFilter(country, countryOperator),+ partnerTagIds: toFilter(partnerTagId, partnerTagIdOperator),+ },};}diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.tsindex c23ed6e72b..15b2a03a17 100644--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts@@ -5,7 +5,11 @@ import {getPartnerSearchableValues,normalizePartnerSearchQuery,} from "../searchable-values";-import type { PartnerSearchDocument, PartnerSearchProvider } from "../types";+import type {+ PartnerSearchCandidateQuery,+ PartnerSearchDocument,+ PartnerSearchProvider,+} from "../types";import { validatePartnerSearchCandidateLimit } from "../types";import { withQueryDeadline, withTransientRetry } from "./resilience";import { getEmailNgrams, getQueryNgrams } from "./shared";@@ -16,7 +20,7 @@ import { getEmailNgrams, getQueryNgrams } from "./shared";* existing namespace. A new version is backfilled alongside the old one and* swapped in, rather than rebuilt in place.*/-const NAMESPACE = "partner-search-v2";+const NAMESPACE = "partner-search-v3";const WRITE_BATCH_SIZE = 500;const QUERY_OPERATION_TIMEOUT_MS = 1_000;@@ -39,6 +43,10 @@ interface TurbopufferPartnerSearchRow extends Record<string, unknown> {searchText: string;identityText: string;emailNgrams: string;+ status: string;+ groupId?: string;+ country?: string;+ partnerTagIds: string[];}/**@@ -58,9 +66,16 @@ const SEARCH_TEXT_TOKENIZER = "word_v2";// ContainsAllTokens, which reads the BM25 index rather than a filter index, so// marking them filterable would buy nothing and cost plenty: turbopuffer bills// an attribute per enabled index, so FTS + filterable is 200% of logical size.-// Only `programId` needs one, for its scalar Eq.+//+// The scalar attributes below are filterable because they narrow *before* the+// ranking truncates. Applying them afterwards is what made a broad query with+// `country=US` return 85 rows out of 7,698 real matches.const TURBOPUFFER_SCHEMA = {programId: { type: "string", filterable: true },+ status: { type: "string", filterable: true },+ groupId: { type: "string", filterable: true },+ country: { type: "string", filterable: true },+ partnerTagIds: { type: "[]string", filterable: true },searchText: {type: "string",full_text_search: { tokenizer: SEARCH_TEXT_TOKENIZER },@@ -121,6 +136,13 @@ function serializeTurbopufferRow(searchText: normalizeValues(getPartnerSearchableValues(document)),identityText: normalizeValues(getPartnerIdentityValues(document)),emailNgrams: getEmailNgrams(document.email),+ status: document.status,+ // Omitted rather than stored null, which is what makes a NotIn filter match+ // the partners that have no group or no country — the same rows the+ // database includes when it ORs the negation with IS NULL.+ ...(document.groupId ? { groupId: document.groupId } : {}),+ ...(document.country ? { country: document.country } : {}),+ partnerTagIds: document.partnerTagIds,};}@@ -143,8 +165,54 @@ function serializeTurbopufferRow(* shares just one, which is how an unrelated address looks like a partial-email* match.*/-function buildQueryBranches(programId: string, query: string) {- const programFilter = ["programId", "Eq", programId];+/**+ * Turns the discrete filters into turbopuffer clauses. Exclusion uses NotIn and+ * NotContainsAny, which match documents that omit the attribute entirely — the+ * partners with no group or no country, which the database also includes when+ * it negates.+ */+function buildFilterClauses(+ filters: PartnerSearchCandidateQuery["filters"],+): unknown[] {+ if (!filters) {+ return [];+ }++ const clauses: unknown[] = [];++ for (const [attribute, filter] of Object.entries(filters)) {+ // An empty value list would become `In []`, which matches nothing and would+ // silently empty the results rather than leaving them unfiltered.+ if (!filter || filter.values.length === 0) {+ continue;+ }++ const isArray = attribute === "partnerTagIds";+ const operator = filter.exclude+ ? isArray+ ? "NotContainsAny"+ : "NotIn"+ : isArray+ ? "ContainsAny"+ : "In";++ clauses.push([attribute, operator, filter.values]);+ }++ return clauses;+}++function buildQueryBranches(+ programId: string,+ query: string,+ filters?: PartnerSearchCandidateQuery["filters"],+) {+ const scope = ["programId", "Eq", programId];+ const filterClauses = buildFilterClauses(filters);+ // Every branch carries the same scope, because a root-level clause beside the+ // branches would not narrow them individually.+ const programFilter =+ filterClauses.length > 0 ? ["And", [scope, ...filterClauses]] : scope;const branches: Record<string, unknown>[] = ["searchText","identityText",@@ -243,13 +311,19 @@ export function createTurbopufferPartnerSearchProvider({const resolvedNamespace = namespace ?? createNamespace(resolvedNamespaceName);return {- async searchCandidates({ programId, query, limit }) {+ async searchCandidates({ programId, query, limit, filters }) {validatePartnerSearchCandidateLimit(limit);const normalizedQuery = normalizePartnerSearchQuery(query);- const branches = buildQueryBranches(programId, normalizedQuery).map(- (branch) => ({ ...branch, top_k: limit, include_attributes: false }),- );+ const branches = buildQueryBranches(+ programId,+ normalizedQuery,+ filters,+ ).map((branch) => ({+ ...branch,+ top_k: limit,+ include_attributes: false,+ }));// One round trip regardless of branch count.const { results } = await withQueryDeadline(diff --git a/apps/web/lib/api/partners/search/serialize-document.ts b/apps/web/lib/api/partners/search/serialize-document.tsindex 42d6729c58..57e8913a58 100644--- a/apps/web/lib/api/partners/search/serialize-document.ts+++ b/apps/web/lib/api/partners/search/serialize-document.ts@@ -5,12 +5,23 @@ export const partnerSearchDocumentSelect = {id: true,programId: true,partnerId: true,+ status: true,+ groupId: true,partner: {select: {name: true,email: true,companyName: true,description: true,+ country: true,+ // Tags are per (program, partner) and this select cannot take a program,+ // so the serializer narrows them to the enrollment's own program.+ programPartnerTags: {+ select: {+ programId: true,+ partnerTagId: true,+ },+ },platforms: {select: {type: true,@@ -58,5 +69,13 @@ export function serializePartnerSearchDocument(linkKeys: unique(links.map(({ key }) => key)),shortLinks: unique(links.map(({ shortLink }) => shortLink)),destinationUrls: unique(links.map(({ url }) => url)),+ status: enrollment.status,+ groupId: enrollment.groupId,+ country: partner.country,+ partnerTagIds: unique(+ partner.programPartnerTags+ .filter(({ programId }) => programId === enrollment.programId)+ .map(({ partnerTagId }) => partnerTagId),+ ),};}diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.tsindex 2a1f4b3f3f..2ebcac9371 100644--- a/apps/web/lib/api/partners/search/types.ts+++ b/apps/web/lib/api/partners/search/types.ts@@ -1,4 +1,4 @@-import { PlatformType } from "@prisma/client";+import { PlatformType, ProgramEnrollmentStatus } from "@prisma/client";/*** How many ranked enrollment IDs a provider may return for one query.@@ -56,6 +56,16 @@ export interface PartnerSearchDocument {linkKeys: string[];shortLinks: string[];destinationUrls: string[];++ // Filterable fields, chosen because they change when someone acts on a+ // partner rather than continuously. Metrics are deliberately absent: they move+ // on every click and conversion, so indexing them would make the document+ // churn constantly and turn the index into a second source of truth for+ // numbers the table displays.+ status: ProgramEnrollmentStatus;+ groupId: string | null;+ country: string | null;+ partnerTagIds: string[];}export interface PartnerSearchHit {@@ -67,10 +77,31 @@ export interface PartnerSearchResult {hits: PartnerSearchHit[];}+/**+ * A discrete filter, plus whether the caller is excluding those values. Exclusion+ * also matches partners that have no value at all, which is what the database+ * does when it ORs the negation with IS NULL.+ */+export interface PartnerSearchCandidateFilter {+ values: string[];+ exclude?: boolean;+}+export interface PartnerSearchCandidateQuery {programId: string;query: string;limit: number;+ /**+ * Applied by the provider before it truncates to `limit`. The database still+ * re-applies every one of them, so the index only ever narrows the candidate+ * pool — it never decides the result on its own.+ */+ filters?: {+ status?: PartnerSearchCandidateFilter;+ groupId?: PartnerSearchCandidateFilter;+ country?: PartnerSearchCandidateFilter;+ partnerTagIds?: PartnerSearchCandidateFilter;+ };}export interface PartnerSearchProvider {diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.tsindex 14e2487928..aa088921bc 100644--- a/apps/web/tests/partners/get-partners-count-search.test.ts+++ b/apps/web/tests/partners/get-partners-count-search.test.ts@@ -59,6 +59,12 @@ describe("getPartnersCount search", () => {programId: "prog_test",query: "examp",limit: 999,+ filters: {+ status: { values: ["approved"], exclude: false },+ groupId: undefined,+ country: { values: ["CA"], exclude: false },+ partnerTagIds: undefined,+ },});expect(mocks.count).toHaveBeenCalledWith({where: expect.objectContaining({diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.tsindex f63cd0f733..cf49356cee 100644--- a/apps/web/tests/partners/get-partners-search.test.ts+++ b/apps/web/tests/partners/get-partners-search.test.ts@@ -98,10 +98,18 @@ describe("getPartners search", () => {{ searchProvider },);+ // The status filter reaches the provider, so it narrows before the ranking+ // truncates rather than after.expect(searchProvider.searchCandidates).toHaveBeenCalledWith({programId: "prog_test",query: "examp",limit: 999,+ filters: {+ status: { values: ["approved"], exclude: false },+ groupId: undefined,+ country: undefined,+ partnerTagIds: undefined,+ },});expect(mocks.findMany).toHaveBeenCalledWith(expect.objectContaining({diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.tsindex ecd599901d..da5b0a5157 100644--- a/apps/web/tests/partners/partner-search-backfill.test.ts+++ b/apps/web/tests/partners/partner-search-backfill.test.ts@@ -23,11 +23,15 @@ function createSource(id: string): PartnerSearchDocumentSource {id,programId: "prog_test",partnerId: `pn_${id}`,+ status: "approved" as const,+ groupId: null,partner: {name: "Rafi Hasan",email: "[email protected]",companyName: "Dub Partners",description: "Developer tools educator",+ country: null,+ programPartnerTags: [],platforms: [],},links: [],diff --git a/apps/web/tests/partners/partner-search-document.test.ts b/apps/web/tests/partners/partner-search-document.test.tsindex e591a3821f..51b1d53f3b 100644--- a/apps/web/tests/partners/partner-search-document.test.ts+++ b/apps/web/tests/partners/partner-search-document.test.ts@@ -8,11 +8,19 @@ const source: PartnerSearchDocumentSource = {id: "pge_test",programId: "prog_test",partnerId: "pn_test",+ status: "approved" as const,+ groupId: "grp_test",partner: {name: "Rafi Hasan",email: "[email protected]",companyName: "Dub Partners",description: "Developer tools educator",+ country: "US",+ programPartnerTags: [+ { programId: "prog_test", partnerTagId: "ptag_a" },+ // A tag from a different program must not leak into this document.+ { programId: "prog_other", partnerTagId: "ptag_other" },+ ],platforms: [{type: "website",@@ -59,6 +67,11 @@ describe("serializePartnerSearchDocument", () => {"https://example.com/referrals/rafi","https://rafi.dev/tools",],+ status: "approved",+ groupId: "grp_test",+ country: "US",+ // The tag from prog_other is dropped: tags are per program.+ partnerTagIds: ["ptag_a"],});});});diff --git a/apps/web/tests/partners/partner-search-query.test.ts b/apps/web/tests/partners/partner-search-query.test.tsindex 2d7c94b5f1..bdc218f47e 100644--- a/apps/web/tests/partners/partner-search-query.test.ts+++ b/apps/web/tests/partners/partner-search-query.test.ts@@ -16,6 +16,34 @@ describe("buildPartnerSearchCandidateQuery", () => {programId: "prog_test",query: "examp",limit: 999,+ filters: {+ status: undefined,+ groupId: undefined,+ country: undefined,+ partnerTagIds: undefined,+ },+ });+ });++ it("passes the discrete filters through, with exclusion", () => {+ expect(+ buildPartnerSearchCandidateQuery({+ ...defaultInput,+ status: "approved",+ groupId: ["grp_1", "grp_2"],+ country: "US",+ countryOperator: "NOT IN",+ partnerTagId: ["ptag_1"],+ }),+ ).toMatchObject({+ filters: {+ status: { values: ["approved"], exclude: false },+ groupId: { values: ["grp_1", "grp_2"], exclude: false },+ // Exclusion also matches partners with no country, matching the+ // database's OR against IS NULL.+ country: { values: ["US"], exclude: true },+ partnerTagIds: { values: ["ptag_1"], exclude: false },+ },});});diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.tsindex 1a4915604f..cb224322cc 100644--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts@@ -23,6 +23,10 @@ const document: PartnerSearchDocument = {linkKeys: ["rafi"],shortLinks: ["https://dub.sh/rafi"],destinationUrls: ["https://example.com/referrals/rafi"],+ status: "approved",+ groupId: "grp_test",+ country: "US",+ partnerTagIds: ["ptag_a", "ptag_b"],};const mocks = vi.hoisted(() => ({@@ -239,6 +243,48 @@ describe("Turbopuffer partner search provider", () => {expect(branchesOf().filteredOn("identityText")).toHaveLength(0);});+ it("narrows every branch with the discrete filters", async () => {+ // The filters have to sit inside each branch: applying them after the+ // ranking truncates is what made a broad query with country=US return 85+ // rows out of 7,698 real matches.+ await createProvider().searchCandidates({+ programId: "prog_test",+ query: "examp",+ limit: 10,+ filters: {+ status: { values: ["approved"] },+ country: { values: ["US", "CA"], exclude: true },+ partnerTagIds: { values: ["ptag_1"] },+ },+ });++ const [{ queries }] = mocks.multiQuery.mock.calls[0];+ expect(queries.length).toBeGreaterThan(1);++ for (const branch of queries) {+ const filters = JSON.stringify(branch.filters);+ expect(filters).toContain('["programId","Eq","prog_test"]');+ expect(filters).toContain('["status","In",["approved"]]');+ // Exclusion uses NotIn, which also matches documents that omit country.+ expect(filters).toContain('["country","NotIn",["US","CA"]]');+ expect(filters).toContain('["partnerTagIds","ContainsAny",["ptag_1"]]');+ }+ });++ it("omits absent filters rather than sending empty clauses", async () => {+ await createProvider().searchCandidates({+ programId: "prog_test",+ query: "examp",+ limit: 10,+ filters: { status: undefined, country: { values: [] } },+ });++ const [{ queries }] = mocks.multiQuery.mock.calls[0];+ for (const branch of queries) {+ expect(JSON.stringify(branch.filters)).not.toContain('"country"');+ }+ });+it("requires every trigram on the n-gram branch", async () => {// BM25 alone would score a document sharing a single trigram, which is how// an unrelated address looks like a partial-email match.@@ -368,6 +414,6 @@ describe("Turbopuffer partner search provider", () => {namespace: createNamespaceMock(),});- expect(namespaceName).toBe("partner-search-v2");+ expect(namespaceName).toBe("partner-search-v3");});});