The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Improved identity text based ranking
Rafif755ce1a7669
diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.tsindex 57f4b87ecf..50f923fca5 100644--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts@@ -1,6 +1,7 @@import { chunk } from "@dub/utils";import { Turbopuffer } from "@turbopuffer/turbopuffer";import {+ getPartnerIdentityValues,getPartnerSearchableValues,normalizePartnerSearchQuery,} from "../searchable-values";@@ -9,7 +10,7 @@ import { validatePartnerSearchCandidateLimit } from "../types";import { withQueryDeadline, withTransientRetry } from "./resilience";import { getEmailNgrams, getQueryNgrams, resolveIndexName } from "./shared";-const DEFAULT_NAMESPACE = "partner-search-v1";+const DEFAULT_NAMESPACE = "partner-search-v2";const WRITE_BATCH_SIZE = 500;const QUERY_OPERATION_TIMEOUT_MS = 1_000;@@ -30,6 +31,7 @@ interface TurbopufferPartnerSearchRow extends Record<string, unknown> {id: string;programId: string;searchText: string;+ identityText: string;emailNgrams: string;}@@ -46,39 +48,24 @@ interface TurbopufferPartnerSearchRow extends Record<string, unknown> {*/const SEARCH_TEXT_TOKENIZER = "word_v2";-/**- * BM25 length normalization, left at turbopuffer's default.- *- * Lowering it is tempting: `searchText` is concatenated field values, so length- * tracks how many links a partner has rather than how much prose they wrote,- * and at 0.75 that demotes the most active partners. Searching "steven te" on- * the production index ranks Steven Tey below a partner whose description- * merely contains "tech", and the ordering flips at b=0.1.- *- * It was measured and rejected. On the same 100K index, b=0.1 against b=0.75- * minutes apart: 16.05% vs 11.01% timeouts and 836ms vs 720ms p99 — flattening- * the score distribution appears to defeat top-k pruning. The ranking gain only- * applies to half-typed multi-word queries, since the all-terms branch already- * handles complete ones.- */-const SEARCH_TEXT_LENGTH_NORMALIZATION = 0.75;--// `searchText` and `emailNgrams` are BM25-indexed. BM25 attributes are not-// filterable by default, but `emailNgrams` needs to be: the n-gram branch uses-// a ContainsAllTokens filter to require every trigram.+// No text attribute is filterable. The n-gram and all-terms branches narrow with+// ContainsAllTokens, which reads the BM25 index rather than a filter index, so+// marking them filterable would buy nothing and cost plenty: turbopuffer bills+// an attribute per enabled index, so FTS + filterable is 200% of logical size.+// Only `programId` needs one, for its scalar Eq.const TURBOPUFFER_SCHEMA = {programId: { type: "string", filterable: true },searchText: {type: "string",- full_text_search: {- tokenizer: SEARCH_TEXT_TOKENIZER,- b: SEARCH_TEXT_LENGTH_NORMALIZATION,- },+ full_text_search: { tokenizer: SEARCH_TEXT_TOKENIZER },+ },+ identityText: {+ type: "string",+ full_text_search: { tokenizer: SEARCH_TEXT_TOKENIZER },},emailNgrams: {type: "string",full_text_search: { tokenizer: SEARCH_TEXT_TOKENIZER },- filterable: true,},} as const;@@ -118,58 +105,67 @@ function createNamespace(namespaceName: string): TurbopufferNamespace {return client.namespace(namespaceName) as unknown as TurbopufferNamespace;}+function normalizeValues(values: string[]): string {+ return values.map(normalizePartnerSearchQuery).join(" ");+}+function serializeTurbopufferRow(document: PartnerSearchDocument,): TurbopufferPartnerSearchRow {return {id: document.id,programId: document.programId,- searchText: getPartnerSearchableValues(document)- .map(normalizePartnerSearchQuery)- .join(" "),+ searchText: normalizeValues(getPartnerSearchableValues(document)),+ identityText: normalizeValues(getPartnerIdentityValues(document)),emailNgrams: getEmailNgrams(document.email),};}/**- * Up to three ranked branches, rank-fused by the caller (see mergeBranchRows).+ * Each branch is ranked independently and rank-fused by the caller (see+ * mergeBranchRows).+ *+ * Identity is searched separately from everything else because a single-token+ * `last_as_prefix` query scores every match at exactly 1, so the broad branch+ * cannot order its own results. Matching an identity field puts a document in+ * two branches, which is what the fusion orders on.*- * The text branch treats the last token as a prefix so "john" reaches- * "johnson", matching how the Redis provider's $smart behaves. The n-gram- * branch carries a ContainsAllTokens filter so every trigram must be present —- * BM25 alone would score a document that shares just one, which is how an- * unrelated address ends up looking like a partial-email match.+ * The all-terms branch requires every query word, which BM25 does not: term+ * frequency saturates while length normalization keeps biting, so a partner+ * matching both words can otherwise rank below one matching only the first. It+ * runs on identityText so a full name is not out-weighted by a long description+ * or a pile of links.+ *+ * The n-gram branch requires every trigram; BM25 alone scores a document that+ * shares just one, which is how an unrelated address looks like a partial-email+ * match.*/function buildQueryBranches(programId: string, query: string) {const programFilter = ["programId", "Eq", programId];const branches: Record<string, unknown>[] = [- {- rank_by: ["searchText", "BM25", query, { last_as_prefix: true }],- filters: programFilter,- },- ];-- // A multi-word query should prefer partners matching every word, and BM25- // alone does not: term frequency saturates while length normalization keeps- // biting, so a long document matching both words can rank below a short one- // matching only the first. Measured on the production index, "steven tey" put- // the actual Steven Tey third, behind two partners with no "tey" at all. This- // branch admits only documents containing every term, so the rank fusion- // lifts them over partial matches without excluding those partial matches.+ "searchText",+ "identityText",+ ].map((attribute) => ({+ rank_by: [attribute, "BM25", query, { last_as_prefix: true }],+ filters: programFilter,+ }));+if (/\s/u.test(query)) {branches.push({- rank_by: ["searchText", "BM25", query, { last_as_prefix: true }],+ rank_by: ["identityText", "BM25", query, { last_as_prefix: true }],filters: ["And",[programFilter,- // Prefix on the last token here too. Without it the branch requires an- // exact final token, so a half-typed "steven te" matches nothing and- // search-as-you-type loses the boost exactly while it is being typed.- // Prefix on the last token here too. Without it the branch requires an+ // Prefix on the last token here too. Without it the branch needs an// exact final token, so a half-typed "steven te" matches nothing and- // the boost disappears exactly while the user is still typing.- ["searchText", "ContainsAllTokens", query, { last_as_prefix: true }],+ // the boost disappears while the user is still typing.+ [+ "identityText",+ "ContainsAllTokens",+ query,+ { last_as_prefix: true },+ ],],],});diff --git a/apps/web/lib/api/partners/search/searchable-values.ts b/apps/web/lib/api/partners/search/searchable-values.tsindex 4b3f8d49ed..6908e72d14 100644--- a/apps/web/lib/api/partners/search/searchable-values.ts+++ b/apps/web/lib/api/partners/search/searchable-values.ts@@ -1,10 +1,12 @@import { PartnerSearchDocument } from "./types";/**- * Returns every value covered by the partner-search assignment. Providers can- * index these as separate weighted fields or as one normalized search field.+ * Who the partner is: the fields someone types when they have a specific person+ * in mind. Short and stable, so a provider can index them separately and score a+ * name match against name length rather than against a document whose length is+ * really a count of how many links the partner has.*/-export function getPartnerSearchableValues(+export function getPartnerIdentityValues(document: PartnerSearchDocument,): string[] {return [@@ -12,11 +14,21 @@ export function getPartnerSearchableValues(document.name,document.email,document.companyName,+ ].filter((value): value is string => Boolean(value));+}++/**+ * Every value covered by the partner-search assignment. Providers can index+ * these as separate weighted fields or as one normalized search field.+ */+export function getPartnerSearchableValues(+ document: PartnerSearchDocument,+): string[] {+ return [+ ...getPartnerIdentityValues(document),document.description,...document.platformTypes,...document.platformIdentifiers,- ...document.linkDomains,- ...document.linkKeys,...document.shortLinks,...document.destinationUrls,].filter((value): value is string => Boolean(value));diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.tsindex 9e1c559f27..aa363b38c6 100644--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts@@ -39,6 +39,23 @@ function createNamespaceMock(): TurbopufferNamespace {} as unknown as TurbopufferNamespace;}+function branchesOf(call: number = 0) {+ const [{ queries }] = mocks.multiQuery.mock.calls[call];+ return {+ all: queries,+ // Located by attribute rather than position, so adding a branch does not+ // break every assertion.+ rankedOn: (attribute: string) =>+ queries.filter((branch: any) => branch.rank_by[0] === attribute),+ filteredOn: (attribute: string) =>+ queries.filter((branch: any) =>+ JSON.stringify(branch.filters).includes(+ `["${attribute}","ContainsAllTokens"`,+ ),+ ),+ };+}+function createProvider() {return createTurbopufferPartnerSearchProvider({namespace: createNamespaceMock(),@@ -61,6 +78,27 @@ describe("Turbopuffer partner search provider", () => {vi.unstubAllEnvs();});+ it("indexes identity fields separately, and only those fields", async () => {+ await createProvider().upsert([document]);++ const [{ upsert_rows, schema }] = mocks.write.mock.calls[0];+ const [row] = upsert_rows;++ expect(row.identityText).toBe(+ "pn_test rafi hasan [email protected] hasan labs",+ );+ // Description, platforms, and links stay out, so a name cannot be+ // out-weighted by how much else a partner has.+ for (const value of ["affiliate marketer", "youtube", "@rafi", "dub.sh"]) {+ expect(row.identityText).not.toContain(value);+ }+ // ContainsAllTokens reads the BM25 index, so no text attribute needs a+ // filter index — and turbopuffer bills FTS + filterable at 200%.+ expect(schema.identityText.filterable).toBeUndefined();+ expect(schema.emailNgrams.filterable).toBeUndefined();+ expect(schema.programId).toMatchObject({ filterable: true });+ });+it("flattens every searchable field into one BM25 attribute", async () => {await createProvider().upsert([document]);@@ -77,17 +115,11 @@ describe("Turbopuffer partner search provider", () => {"affiliate marketer","youtube","@rafi",- "dub.sh","https://dub.sh/rafi","https://example.com/referrals/rafi",]) {expect(row.searchText).toContain(value);}-- // BM25 attributes are not filterable by default, but the n-gram branch- // filters on emailNgrams.- expect(schema.emailNgrams).toMatchObject({ filterable: true });- expect(schema.programId).toMatchObject({ filterable: true });});it("pins a tokenizer that splits URLs into their components", async () => {@@ -105,6 +137,22 @@ describe("Turbopuffer partner search provider", () => {});});+ it("always searches identity separately, so single-word queries can rank", async () => {+ // A single-token last_as_prefix query scores every match at exactly 1, so+ // the broad branch returns them unordered. Matching an identity field puts a+ // document in two branches, which is what the rank fusion orders on.+ await createProvider().searchCandidates({+ programId: "prog_test",+ query: "rafi",+ limit: 10,+ });++ const [{ queries }] = mocks.multiQuery.mock.calls[0];+ expect(queries.map((branch: any) => branch.rank_by[0])).toEqual(+ expect.arrayContaining(["searchText", "identityText"]),+ );+ });+it("scopes both branches to the program and matches prefixes", async () => {await createProvider().searchCandidates({programId: "prog_test",@@ -139,12 +187,9 @@ describe("Turbopuffer partner search provider", () => {limit: 10,});- const [{ queries }] = mocks.multiQuery.mock.calls[0];- expect(queries).toHaveLength(2);-- const [, allTermsBranch] = queries;+ const [allTermsBranch] = branchesOf().filteredOn("identityText");expect(allTermsBranch.rank_by).toEqual([- "searchText",+ "identityText","BM25","steven tey",{ last_as_prefix: true },@@ -154,7 +199,7 @@ describe("Turbopuffer partner search provider", () => {[["programId", "Eq", "prog_test"],[- "searchText",+ "identityText","ContainsAllTokens","steven tey",{ last_as_prefix: true },@@ -174,11 +219,10 @@ describe("Turbopuffer partner search provider", () => {limit: 10,});- const [{ queries }] = mocks.multiQuery.mock.calls[0];- const [, allTermsBranch] = queries;+ const [allTermsBranch] = branchesOf().filteredOn("identityText");expect(allTermsBranch.filters[1][1]).toEqual([- "searchText",+ "identityText","ContainsAllTokens","steven te",{ last_as_prefix: true },@@ -192,16 +236,7 @@ describe("Turbopuffer partner search provider", () => {limit: 10,});- // The n-gram branch still uses ContainsAllTokens, so this checks for the- // all-terms filter on searchText specifically.- const [{ queries }] = mocks.multiQuery.mock.calls[0];- expect(- queries.some((branch: any) =>- JSON.stringify(branch.filters).includes(- '["searchText","ContainsAllTokens"',- ),- ),- ).toBe(false);+ expect(branchesOf().filteredOn("identityText")).toHaveLength(0);});it("requires every trigram on the n-gram branch", async () => {@@ -213,10 +248,7 @@ describe("Turbopuffer partner search provider", () => {limit: 10,});- const [{ queries }] = mocks.multiQuery.mock.calls[0];- expect(queries).toHaveLength(2);-- const [, ngramBranch] = queries;+ const [ngramBranch] = branchesOf().rankedOn("emailNgrams");expect(ngramBranch.rank_by).toEqual(["emailNgrams", "BM25", "exa xam amp"]);expect(ngramBranch.filters).toEqual(["And",@@ -234,8 +266,7 @@ describe("Turbopuffer partner search provider", () => {limit: 10,});- const [{ queries }] = mocks.multiQuery.mock.calls[0];- expect(queries).toHaveLength(1);+ expect(branchesOf().rankedOn("emailNgrams")).toHaveLength(0);});it("fuses branches by rank, boosting documents found by both", async () => {@@ -343,6 +374,6 @@ describe("Turbopuffer partner search provider", () => {namespace: createNamespaceMock(),});- expect(namespaceName).toBe("partner-search-v1");+ expect(namespaceName).toBe("partner-search-v2");});});