The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Improve turbopuffer ranking
Rafid88cfade3ca7
diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.tsindex 7ffffff7e2..57f4b87ecf 100644--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts@@ -46,6 +46,23 @@ interface TurbopufferPartnerSearchRow extends Record<string, unknown> {*/const SEARCH_TEXT_TOKENIZER = "word_v2";+/**+ * BM25 length normalization, left at turbopuffer's default.+ *+ * Lowering it is tempting: `searchText` is concatenated field values, so length+ * tracks how many links a partner has rather than how much prose they wrote,+ * and at 0.75 that demotes the most active partners. Searching "steven te" on+ * the production index ranks Steven Tey below a partner whose description+ * merely contains "tech", and the ordering flips at b=0.1.+ *+ * It was measured and rejected. On the same 100K index, b=0.1 against b=0.75+ * minutes apart: 16.05% vs 11.01% timeouts and 836ms vs 720ms p99 — flattening+ * the score distribution appears to defeat top-k pruning. The ranking gain only+ * applies to half-typed multi-word queries, since the all-terms branch already+ * handles complete ones.+ */+const SEARCH_TEXT_LENGTH_NORMALIZATION = 0.75;+// `searchText` and `emailNgrams` are BM25-indexed. BM25 attributes are not// filterable by default, but `emailNgrams` needs to be: the n-gram branch uses// a ContainsAllTokens filter to require every trigram.@@ -53,7 +70,10 @@ const TURBOPUFFER_SCHEMA = {programId: { type: "string", filterable: true },searchText: {type: "string",- full_text_search: { tokenizer: SEARCH_TEXT_TOKENIZER },+ full_text_search: {+ tokenizer: SEARCH_TEXT_TOKENIZER,+ b: SEARCH_TEXT_LENGTH_NORMALIZATION,+ },},emailNgrams: {type: "string",@@ -112,7 +132,7 @@ function serializeTurbopufferRow(}/**- * Two ranked branches, rank-fused by the caller (see mergeBranchRows).+ * Up to three ranked branches, rank-fused by the caller (see mergeBranchRows).** The text branch treats the last token as a prefix so "john" reaches* "johnson", matching how the Redis provider's $smart behaves. The n-gram@@ -129,6 +149,32 @@ function buildQueryBranches(programId: string, query: string) {},];+ // A multi-word query should prefer partners matching every word, and BM25+ // alone does not: term frequency saturates while length normalization keeps+ // biting, so a long document matching both words can rank below a short one+ // matching only the first. Measured on the production index, "steven tey" put+ // the actual Steven Tey third, behind two partners with no "tey" at all. This+ // branch admits only documents containing every term, so the rank fusion+ // lifts them over partial matches without excluding those partial matches.+ if (/\s/u.test(query)) {+ branches.push({+ rank_by: ["searchText", "BM25", query, { last_as_prefix: true }],+ filters: [+ "And",+ [+ programFilter,+ // Prefix on the last token here too. Without it the branch requires an+ // exact final token, so a half-typed "steven te" matches nothing and+ // search-as-you-type loses the boost exactly while it is being typed.+ // Prefix on the last token here too. Without it the branch requires an+ // exact final token, so a half-typed "steven te" matches nothing and+ // the boost disappears exactly while the user is still typing.+ ["searchText", "ContainsAllTokens", query, { last_as_prefix: true }],+ ],+ ],+ });+ }+const ngrams = getQueryNgrams(query).join(" ");if (ngrams) {branches.push({diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.tsindex a94d038185..9e1c559f27 100644--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts@@ -97,7 +97,7 @@ describe("Turbopuffer partner search provider", () => {await createProvider().upsert([document]);const [{ schema }] = mocks.write.mock.calls[0];- expect(schema.searchText.full_text_search).toEqual({+ expect(schema.searchText.full_text_search).toMatchObject({tokenizer: "word_v2",});expect(schema.emailNgrams.full_text_search).toEqual({@@ -129,6 +129,81 @@ describe("Turbopuffer partner search provider", () => {}});+ it("adds an all-terms branch for a multi-word query", async () => {+ // BM25 alone does not require every word, and length normalization can rank+ // a long document matching both words below a short one matching only the+ // first. This branch admits only documents containing every term.+ await createProvider().searchCandidates({+ programId: "prog_test",+ query: "steven tey",+ limit: 10,+ });++ const [{ queries }] = mocks.multiQuery.mock.calls[0];+ expect(queries).toHaveLength(2);++ const [, allTermsBranch] = queries;+ expect(allTermsBranch.rank_by).toEqual([+ "searchText",+ "BM25",+ "steven tey",+ { last_as_prefix: true },+ ]);+ expect(allTermsBranch.filters).toEqual([+ "And",+ [+ ["programId", "Eq", "prog_test"],+ [+ "searchText",+ "ContainsAllTokens",+ "steven tey",+ { last_as_prefix: true },+ ],+ ],+ ]);+ });++ it("keeps the last token a prefix while it is still being typed", async () => {+ // An exact-token filter would match nothing for a half-typed final word, so+ // the all-terms boost would vanish exactly while the user is typing.+ // Measured on the production index: "steven te" puts Steven Tey at rank 2+ // with the prefix and rank 4 without it.+ await createProvider().searchCandidates({+ programId: "prog_test",+ query: "steven te",+ limit: 10,+ });++ const [{ queries }] = mocks.multiQuery.mock.calls[0];+ const [, allTermsBranch] = queries;++ expect(allTermsBranch.filters[1][1]).toEqual([+ "searchText",+ "ContainsAllTokens",+ "steven te",+ { last_as_prefix: true },+ ]);+ });++ it("skips the all-terms branch for a single-word query", async () => {+ await createProvider().searchCandidates({+ programId: "prog_test",+ query: "steven",+ limit: 10,+ });++ // The n-gram branch still uses ContainsAllTokens, so this checks for the+ // all-terms filter on searchText specifically.+ const [{ queries }] = mocks.multiQuery.mock.calls[0];+ expect(+ queries.some((branch: any) =>+ JSON.stringify(branch.filters).includes(+ '["searchText","ContainsAllTokens"',+ ),+ ),+ ).toBe(false);+ });+it("requires every trigram on the n-gram branch", async () => {// BM25 alone would score a document sharing a single trigram, which is how// an unrelated address looks like a partial-email match.