Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Improved identity text based ranking

Rafif755ce1a7669
diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
index 57f4b87ecf..50f923fca5 100644
--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts
+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
@@ -1,6 +1,7 @@
import { chunk } from "@dub/utils";
import { Turbopuffer } from "@turbopuffer/turbopuffer";
import {
+ getPartnerIdentityValues,
getPartnerSearchableValues,
normalizePartnerSearchQuery,
} from "../searchable-values";
@@ -9,7 +10,7 @@ import { validatePartnerSearchCandidateLimit } from "../types";
import { withQueryDeadline, withTransientRetry } from "./resilience";
import { getEmailNgrams, getQueryNgrams, resolveIndexName } from "./shared";
-const DEFAULT_NAMESPACE = "partner-search-v1";
+const DEFAULT_NAMESPACE = "partner-search-v2";
const WRITE_BATCH_SIZE = 500;
const QUERY_OPERATION_TIMEOUT_MS = 1_000;
@@ -30,6 +31,7 @@ interface TurbopufferPartnerSearchRow extends Record<string, unknown> {
id: string;
programId: string;
searchText: string;
+ identityText: string;
emailNgrams: string;
}
@@ -46,39 +48,24 @@ interface TurbopufferPartnerSearchRow extends Record<string, unknown> {
*/
const SEARCH_TEXT_TOKENIZER = "word_v2";
-/**
- * BM25 length normalization, left at turbopuffer's default.
- *
- * Lowering it is tempting: `searchText` is concatenated field values, so length
- * tracks how many links a partner has rather than how much prose they wrote,
- * and at 0.75 that demotes the most active partners. Searching "steven te" on
- * the production index ranks Steven Tey below a partner whose description
- * merely contains "tech", and the ordering flips at b=0.1.
- *
- * It was measured and rejected. On the same 100K index, b=0.1 against b=0.75
- * minutes apart: 16.05% vs 11.01% timeouts and 836ms vs 720ms p99 — flattening
- * the score distribution appears to defeat top-k pruning. The ranking gain only
- * applies to half-typed multi-word queries, since the all-terms branch already
- * handles complete ones.
- */
-const SEARCH_TEXT_LENGTH_NORMALIZATION = 0.75;
-
-// `searchText` and `emailNgrams` are BM25-indexed. BM25 attributes are not
-// filterable by default, but `emailNgrams` needs to be: the n-gram branch uses
-// a ContainsAllTokens filter to require every trigram.
+// No text attribute is filterable. The n-gram and all-terms branches narrow with
+// ContainsAllTokens, which reads the BM25 index rather than a filter index, so
+// marking them filterable would buy nothing and cost plenty: turbopuffer bills
+// an attribute per enabled index, so FTS + filterable is 200% of logical size.
+// Only `programId` needs one, for its scalar Eq.
const TURBOPUFFER_SCHEMA = {
programId: { type: "string", filterable: true },
searchText: {
type: "string",
- full_text_search: {
- tokenizer: SEARCH_TEXT_TOKENIZER,
- b: SEARCH_TEXT_LENGTH_NORMALIZATION,
- },
+ full_text_search: { tokenizer: SEARCH_TEXT_TOKENIZER },
+ },
+ identityText: {
+ type: "string",
+ full_text_search: { tokenizer: SEARCH_TEXT_TOKENIZER },
},
emailNgrams: {
type: "string",
full_text_search: { tokenizer: SEARCH_TEXT_TOKENIZER },
- filterable: true,
},
} as const;
@@ -118,58 +105,67 @@ function createNamespace(namespaceName: string): TurbopufferNamespace {
return client.namespace(namespaceName) as unknown as TurbopufferNamespace;
}
+function normalizeValues(values: string[]): string {
+ return values.map(normalizePartnerSearchQuery).join(" ");
+}
+
function serializeTurbopufferRow(
document: PartnerSearchDocument,
): TurbopufferPartnerSearchRow {
return {
id: document.id,
programId: document.programId,
- searchText: getPartnerSearchableValues(document)
- .map(normalizePartnerSearchQuery)
- .join(" "),
+ searchText: normalizeValues(getPartnerSearchableValues(document)),
+ identityText: normalizeValues(getPartnerIdentityValues(document)),
emailNgrams: getEmailNgrams(document.email),
};
}
/**
- * Up to three ranked branches, rank-fused by the caller (see mergeBranchRows).
+ * Each branch is ranked independently and rank-fused by the caller (see
+ * mergeBranchRows).
+ *
+ * Identity is searched separately from everything else because a single-token
+ * `last_as_prefix` query scores every match at exactly 1, so the broad branch
+ * cannot order its own results. Matching an identity field puts a document in
+ * two branches, which is what the fusion orders on.
*
- * The text branch treats the last token as a prefix so "john" reaches
- * "johnson", matching how the Redis provider's $smart behaves. The n-gram
- * branch carries a ContainsAllTokens filter so every trigram must be present —
- * BM25 alone would score a document that shares just one, which is how an
- * unrelated address ends up looking like a partial-email match.
+ * The all-terms branch requires every query word, which BM25 does not: term
+ * frequency saturates while length normalization keeps biting, so a partner
+ * matching both words can otherwise rank below one matching only the first. It
+ * runs on identityText so a full name is not out-weighted by a long description
+ * or a pile of links.
+ *
+ * The n-gram branch requires every trigram; BM25 alone scores a document that
+ * shares just one, which is how an unrelated address looks like a partial-email
+ * match.
*/
function buildQueryBranches(programId: string, query: string) {
const programFilter = ["programId", "Eq", programId];
const branches: Record<string, unknown>[] = [
- {
- rank_by: ["searchText", "BM25", query, { last_as_prefix: true }],
- filters: programFilter,
- },
- ];
-
- // A multi-word query should prefer partners matching every word, and BM25
- // alone does not: term frequency saturates while length normalization keeps
- // biting, so a long document matching both words can rank below a short one
- // matching only the first. Measured on the production index, "steven tey" put
- // the actual Steven Tey third, behind two partners with no "tey" at all. This
- // branch admits only documents containing every term, so the rank fusion
- // lifts them over partial matches without excluding those partial matches.
+ "searchText",
+ "identityText",
+ ].map((attribute) => ({
+ rank_by: [attribute, "BM25", query, { last_as_prefix: true }],
+ filters: programFilter,
+ }));
+
if (/\s/u.test(query)) {
branches.push({
- rank_by: ["searchText", "BM25", query, { last_as_prefix: true }],
+ rank_by: ["identityText", "BM25", query, { last_as_prefix: true }],
filters: [
"And",
[
programFilter,
- // Prefix on the last token here too. Without it the branch requires an
- // exact final token, so a half-typed "steven te" matches nothing and
- // search-as-you-type loses the boost exactly while it is being typed.
- // Prefix on the last token here too. Without it the branch requires an
+ // Prefix on the last token here too. Without it the branch needs an
// exact final token, so a half-typed "steven te" matches nothing and
- // the boost disappears exactly while the user is still typing.
- ["searchText", "ContainsAllTokens", query, { last_as_prefix: true }],
+ // the boost disappears while the user is still typing.
+ [
+ "identityText",
+ "ContainsAllTokens",
+ query,
+ { last_as_prefix: true },
+ ],
],
],
});
diff --git a/apps/web/lib/api/partners/search/searchable-values.ts b/apps/web/lib/api/partners/search/searchable-values.ts
index 4b3f8d49ed..6908e72d14 100644
--- a/apps/web/lib/api/partners/search/searchable-values.ts
+++ b/apps/web/lib/api/partners/search/searchable-values.ts
@@ -1,10 +1,12 @@
import { PartnerSearchDocument } from "./types";
/**
- * Returns every value covered by the partner-search assignment. Providers can
- * index these as separate weighted fields or as one normalized search field.
+ * Who the partner is: the fields someone types when they have a specific person
+ * in mind. Short and stable, so a provider can index them separately and score a
+ * name match against name length rather than against a document whose length is
+ * really a count of how many links the partner has.
*/
-export function getPartnerSearchableValues(
+export function getPartnerIdentityValues(
document: PartnerSearchDocument,
): string[] {
return [
@@ -12,11 +14,21 @@ export function getPartnerSearchableValues(
document.name,
document.email,
document.companyName,
+ ].filter((value): value is string => Boolean(value));
+}
+
+/**
+ * Every value covered by the partner-search assignment. Providers can index
+ * these as separate weighted fields or as one normalized search field.
+ */
+export function getPartnerSearchableValues(
+ document: PartnerSearchDocument,
+): string[] {
+ return [
+ ...getPartnerIdentityValues(document),
document.description,
...document.platformTypes,
...document.platformIdentifiers,
- ...document.linkDomains,
- ...document.linkKeys,
...document.shortLinks,
...document.destinationUrls,
].filter((value): value is string => Boolean(value));
diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
index 9e1c559f27..aa363b38c6 100644
--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
@@ -39,6 +39,23 @@ function createNamespaceMock(): TurbopufferNamespace {
} as unknown as TurbopufferNamespace;
}
+function branchesOf(call: number = 0) {
+ const [{ queries }] = mocks.multiQuery.mock.calls[call];
+ return {
+ all: queries,
+ // Located by attribute rather than position, so adding a branch does not
+ // break every assertion.
+ rankedOn: (attribute: string) =>
+ queries.filter((branch: any) => branch.rank_by[0] === attribute),
+ filteredOn: (attribute: string) =>
+ queries.filter((branch: any) =>
+ JSON.stringify(branch.filters).includes(
+ `["${attribute}","ContainsAllTokens"`,
+ ),
+ ),
+ };
+}
+
function createProvider() {
return createTurbopufferPartnerSearchProvider({
namespace: createNamespaceMock(),
@@ -61,6 +78,27 @@ describe("Turbopuffer partner search provider", () => {
vi.unstubAllEnvs();
});
+ it("indexes identity fields separately, and only those fields", async () => {
+ await createProvider().upsert([document]);
+
+ const [{ upsert_rows, schema }] = mocks.write.mock.calls[0];
+ const [row] = upsert_rows;
+
+ expect(row.identityText).toBe(
+ "pn_test rafi hasan [email protected] hasan labs",
+ );
+ // Description, platforms, and links stay out, so a name cannot be
+ // out-weighted by how much else a partner has.
+ for (const value of ["affiliate marketer", "youtube", "@rafi", "dub.sh"]) {
+ expect(row.identityText).not.toContain(value);
+ }
+ // ContainsAllTokens reads the BM25 index, so no text attribute needs a
+ // filter index — and turbopuffer bills FTS + filterable at 200%.
+ expect(schema.identityText.filterable).toBeUndefined();
+ expect(schema.emailNgrams.filterable).toBeUndefined();
+ expect(schema.programId).toMatchObject({ filterable: true });
+ });
+
it("flattens every searchable field into one BM25 attribute", async () => {
await createProvider().upsert([document]);
@@ -77,17 +115,11 @@ describe("Turbopuffer partner search provider", () => {
"affiliate marketer",
"youtube",
"@rafi",
- "dub.sh",
"https://dub.sh/rafi",
"https://example.com/referrals/rafi",
]) {
expect(row.searchText).toContain(value);
}
-
- // BM25 attributes are not filterable by default, but the n-gram branch
- // filters on emailNgrams.
- expect(schema.emailNgrams).toMatchObject({ filterable: true });
- expect(schema.programId).toMatchObject({ filterable: true });
});
it("pins a tokenizer that splits URLs into their components", async () => {
@@ -105,6 +137,22 @@ describe("Turbopuffer partner search provider", () => {
});
});
+ it("always searches identity separately, so single-word queries can rank", async () => {
+ // A single-token last_as_prefix query scores every match at exactly 1, so
+ // the broad branch returns them unordered. Matching an identity field puts a
+ // document in two branches, which is what the rank fusion orders on.
+ await createProvider().searchCandidates({
+ programId: "prog_test",
+ query: "rafi",
+ limit: 10,
+ });
+
+ const [{ queries }] = mocks.multiQuery.mock.calls[0];
+ expect(queries.map((branch: any) => branch.rank_by[0])).toEqual(
+ expect.arrayContaining(["searchText", "identityText"]),
+ );
+ });
+
it("scopes both branches to the program and matches prefixes", async () => {
await createProvider().searchCandidates({
programId: "prog_test",
@@ -139,12 +187,9 @@ describe("Turbopuffer partner search provider", () => {
limit: 10,
});
- const [{ queries }] = mocks.multiQuery.mock.calls[0];
- expect(queries).toHaveLength(2);
-
- const [, allTermsBranch] = queries;
+ const [allTermsBranch] = branchesOf().filteredOn("identityText");
expect(allTermsBranch.rank_by).toEqual([
- "searchText",
+ "identityText",
"BM25",
"steven tey",
{ last_as_prefix: true },
@@ -154,7 +199,7 @@ describe("Turbopuffer partner search provider", () => {
[
["programId", "Eq", "prog_test"],
[
- "searchText",
+ "identityText",
"ContainsAllTokens",
"steven tey",
{ last_as_prefix: true },
@@ -174,11 +219,10 @@ describe("Turbopuffer partner search provider", () => {
limit: 10,
});
- const [{ queries }] = mocks.multiQuery.mock.calls[0];
- const [, allTermsBranch] = queries;
+ const [allTermsBranch] = branchesOf().filteredOn("identityText");
expect(allTermsBranch.filters[1][1]).toEqual([
- "searchText",
+ "identityText",
"ContainsAllTokens",
"steven te",
{ last_as_prefix: true },
@@ -192,16 +236,7 @@ describe("Turbopuffer partner search provider", () => {
limit: 10,
});
- // The n-gram branch still uses ContainsAllTokens, so this checks for the
- // all-terms filter on searchText specifically.
- const [{ queries }] = mocks.multiQuery.mock.calls[0];
- expect(
- queries.some((branch: any) =>
- JSON.stringify(branch.filters).includes(
- '["searchText","ContainsAllTokens"',
- ),
- ),
- ).toBe(false);
+ expect(branchesOf().filteredOn("identityText")).toHaveLength(0);
});
it("requires every trigram on the n-gram branch", async () => {
@@ -213,10 +248,7 @@ describe("Turbopuffer partner search provider", () => {
limit: 10,
});
- const [{ queries }] = mocks.multiQuery.mock.calls[0];
- expect(queries).toHaveLength(2);
-
- const [, ngramBranch] = queries;
+ const [ngramBranch] = branchesOf().rankedOn("emailNgrams");
expect(ngramBranch.rank_by).toEqual(["emailNgrams", "BM25", "exa xam amp"]);
expect(ngramBranch.filters).toEqual([
"And",
@@ -234,8 +266,7 @@ describe("Turbopuffer partner search provider", () => {
limit: 10,
});
- const [{ queries }] = mocks.multiQuery.mock.calls[0];
- expect(queries).toHaveLength(1);
+ expect(branchesOf().rankedOn("emailNgrams")).toHaveLength(0);
});
it("fuses branches by rank, boosting documents found by both", async () => {
@@ -343,6 +374,6 @@ describe("Turbopuffer partner search provider", () => {
namespace: createNamespaceMock(),
});
- expect(namespaceName).toBe("partner-search-v1");
+ expect(namespaceName).toBe("partner-search-v2");
});
});