Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Filter status, group, country, and tags before ranking

Rafi37faaf2d44cb
diff --git a/apps/web/lib/api/partners/search/build-search-query.ts b/apps/web/lib/api/partners/search/build-search-query.ts
index f0951fbeb0..af7894ad3a 100644
--- a/apps/web/lib/api/partners/search/build-search-query.ts
+++ b/apps/web/lib/api/partners/search/build-search-query.ts
@@ -3,6 +3,7 @@ import { getPartnersQuerySchemaExtended } from "@/lib/zod/schemas/partners";
import * as z from "zod/v4";
import {
PARTNER_SEARCH_CANDIDATE_LIMIT,
+ PartnerSearchCandidateFilter,
PartnerSearchCandidateQuery,
} from "./types";
@@ -17,14 +18,44 @@ export type PartnerSearchQueryInput = z.infer<
type PartnerSearchCandidateQueryInput = Pick<
PartnerSearchQueryInput,
- "programId" | "search" | "email" | "tenantId"
+ | "programId"
+ | "search"
+ | "email"
+ | "tenantId"
+ | "status"
+ | "groupId"
+ | "country"
+ | "partnerTagId"
+ | "groupIdOperator"
+ | "countryOperator"
+ | "partnerTagIdOperator"
>;
+function toFilter(
+ value: string | string[] | undefined | null,
+ operator?: "IN" | "NOT IN",
+): PartnerSearchCandidateFilter | undefined {
+ const values = (Array.isArray(value) ? value : [value]).filter(
+ (entry): entry is string => Boolean(entry),
+ );
+
+ return values.length > 0
+ ? { values, exclude: operator === "NOT IN" }
+ : undefined;
+}
+
export function buildPartnerSearchCandidateQuery({
programId,
search,
email,
tenantId,
+ status,
+ groupId,
+ country,
+ partnerTagId,
+ groupIdOperator,
+ countryOperator,
+ partnerTagIdOperator,
}: PartnerSearchCandidateQueryInput): PartnerSearchCandidateQuery | null {
const query = search?.trim();
@@ -42,5 +73,11 @@ export function buildPartnerSearchCandidateQuery({
programId,
query,
limit: PARTNER_SEARCH_CANDIDATE_LIMIT,
+ filters: {
+ status: toFilter(status),
+ groupId: toFilter(groupId, groupIdOperator),
+ country: toFilter(country, countryOperator),
+ partnerTagIds: toFilter(partnerTagId, partnerTagIdOperator),
+ },
};
}
diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
index c23ed6e72b..15b2a03a17 100644
--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts
+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
@@ -5,7 +5,11 @@ import {
getPartnerSearchableValues,
normalizePartnerSearchQuery,
} from "../searchable-values";
-import type { PartnerSearchDocument, PartnerSearchProvider } from "../types";
+import type {
+ PartnerSearchCandidateQuery,
+ PartnerSearchDocument,
+ PartnerSearchProvider,
+} from "../types";
import { validatePartnerSearchCandidateLimit } from "../types";
import { withQueryDeadline, withTransientRetry } from "./resilience";
import { getEmailNgrams, getQueryNgrams } from "./shared";
@@ -16,7 +20,7 @@ import { getEmailNgrams, getQueryNgrams } from "./shared";
* existing namespace. A new version is backfilled alongside the old one and
* swapped in, rather than rebuilt in place.
*/
-const NAMESPACE = "partner-search-v2";
+const NAMESPACE = "partner-search-v3";
const WRITE_BATCH_SIZE = 500;
const QUERY_OPERATION_TIMEOUT_MS = 1_000;
@@ -39,6 +43,10 @@ interface TurbopufferPartnerSearchRow extends Record<string, unknown> {
searchText: string;
identityText: string;
emailNgrams: string;
+ status: string;
+ groupId?: string;
+ country?: string;
+ partnerTagIds: string[];
}
/**
@@ -58,9 +66,16 @@ const SEARCH_TEXT_TOKENIZER = "word_v2";
// ContainsAllTokens, which reads the BM25 index rather than a filter index, so
// marking them filterable would buy nothing and cost plenty: turbopuffer bills
// an attribute per enabled index, so FTS + filterable is 200% of logical size.
-// Only `programId` needs one, for its scalar Eq.
+//
+// The scalar attributes below are filterable because they narrow *before* the
+// ranking truncates. Applying them afterwards is what made a broad query with
+// `country=US` return 85 rows out of 7,698 real matches.
const TURBOPUFFER_SCHEMA = {
programId: { type: "string", filterable: true },
+ status: { type: "string", filterable: true },
+ groupId: { type: "string", filterable: true },
+ country: { type: "string", filterable: true },
+ partnerTagIds: { type: "[]string", filterable: true },
searchText: {
type: "string",
full_text_search: { tokenizer: SEARCH_TEXT_TOKENIZER },
@@ -121,6 +136,13 @@ function serializeTurbopufferRow(
searchText: normalizeValues(getPartnerSearchableValues(document)),
identityText: normalizeValues(getPartnerIdentityValues(document)),
emailNgrams: getEmailNgrams(document.email),
+ status: document.status,
+ // Omitted rather than stored null, which is what makes a NotIn filter match
+ // the partners that have no group or no country — the same rows the
+ // database includes when it ORs the negation with IS NULL.
+ ...(document.groupId ? { groupId: document.groupId } : {}),
+ ...(document.country ? { country: document.country } : {}),
+ partnerTagIds: document.partnerTagIds,
};
}
@@ -143,8 +165,54 @@ function serializeTurbopufferRow(
* shares just one, which is how an unrelated address looks like a partial-email
* match.
*/
-function buildQueryBranches(programId: string, query: string) {
- const programFilter = ["programId", "Eq", programId];
+/**
+ * Turns the discrete filters into turbopuffer clauses. Exclusion uses NotIn and
+ * NotContainsAny, which match documents that omit the attribute entirely — the
+ * partners with no group or no country, which the database also includes when
+ * it negates.
+ */
+function buildFilterClauses(
+ filters: PartnerSearchCandidateQuery["filters"],
+): unknown[] {
+ if (!filters) {
+ return [];
+ }
+
+ const clauses: unknown[] = [];
+
+ for (const [attribute, filter] of Object.entries(filters)) {
+ // An empty value list would become `In []`, which matches nothing and would
+ // silently empty the results rather than leaving them unfiltered.
+ if (!filter || filter.values.length === 0) {
+ continue;
+ }
+
+ const isArray = attribute === "partnerTagIds";
+ const operator = filter.exclude
+ ? isArray
+ ? "NotContainsAny"
+ : "NotIn"
+ : isArray
+ ? "ContainsAny"
+ : "In";
+
+ clauses.push([attribute, operator, filter.values]);
+ }
+
+ return clauses;
+}
+
+function buildQueryBranches(
+ programId: string,
+ query: string,
+ filters?: PartnerSearchCandidateQuery["filters"],
+) {
+ const scope = ["programId", "Eq", programId];
+ const filterClauses = buildFilterClauses(filters);
+ // Every branch carries the same scope, because a root-level clause beside the
+ // branches would not narrow them individually.
+ const programFilter =
+ filterClauses.length > 0 ? ["And", [scope, ...filterClauses]] : scope;
const branches: Record<string, unknown>[] = [
"searchText",
"identityText",
@@ -243,13 +311,19 @@ export function createTurbopufferPartnerSearchProvider({
const resolvedNamespace = namespace ?? createNamespace(resolvedNamespaceName);
return {
- async searchCandidates({ programId, query, limit }) {
+ async searchCandidates({ programId, query, limit, filters }) {
validatePartnerSearchCandidateLimit(limit);
const normalizedQuery = normalizePartnerSearchQuery(query);
- const branches = buildQueryBranches(programId, normalizedQuery).map(
- (branch) => ({ ...branch, top_k: limit, include_attributes: false }),
- );
+ const branches = buildQueryBranches(
+ programId,
+ normalizedQuery,
+ filters,
+ ).map((branch) => ({
+ ...branch,
+ top_k: limit,
+ include_attributes: false,
+ }));
// One round trip regardless of branch count.
const { results } = await withQueryDeadline(
diff --git a/apps/web/lib/api/partners/search/serialize-document.ts b/apps/web/lib/api/partners/search/serialize-document.ts
index 42d6729c58..57e8913a58 100644
--- a/apps/web/lib/api/partners/search/serialize-document.ts
+++ b/apps/web/lib/api/partners/search/serialize-document.ts
@@ -5,12 +5,23 @@ export const partnerSearchDocumentSelect = {
id: true,
programId: true,
partnerId: true,
+ status: true,
+ groupId: true,
partner: {
select: {
name: true,
email: true,
companyName: true,
description: true,
+ country: true,
+ // Tags are per (program, partner) and this select cannot take a program,
+ // so the serializer narrows them to the enrollment's own program.
+ programPartnerTags: {
+ select: {
+ programId: true,
+ partnerTagId: true,
+ },
+ },
platforms: {
select: {
type: true,
@@ -58,5 +69,13 @@ export function serializePartnerSearchDocument(
linkKeys: unique(links.map(({ key }) => key)),
shortLinks: unique(links.map(({ shortLink }) => shortLink)),
destinationUrls: unique(links.map(({ url }) => url)),
+ status: enrollment.status,
+ groupId: enrollment.groupId,
+ country: partner.country,
+ partnerTagIds: unique(
+ partner.programPartnerTags
+ .filter(({ programId }) => programId === enrollment.programId)
+ .map(({ partnerTagId }) => partnerTagId),
+ ),
};
}
diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.ts
index 2a1f4b3f3f..2ebcac9371 100644
--- a/apps/web/lib/api/partners/search/types.ts
+++ b/apps/web/lib/api/partners/search/types.ts
@@ -1,4 +1,4 @@
-import { PlatformType } from "@prisma/client";
+import { PlatformType, ProgramEnrollmentStatus } from "@prisma/client";
/**
* How many ranked enrollment IDs a provider may return for one query.
@@ -56,6 +56,16 @@ export interface PartnerSearchDocument {
linkKeys: string[];
shortLinks: string[];
destinationUrls: string[];
+
+ // Filterable fields, chosen because they change when someone acts on a
+ // partner rather than continuously. Metrics are deliberately absent: they move
+ // on every click and conversion, so indexing them would make the document
+ // churn constantly and turn the index into a second source of truth for
+ // numbers the table displays.
+ status: ProgramEnrollmentStatus;
+ groupId: string | null;
+ country: string | null;
+ partnerTagIds: string[];
}
export interface PartnerSearchHit {
@@ -67,10 +77,31 @@ export interface PartnerSearchResult {
hits: PartnerSearchHit[];
}
+/**
+ * A discrete filter, plus whether the caller is excluding those values. Exclusion
+ * also matches partners that have no value at all, which is what the database
+ * does when it ORs the negation with IS NULL.
+ */
+export interface PartnerSearchCandidateFilter {
+ values: string[];
+ exclude?: boolean;
+}
+
export interface PartnerSearchCandidateQuery {
programId: string;
query: string;
limit: number;
+ /**
+ * Applied by the provider before it truncates to `limit`. The database still
+ * re-applies every one of them, so the index only ever narrows the candidate
+ * pool — it never decides the result on its own.
+ */
+ filters?: {
+ status?: PartnerSearchCandidateFilter;
+ groupId?: PartnerSearchCandidateFilter;
+ country?: PartnerSearchCandidateFilter;
+ partnerTagIds?: PartnerSearchCandidateFilter;
+ };
}
export interface PartnerSearchProvider {
diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.ts
index 14e2487928..aa088921bc 100644
--- a/apps/web/tests/partners/get-partners-count-search.test.ts
+++ b/apps/web/tests/partners/get-partners-count-search.test.ts
@@ -59,6 +59,12 @@ describe("getPartnersCount search", () => {
programId: "prog_test",
query: "examp",
limit: 999,
+ filters: {
+ status: { values: ["approved"], exclude: false },
+ groupId: undefined,
+ country: { values: ["CA"], exclude: false },
+ partnerTagIds: undefined,
+ },
});
expect(mocks.count).toHaveBeenCalledWith({
where: expect.objectContaining({
diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.ts
index f63cd0f733..cf49356cee 100644
--- a/apps/web/tests/partners/get-partners-search.test.ts
+++ b/apps/web/tests/partners/get-partners-search.test.ts
@@ -98,10 +98,18 @@ describe("getPartners search", () => {
{ searchProvider },
);
+ // The status filter reaches the provider, so it narrows before the ranking
+ // truncates rather than after.
expect(searchProvider.searchCandidates).toHaveBeenCalledWith({
programId: "prog_test",
query: "examp",
limit: 999,
+ filters: {
+ status: { values: ["approved"], exclude: false },
+ groupId: undefined,
+ country: undefined,
+ partnerTagIds: undefined,
+ },
});
expect(mocks.findMany).toHaveBeenCalledWith(
expect.objectContaining({
diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.ts
index ecd599901d..da5b0a5157 100644
--- a/apps/web/tests/partners/partner-search-backfill.test.ts
+++ b/apps/web/tests/partners/partner-search-backfill.test.ts
@@ -23,11 +23,15 @@ function createSource(id: string): PartnerSearchDocumentSource {
id,
programId: "prog_test",
partnerId: `pn_${id}`,
+ status: "approved" as const,
+ groupId: null,
partner: {
name: "Rafi Hasan",
companyName: "Dub Partners",
description: "Developer tools educator",
+ country: null,
+ programPartnerTags: [],
platforms: [],
},
links: [],
diff --git a/apps/web/tests/partners/partner-search-document.test.ts b/apps/web/tests/partners/partner-search-document.test.ts
index e591a3821f..51b1d53f3b 100644
--- a/apps/web/tests/partners/partner-search-document.test.ts
+++ b/apps/web/tests/partners/partner-search-document.test.ts
@@ -8,11 +8,19 @@ const source: PartnerSearchDocumentSource = {
id: "pge_test",
programId: "prog_test",
partnerId: "pn_test",
+ status: "approved" as const,
+ groupId: "grp_test",
partner: {
name: "Rafi Hasan",
companyName: "Dub Partners",
description: "Developer tools educator",
+ country: "US",
+ programPartnerTags: [
+ { programId: "prog_test", partnerTagId: "ptag_a" },
+ // A tag from a different program must not leak into this document.
+ { programId: "prog_other", partnerTagId: "ptag_other" },
+ ],
platforms: [
{
type: "website",
@@ -59,6 +67,11 @@ describe("serializePartnerSearchDocument", () => {
"https://example.com/referrals/rafi",
"https://rafi.dev/tools",
],
+ status: "approved",
+ groupId: "grp_test",
+ country: "US",
+ // The tag from prog_other is dropped: tags are per program.
+ partnerTagIds: ["ptag_a"],
});
});
});
diff --git a/apps/web/tests/partners/partner-search-query.test.ts b/apps/web/tests/partners/partner-search-query.test.ts
index 2d7c94b5f1..bdc218f47e 100644
--- a/apps/web/tests/partners/partner-search-query.test.ts
+++ b/apps/web/tests/partners/partner-search-query.test.ts
@@ -16,6 +16,34 @@ describe("buildPartnerSearchCandidateQuery", () => {
programId: "prog_test",
query: "examp",
limit: 999,
+ filters: {
+ status: undefined,
+ groupId: undefined,
+ country: undefined,
+ partnerTagIds: undefined,
+ },
+ });
+ });
+
+ it("passes the discrete filters through, with exclusion", () => {
+ expect(
+ buildPartnerSearchCandidateQuery({
+ ...defaultInput,
+ status: "approved",
+ groupId: ["grp_1", "grp_2"],
+ country: "US",
+ countryOperator: "NOT IN",
+ partnerTagId: ["ptag_1"],
+ }),
+ ).toMatchObject({
+ filters: {
+ status: { values: ["approved"], exclude: false },
+ groupId: { values: ["grp_1", "grp_2"], exclude: false },
+ // Exclusion also matches partners with no country, matching the
+ // database's OR against IS NULL.
+ country: { values: ["US"], exclude: true },
+ partnerTagIds: { values: ["ptag_1"], exclude: false },
+ },
});
});
diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
index 1a4915604f..cb224322cc 100644
--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
@@ -23,6 +23,10 @@ const document: PartnerSearchDocument = {
linkKeys: ["rafi"],
shortLinks: ["https://dub.sh/rafi"],
destinationUrls: ["https://example.com/referrals/rafi"],
+ status: "approved",
+ groupId: "grp_test",
+ country: "US",
+ partnerTagIds: ["ptag_a", "ptag_b"],
};
const mocks = vi.hoisted(() => ({
@@ -239,6 +243,48 @@ describe("Turbopuffer partner search provider", () => {
expect(branchesOf().filteredOn("identityText")).toHaveLength(0);
});
+ it("narrows every branch with the discrete filters", async () => {
+ // The filters have to sit inside each branch: applying them after the
+ // ranking truncates is what made a broad query with country=US return 85
+ // rows out of 7,698 real matches.
+ await createProvider().searchCandidates({
+ programId: "prog_test",
+ query: "examp",
+ limit: 10,
+ filters: {
+ status: { values: ["approved"] },
+ country: { values: ["US", "CA"], exclude: true },
+ partnerTagIds: { values: ["ptag_1"] },
+ },
+ });
+
+ const [{ queries }] = mocks.multiQuery.mock.calls[0];
+ expect(queries.length).toBeGreaterThan(1);
+
+ for (const branch of queries) {
+ const filters = JSON.stringify(branch.filters);
+ expect(filters).toContain('["programId","Eq","prog_test"]');
+ expect(filters).toContain('["status","In",["approved"]]');
+ // Exclusion uses NotIn, which also matches documents that omit country.
+ expect(filters).toContain('["country","NotIn",["US","CA"]]');
+ expect(filters).toContain('["partnerTagIds","ContainsAny",["ptag_1"]]');
+ }
+ });
+
+ it("omits absent filters rather than sending empty clauses", async () => {
+ await createProvider().searchCandidates({
+ programId: "prog_test",
+ query: "examp",
+ limit: 10,
+ filters: { status: undefined, country: { values: [] } },
+ });
+
+ const [{ queries }] = mocks.multiQuery.mock.calls[0];
+ for (const branch of queries) {
+ expect(JSON.stringify(branch.filters)).not.toContain('"country"');
+ }
+ });
+
it("requires every trigram on the n-gram branch", async () => {
// BM25 alone would score a document sharing a single trigram, which is how
// an unrelated address looks like a partial-email match.
@@ -368,6 +414,6 @@ describe("Turbopuffer partner search provider", () => {
namespace: createNamespaceMock(),
});
- expect(namespaceName).toBe("partner-search-v2");
+ expect(namespaceName).toBe("partner-search-v3");
});
});