The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Make pasted short links exact and stop indexing shared domains
Rafi5b1e41b5e2a3
diff --git a/.pnpm-store/v11/index.db b/.pnpm-store/v11/index.dbnew file mode 100644index 0000000000..4d0f4d75c0Binary files /dev/null and b/.pnpm-store/v11/index.db differdiff --git a/.pnpm-store/v11/index.db-shm b/.pnpm-store/v11/index.db-shmnew file mode 100644index 0000000000..adf81a5561Binary files /dev/null and b/.pnpm-store/v11/index.db-shm differdiff --git a/.pnpm-store/v11/index.db-wal b/.pnpm-store/v11/index.db-walnew file mode 100644index 0000000000..60bb626bbaBinary files /dev/null and b/.pnpm-store/v11/index.db-wal differdiff --git a/apps/web/lib/api/partners/get-partners-count.ts b/apps/web/lib/api/partners/get-partners-count.tsindex 171073316c..49ff55fd71 100644--- a/apps/web/lib/api/partners/get-partners-count.ts+++ b/apps/web/lib/api/partners/get-partners-count.ts@@ -275,6 +275,9 @@ export async function getPartnersCount<T>(searchProvider &&candidateQuery &&candidateResult &&+ // An exact lookup already names its hits; the text aggregation would count+ // every document sharing a token with the pasted link instead.+ !candidateResult.exact &&!databaseOnlyFilters) {try {diff --git a/apps/web/lib/api/partners/search/find-candidates.ts b/apps/web/lib/api/partners/search/find-candidates.tsindex 0671a7619b..2e94c39f51 100644--- a/apps/web/lib/api/partners/search/find-candidates.ts+++ b/apps/web/lib/api/partners/search/find-candidates.ts@@ -6,6 +6,58 @@ import type {PartnerSearchResult,} from "./types";+/**+ * A pasted short link: a host with a path, with or without the protocol, e.g.+ * `go.acme.com/partnername`. A bare domain is not one, because typing a domain+ * is a text search, while a host plus path only ever comes from pasting a link.+ */+const SHORT_LINK_QUERY_PATTERN =+ /^(?:https?:\/\/)?([a-z0-9-]+(?:\.[a-z0-9-]+)+)(\/\S+)$/i;++/**+ * Resolves a pasted short link to its enrollment through `Link.shortLink`,+ * which is unique and indexed, so the paste matches exactly the partner who+ * owns that link. The domain tokens of a short link are shared by every+ * partner in the program, so the ranked path cannot make a paste precise.+ *+ * Misses fall through to the ranked search: a partial paste still matches the+ * key by prefix. A link from another program returns no hits rather than+ * falling through, since its tokens would only flood the ranked results.+ */+async function findExactShortLinkResult({+ programId,+ query,+}: PartnerSearchCandidateQuery): Promise<PartnerSearchResult | null> {+ const match = query.trim().match(SHORT_LINK_QUERY_PATTERN);++ if (!match) {+ return null;+ }++ const [, host, path] = match;+ const link = await prisma.link.findUnique({+ where: { shortLink: `https://${host}${path}` },+ select: { programId: true, partnerId: true },+ });++ if (!link?.partnerId || !link.programId) {+ return null;+ }++ if (link.programId !== programId) {+ return { hits: [], exact: true };+ }++ const enrollment = await prisma.programEnrollment.findUnique({+ where: {+ partnerId_programId: { partnerId: link.partnerId, programId },+ },+ select: { id: true },+ });++ return { hits: enrollment ? [{ id: enrollment.id }] : [], exact: true };+}+interface FindPartnerSearchCandidatesOptions {/*** Surface provider failures instead of degrading. The benchmark needs this so@@ -42,6 +94,12 @@ export async function findPartnerSearchCandidates(}}+ const exactShortLinkResult = await findExactShortLinkResult(query);++ if (exactShortLinkResult) {+ return exactShortLinkResult;+ }+try {return await searchProvider.searchCandidates(query);} catch (error) {diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.tsindex 34a20d4d53..7c94aa01c0 100644--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts@@ -31,7 +31,7 @@ import { getEmailNgrams, getQueryNgrams } from "./shared";* existing namespace. A new version is backfilled alongside the old one and* swapped in, rather than rebuilt in place.*/-export const PARTNER_SEARCH_NAMESPACE = "partner-search-v4";+export const PARTNER_SEARCH_NAMESPACE = "partner-search-v5";const WRITE_BATCH_SIZE = 500;/**diff --git a/apps/web/lib/api/partners/search/searchable-values.ts b/apps/web/lib/api/partners/search/searchable-values.tsindex 6908e72d14..131dd9ef73 100644--- a/apps/web/lib/api/partners/search/searchable-values.ts+++ b/apps/web/lib/api/partners/search/searchable-values.ts@@ -29,7 +29,7 @@ export function getPartnerSearchableValues(document.description,...document.platformTypes,...document.platformIdentifiers,- ...document.shortLinks,+ ...document.linkKeys,...document.destinationUrls,].filter((value): value is string => Boolean(value));}diff --git a/apps/web/lib/api/partners/search/serialize-document.ts b/apps/web/lib/api/partners/search/serialize-document.tsindex 57e8913a58..8846fb76bd 100644--- a/apps/web/lib/api/partners/search/serialize-document.ts+++ b/apps/web/lib/api/partners/search/serialize-document.ts@@ -7,6 +7,13 @@ export const partnerSearchDocumentSelect = {partnerId: true,status: true,groupId: true,+ // The program's destination host, so the serializer can drop it from+ // destination URLs below.+ program: {+ select: {+ url: true,+ },+ },partner: {select: {name: true,@@ -32,9 +39,7 @@ export const partnerSearchDocumentSelect = {},links: {select: {- domain: true,key: true,- shortLink: true,url: true,},},@@ -48,10 +53,46 @@ function unique<T>(values: T[]): T[] {return Array.from(new Set(values));}+function getHost(url: string | null): string | null {+ if (!url) {+ return null;+ }++ try {+ return new URL(url).hostname.replace(/^www\./, "");+ } catch {+ return null;+ }+}++/**+ * The distinctive part of a destination URL. Query strings and fragments are+ * stripped so tracking parameters never reach the index, and the host is+ * dropped when it is the program's own: measured in production, ~99% of+ * destination hosts in a program are its one domain, so indexing it floods+ * every search containing a domain token. A host that differs (a partner's own+ * site) identifies the partner and is kept.+ */+function sanitizeDestinationUrl(+ url: string,+ programHost: string | null,+): string | null {+ try {+ const parsed = new URL(url);+ const host = parsed.hostname.replace(/^www\./, "");+ const path = parsed.pathname === "/" ? "" : parsed.pathname;++ return `${host === programHost ? "" : host}${path}` || null;+ } catch {+ return url;+ }+}+export function serializePartnerSearchDocument(enrollment: PartnerSearchDocumentSource,): PartnerSearchDocument {- const { partner, links } = enrollment;+ const { partner, links, program } = enrollment;+ const programHost = getHost(program.url);return {id: enrollment.id,@@ -65,10 +106,13 @@ export function serializePartnerSearchDocument(platformIdentifiers: unique(partner.platforms.map(({ identifier }) => identifier),),- linkDomains: unique(links.map(({ domain }) => domain)),linkKeys: unique(links.map(({ key }) => key)),- shortLinks: unique(links.map(({ shortLink }) => shortLink)),- destinationUrls: unique(links.map(({ url }) => url)),+ destinationUrls: unique(+ links.flatMap(({ url }) => {+ const sanitized = sanitizeDestinationUrl(url, programHost);+ return sanitized ? [sanitized] : [];+ }),+ ),status: enrollment.status,groupId: enrollment.groupId,country: partner.country,diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.tsindex 30ee17d2d4..c7fc56fe3a 100644--- a/apps/web/lib/api/partners/search/types.ts+++ b/apps/web/lib/api/partners/search/types.ts@@ -53,10 +53,11 @@ export interface PartnerSearchDocument {platformTypes: PlatformType[];platformIdentifiers: string[];- // Searchable link fields- linkDomains: string[];+ // Searchable link fields. Short-link domains are absent: a program's+ // partners share one, so it never narrows a search. A full short link is an+ // exact database lookup instead. Destination URLs are stored sanitized (see+ // serialize-document).linkKeys: string[];- shortLinks: string[];destinationUrls: string[];// Filterable fields. Metrics are deliberately absent: they move on every click@@ -74,6 +75,12 @@ export interface PartnerSearchHit {export interface PartnerSearchResult {hits: PartnerSearchHit[];+ /**+ * The hits are an exact lookup rather than ranked matches, so the count must+ * come from them and never from the text aggregation, which would count+ * every document sharing a token with the pasted value.+ */+ exact?: boolean;}/**diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.tsindex 3af27f49f7..642896594b 100644--- a/apps/web/scripts/dev/benchmark-partner-search.ts+++ b/apps/web/scripts/dev/benchmark-partner-search.ts@@ -203,10 +203,6 @@ function longestSearchToken(value: string): string {return token.slice(0, 12);}-function stripProtocol(url: string): string {- return url.replace(/^https?:\/\//, "");-}-function emailInfix(email: string): string {const domain = email.split("@")[1];if (!domain) {@@ -220,9 +216,7 @@ function emailInfix(email: string): string {function createSearchCases(document: PartnerSearchDocument): SearchCase[] {const platformType = document.platformTypes[0];const platformIdentifier = document.platformIdentifiers[0];- const linkDomain = document.linkDomains[0];const linkKey = document.linkKeys[0];- const shortLink = document.shortLinks[0];const destinationUrl = document.destinationUrls[0];if (@@ -231,9 +225,7 @@ function createSearchCases(document: PartnerSearchDocument): SearchCase[] {!document.description ||!platformType ||!platformIdentifier ||- !linkDomain ||!linkKey ||- !shortLink ||!destinationUrl) {throw new Error(@@ -251,9 +243,7 @@ function createSearchCases(document: PartnerSearchDocument): SearchCase[] {field: "platform identifier",query: longestSearchToken(platformIdentifier),},- { field: "link domain", query: longestSearchToken(linkDomain) },{ field: "link key", query: linkKey },- { field: "short link", query: stripProtocol(shortLink) },{field: "destination URL",query: longestSearchToken(destinationUrl),diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.tsindex e829a164bd..e22e43dec6 100644--- a/apps/web/tests/partners/get-partners-count-search.test.ts+++ b/apps/web/tests/partners/get-partners-count-search.test.ts@@ -11,6 +11,8 @@ const mocks = vi.hoisted(() => ({partnerGroupBy: vi.fn(),partnerTagGroupBy: vi.fn(),applicationEventGroupBy: vi.fn(),+ enrollmentFindUnique: vi.fn(),+ linkFindUnique: vi.fn(),}));vi.mock("@/lib/prisma", () => ({@@ -19,10 +21,12 @@ vi.mock("@/lib/prisma", () => ({programEnrollment: {count: mocks.count,groupBy: mocks.enrollmentGroupBy,+ findUnique: mocks.enrollmentFindUnique,},- partner: { groupBy: mocks.partnerGroupBy },+ partner: { groupBy: mocks.partnerGroupBy, findUnique: vi.fn() },programPartnerTag: { groupBy: mocks.partnerTagGroupBy },programApplicationEvent: { groupBy: mocks.applicationEventGroupBy },+ link: { findUnique: mocks.linkFindUnique },},}));@@ -73,6 +77,29 @@ describe("getPartnersCount search", () => {expect(mocks.count).not.toHaveBeenCalled();});+ it("counts an exact short-link result from its hits, not the aggregation", async () => {+ // The aggregation would count every document sharing a token with the+ // pasted link, which is most of the program.+ mocks.linkFindUnique.mockResolvedValue({+ programId: "prog_test",+ partnerId: "pn_1",+ });+ mocks.enrollmentFindUnique.mockResolvedValue({ id: "pge_9" });+ mocks.count.mockResolvedValue(1);+ const searchProvider = createSearchProvider(12_000);++ const count = await getPartnersCount<number>(+ { programId: "prog_test", search: "go.acme.com/partnername" },+ { searchProvider },+ );++ expect(count).toBe(1);+ expect(searchProvider.countCandidates).not.toHaveBeenCalled();+ expect(mocks.count).toHaveBeenCalledWith({+ where: expect.objectContaining({ id: { in: ["pge_9"] } }),+ });+ });+it.each([["a tenant ID", { tenantId: "tenant_1" }],["explicit partner IDs", { partnerIds: ["pn_1"] }],diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.tsindex 62730b1294..cc208f4dd9 100644--- a/apps/web/tests/partners/partner-search-backfill.test.ts+++ b/apps/web/tests/partners/partner-search-backfill.test.ts@@ -25,6 +25,9 @@ function createSource(id: string): PartnerSearchDocumentSource {partnerId: `pn_${id}`,status: "approved" as const,groupId: null,+ program: {+ url: "https://example.com",+ },partner: {name: "Rafi Hasan",email: "[email protected]",diff --git a/apps/web/tests/partners/partner-search-document.test.ts b/apps/web/tests/partners/partner-search-document.test.tsindex 51b1d53f3b..f0c4002540 100644--- a/apps/web/tests/partners/partner-search-document.test.ts+++ b/apps/web/tests/partners/partner-search-document.test.ts@@ -10,6 +10,9 @@ const source: PartnerSearchDocumentSource = {partnerId: "pn_test",status: "approved" as const,groupId: "grp_test",+ program: {+ url: "https://www.example.com/landing",+ },partner: {name: "Rafi Hasan",email: "[email protected]",@@ -34,16 +37,12 @@ const source: PartnerSearchDocumentSource = {},links: [{- domain: "dub.sh",key: "rafi",- shortLink: "https://dub.sh/rafi",- url: "https://example.com/referrals/rafi",+ url: "https://example.com/referrals/rafi?utm_source=partner&token=abc",},{- domain: "dub.sh",key: "rafi-tools",- shortLink: "https://dub.sh/rafi-tools",- url: "https://rafi.dev/tools",+ url: "https://rafi.dev/tools#pricing",},],};@@ -60,12 +59,12 @@ describe("serializePartnerSearchDocument", () => {description: "Developer tools educator",platformTypes: ["website", "twitter"],platformIdentifiers: ["https://rafi.dev", "@rafi-on-x"],- linkDomains: ["dub.sh"],linkKeys: ["rafi", "rafi-tools"],- shortLinks: ["https://dub.sh/rafi", "https://dub.sh/rafi-tools"],destinationUrls: [- "https://example.com/referrals/rafi",- "https://rafi.dev/tools",+ // The program's own host is dropped, the query string with it. A+ // partner-specific host is kept: it identifies the partner.+ "/referrals/rafi",+ "rafi.dev/tools",],status: "approved",groupId: "grp_test",@@ -74,4 +73,32 @@ describe("serializePartnerSearchDocument", () => {partnerTagIds: ["ptag_a"],});});++ it("drops a destination that is only the program's root", () => {+ const document = serializePartnerSearchDocument({+ ...source,+ links: [{ key: "root", url: "https://example.com/" }],+ });++ expect(document.destinationUrls).toEqual([]);+ });++ it("keeps the whole host when the program has no URL", () => {+ const document = serializePartnerSearchDocument({+ ...source,+ program: { url: null },+ links: [{ key: "rafi", url: "https://example.com/referrals/rafi" }],+ });++ expect(document.destinationUrls).toEqual(["example.com/referrals/rafi"]);+ });++ it("keeps an unparseable destination as is", () => {+ const document = serializePartnerSearchDocument({+ ...source,+ links: [{ key: "raw", url: "not a url" }],+ });++ expect(document.destinationUrls).toEqual(["not a url"]);+ });});diff --git a/apps/web/tests/partners/partner-search-find-candidates.test.ts b/apps/web/tests/partners/partner-search-find-candidates.test.tsindex e06bc59b64..0a24c236af 100644--- a/apps/web/tests/partners/partner-search-find-candidates.test.ts+++ b/apps/web/tests/partners/partner-search-find-candidates.test.ts@@ -2,10 +2,18 @@ import type { PartnerSearchProvider } from "@/lib/api/partners/search";import { findPartnerSearchCandidates } from "@/lib/api/partners/search";import { beforeEach, describe, expect, it, vi } from "vitest";-const mocks = vi.hoisted(() => ({ findUnique: vi.fn() }));+const mocks = vi.hoisted(() => ({+ findUnique: vi.fn(),+ linkFindUnique: vi.fn(),+ enrollmentFindUnique: vi.fn(),+}));vi.mock("@/lib/prisma", () => ({- prisma: { partner: { findUnique: mocks.findUnique } },+ prisma: {+ partner: { findUnique: mocks.findUnique },+ link: { findUnique: mocks.linkFindUnique },+ programEnrollment: { findUnique: mocks.enrollmentFindUnique },+ },}));function createProvider(@@ -30,9 +38,75 @@ const query = (search: string) => ({describe("findPartnerSearchCandidates", () => {beforeEach(() => {mocks.findUnique.mockReset();+ mocks.linkFindUnique.mockReset();+ mocks.enrollmentFindUnique.mockReset();vi.spyOn(console, "error").mockImplementation(() => {});});+ describe("pasted short links", () => {+ it.each([+ ["with protocol", "https://go.acme.com/partnername"],+ ["without protocol", "go.acme.com/partnername"],+ ])("resolves %s to exactly that enrollment", async (_label, search) => {+ mocks.linkFindUnique.mockResolvedValue({+ programId: "prog_1",+ partnerId: "pn_1",+ });+ mocks.enrollmentFindUnique.mockResolvedValue({ id: "pge_9" });+ const provider = createProvider();++ await expect(+ findPartnerSearchCandidates(provider, query(search)),+ ).resolves.toEqual({ hits: [{ id: "pge_9" }], exact: true });++ expect(mocks.linkFindUnique).toHaveBeenCalledWith({+ where: { shortLink: "https://go.acme.com/partnername" },+ select: { programId: true, partnerId: true },+ });+ expect(provider.searchCandidates).not.toHaveBeenCalled();+ });++ it("returns no hits for a link from another program", async () => {+ // Its domain tokens would only flood the ranked results.+ mocks.linkFindUnique.mockResolvedValue({+ programId: "prog_other",+ partnerId: "pn_1",+ });+ const provider = createProvider();++ await expect(+ findPartnerSearchCandidates(provider, query("go.acme.com/theirs")),+ ).resolves.toEqual({ hits: [], exact: true });++ expect(provider.searchCandidates).not.toHaveBeenCalled();+ });++ it("falls through to the ranked search on a miss", async () => {+ // A partial paste still matches the link key by prefix.+ mocks.linkFindUnique.mockResolvedValue(null);+ const provider = createProvider();++ await expect(+ findPartnerSearchCandidates(provider, query("go.acme.com/partnern")),+ ).resolves.toEqual({ hits: [{ id: "pge_1" }] });++ expect(provider.searchCandidates).toHaveBeenCalledOnce();+ });++ it.each([+ ["a bare domain", "go.acme.com"],+ ["a name", "drew moore"],+ ["a key alone", "partnername"],+ ])("does not treat %s as a short link", async (_label, search) => {+ const provider = createProvider();++ await findPartnerSearchCandidates(provider, query(search));++ expect(mocks.linkFindUnique).not.toHaveBeenCalled();+ expect(provider.searchCandidates).toHaveBeenCalledOnce();+ });+ });+it("answers a known email from the database, without calling the provider", async () => {mocks.findUnique.mockResolvedValue({ id: "pn_1" });const provider = createProvider();diff --git a/apps/web/tests/partners/partner-search-sync.test.ts b/apps/web/tests/partners/partner-search-sync.test.tsindex 6973a2e49f..195418fd07 100644--- a/apps/web/tests/partners/partner-search-sync.test.ts+++ b/apps/web/tests/partners/partner-search-sync.test.ts@@ -28,6 +28,9 @@ function createSource(partnerId: `pn_${id}`,status: "approved" as const,groupId: null,+ program: {+ url: "https://example.com",+ },partner: {name: "Rafi Hasan",email: "[email protected]",diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.tsindex 7f08336bfa..1fe6cb043c 100644--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts@@ -20,10 +20,8 @@ const document: PartnerSearchDocument = {description: "Affiliate marketer",platformTypes: ["youtube"],platformIdentifiers: ["@rafi"],- linkDomains: ["dub.sh"],- linkKeys: ["rafi"],- shortLinks: ["https://dub.sh/rafi"],- destinationUrls: ["https://example.com/referrals/rafi"],+ linkKeys: ["rafi-link"],+ destinationUrls: ["/referrals/rafi"],status: "approved",groupId: "grp_test",country: "US",@@ -97,7 +95,7 @@ describe("Turbopuffer partner search provider", () => {);// Description, platforms, and links stay out, so a name cannot be// out-weighted by how much else a partner has.- for (const value of ["affiliate marketer", "youtube", "@rafi", "dub.sh"]) {+ for (const value of ["affiliate marketer", "youtube", "@rafi", "rafi-link"]) {expect(row.identityText).not.toContain(value);}// ContainsAllTokens reads the BM25 index, so no text attribute needs a@@ -123,8 +121,8 @@ describe("Turbopuffer partner search provider", () => {"affiliate marketer","youtube","@rafi",- "https://dub.sh/rafi",- "https://example.com/referrals/rafi",+ "rafi-link",+ "/referrals/rafi",]) {expect(row.searchText).toContain(value);}