Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Make pasted short links exact and stop indexing shared domains

Rafi5b1e41b5e2a3
diff --git a/.pnpm-store/v11/index.db b/.pnpm-store/v11/index.db
new file mode 100644
index 0000000000..4d0f4d75c0
Binary files /dev/null and b/.pnpm-store/v11/index.db differ
diff --git a/.pnpm-store/v11/index.db-shm b/.pnpm-store/v11/index.db-shm
new file mode 100644
index 0000000000..adf81a5561
Binary files /dev/null and b/.pnpm-store/v11/index.db-shm differ
diff --git a/.pnpm-store/v11/index.db-wal b/.pnpm-store/v11/index.db-wal
new file mode 100644
index 0000000000..60bb626bba
Binary files /dev/null and b/.pnpm-store/v11/index.db-wal differ
diff --git a/apps/web/lib/api/partners/get-partners-count.ts b/apps/web/lib/api/partners/get-partners-count.ts
index 171073316c..49ff55fd71 100644
--- a/apps/web/lib/api/partners/get-partners-count.ts
+++ b/apps/web/lib/api/partners/get-partners-count.ts
@@ -275,6 +275,9 @@ export async function getPartnersCount<T>(
searchProvider &&
candidateQuery &&
candidateResult &&
+ // An exact lookup already names its hits; the text aggregation would count
+ // every document sharing a token with the pasted link instead.
+ !candidateResult.exact &&
!databaseOnlyFilters
) {
try {
diff --git a/apps/web/lib/api/partners/search/find-candidates.ts b/apps/web/lib/api/partners/search/find-candidates.ts
index 0671a7619b..2e94c39f51 100644
--- a/apps/web/lib/api/partners/search/find-candidates.ts
+++ b/apps/web/lib/api/partners/search/find-candidates.ts
@@ -6,6 +6,58 @@ import type {
PartnerSearchResult,
} from "./types";
+/**
+ * A pasted short link: a host with a path, with or without the protocol, e.g.
+ * `go.acme.com/partnername`. A bare domain is not one, because typing a domain
+ * is a text search, while a host plus path only ever comes from pasting a link.
+ */
+const SHORT_LINK_QUERY_PATTERN =
+ /^(?:https?:\/\/)?([a-z0-9-]+(?:\.[a-z0-9-]+)+)(\/\S+)$/i;
+
+/**
+ * Resolves a pasted short link to its enrollment through `Link.shortLink`,
+ * which is unique and indexed, so the paste matches exactly the partner who
+ * owns that link. The domain tokens of a short link are shared by every
+ * partner in the program, so the ranked path cannot make a paste precise.
+ *
+ * Misses fall through to the ranked search: a partial paste still matches the
+ * key by prefix. A link from another program returns no hits rather than
+ * falling through, since its tokens would only flood the ranked results.
+ */
+async function findExactShortLinkResult({
+ programId,
+ query,
+}: PartnerSearchCandidateQuery): Promise<PartnerSearchResult | null> {
+ const match = query.trim().match(SHORT_LINK_QUERY_PATTERN);
+
+ if (!match) {
+ return null;
+ }
+
+ const [, host, path] = match;
+ const link = await prisma.link.findUnique({
+ where: { shortLink: `https://${host}${path}` },
+ select: { programId: true, partnerId: true },
+ });
+
+ if (!link?.partnerId || !link.programId) {
+ return null;
+ }
+
+ if (link.programId !== programId) {
+ return { hits: [], exact: true };
+ }
+
+ const enrollment = await prisma.programEnrollment.findUnique({
+ where: {
+ partnerId_programId: { partnerId: link.partnerId, programId },
+ },
+ select: { id: true },
+ });
+
+ return { hits: enrollment ? [{ id: enrollment.id }] : [], exact: true };
+}
+
interface FindPartnerSearchCandidatesOptions {
/**
* Surface provider failures instead of degrading. The benchmark needs this so
@@ -42,6 +94,12 @@ export async function findPartnerSearchCandidates(
}
}
+ const exactShortLinkResult = await findExactShortLinkResult(query);
+
+ if (exactShortLinkResult) {
+ return exactShortLinkResult;
+ }
+
try {
return await searchProvider.searchCandidates(query);
} catch (error) {
diff --git a/apps/web/lib/api/partners/search/providers/turbopuffer.ts b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
index 34a20d4d53..7c94aa01c0 100644
--- a/apps/web/lib/api/partners/search/providers/turbopuffer.ts
+++ b/apps/web/lib/api/partners/search/providers/turbopuffer.ts
@@ -31,7 +31,7 @@ import { getEmailNgrams, getQueryNgrams } from "./shared";
* existing namespace. A new version is backfilled alongside the old one and
* swapped in, rather than rebuilt in place.
*/
-export const PARTNER_SEARCH_NAMESPACE = "partner-search-v4";
+export const PARTNER_SEARCH_NAMESPACE = "partner-search-v5";
const WRITE_BATCH_SIZE = 500;
/**
diff --git a/apps/web/lib/api/partners/search/searchable-values.ts b/apps/web/lib/api/partners/search/searchable-values.ts
index 6908e72d14..131dd9ef73 100644
--- a/apps/web/lib/api/partners/search/searchable-values.ts
+++ b/apps/web/lib/api/partners/search/searchable-values.ts
@@ -29,7 +29,7 @@ export function getPartnerSearchableValues(
document.description,
...document.platformTypes,
...document.platformIdentifiers,
- ...document.shortLinks,
+ ...document.linkKeys,
...document.destinationUrls,
].filter((value): value is string => Boolean(value));
}
diff --git a/apps/web/lib/api/partners/search/serialize-document.ts b/apps/web/lib/api/partners/search/serialize-document.ts
index 57e8913a58..8846fb76bd 100644
--- a/apps/web/lib/api/partners/search/serialize-document.ts
+++ b/apps/web/lib/api/partners/search/serialize-document.ts
@@ -7,6 +7,13 @@ export const partnerSearchDocumentSelect = {
partnerId: true,
status: true,
groupId: true,
+ // The program's destination host, so the serializer can drop it from
+ // destination URLs below.
+ program: {
+ select: {
+ url: true,
+ },
+ },
partner: {
select: {
name: true,
@@ -32,9 +39,7 @@ export const partnerSearchDocumentSelect = {
},
links: {
select: {
- domain: true,
key: true,
- shortLink: true,
url: true,
},
},
@@ -48,10 +53,46 @@ function unique<T>(values: T[]): T[] {
return Array.from(new Set(values));
}
+function getHost(url: string | null): string | null {
+ if (!url) {
+ return null;
+ }
+
+ try {
+ return new URL(url).hostname.replace(/^www\./, "");
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * The distinctive part of a destination URL. Query strings and fragments are
+ * stripped so tracking parameters never reach the index, and the host is
+ * dropped when it is the program's own: measured in production, ~99% of
+ * destination hosts in a program are its one domain, so indexing it floods
+ * every search containing a domain token. A host that differs (a partner's own
+ * site) identifies the partner and is kept.
+ */
+function sanitizeDestinationUrl(
+ url: string,
+ programHost: string | null,
+): string | null {
+ try {
+ const parsed = new URL(url);
+ const host = parsed.hostname.replace(/^www\./, "");
+ const path = parsed.pathname === "/" ? "" : parsed.pathname;
+
+ return `${host === programHost ? "" : host}${path}` || null;
+ } catch {
+ return url;
+ }
+}
+
export function serializePartnerSearchDocument(
enrollment: PartnerSearchDocumentSource,
): PartnerSearchDocument {
- const { partner, links } = enrollment;
+ const { partner, links, program } = enrollment;
+ const programHost = getHost(program.url);
return {
id: enrollment.id,
@@ -65,10 +106,13 @@ export function serializePartnerSearchDocument(
platformIdentifiers: unique(
partner.platforms.map(({ identifier }) => identifier),
),
- linkDomains: unique(links.map(({ domain }) => domain)),
linkKeys: unique(links.map(({ key }) => key)),
- shortLinks: unique(links.map(({ shortLink }) => shortLink)),
- destinationUrls: unique(links.map(({ url }) => url)),
+ destinationUrls: unique(
+ links.flatMap(({ url }) => {
+ const sanitized = sanitizeDestinationUrl(url, programHost);
+ return sanitized ? [sanitized] : [];
+ }),
+ ),
status: enrollment.status,
groupId: enrollment.groupId,
country: partner.country,
diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.ts
index 30ee17d2d4..c7fc56fe3a 100644
--- a/apps/web/lib/api/partners/search/types.ts
+++ b/apps/web/lib/api/partners/search/types.ts
@@ -53,10 +53,11 @@ export interface PartnerSearchDocument {
platformTypes: PlatformType[];
platformIdentifiers: string[];
- // Searchable link fields
- linkDomains: string[];
+ // Searchable link fields. Short-link domains are absent: a program's
+ // partners share one, so it never narrows a search. A full short link is an
+ // exact database lookup instead. Destination URLs are stored sanitized (see
+ // serialize-document).
linkKeys: string[];
- shortLinks: string[];
destinationUrls: string[];
// Filterable fields. Metrics are deliberately absent: they move on every click
@@ -74,6 +75,12 @@ export interface PartnerSearchHit {
export interface PartnerSearchResult {
hits: PartnerSearchHit[];
+ /**
+ * The hits are an exact lookup rather than ranked matches, so the count must
+ * come from them and never from the text aggregation, which would count
+ * every document sharing a token with the pasted value.
+ */
+ exact?: boolean;
}
/**
diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.ts
index 3af27f49f7..642896594b 100644
--- a/apps/web/scripts/dev/benchmark-partner-search.ts
+++ b/apps/web/scripts/dev/benchmark-partner-search.ts
@@ -203,10 +203,6 @@ function longestSearchToken(value: string): string {
return token.slice(0, 12);
}
-function stripProtocol(url: string): string {
- return url.replace(/^https?:\/\//, "");
-}
-
function emailInfix(email: string): string {
const domain = email.split("@")[1];
if (!domain) {
@@ -220,9 +216,7 @@ function emailInfix(email: string): string {
function createSearchCases(document: PartnerSearchDocument): SearchCase[] {
const platformType = document.platformTypes[0];
const platformIdentifier = document.platformIdentifiers[0];
- const linkDomain = document.linkDomains[0];
const linkKey = document.linkKeys[0];
- const shortLink = document.shortLinks[0];
const destinationUrl = document.destinationUrls[0];
if (
@@ -231,9 +225,7 @@ function createSearchCases(document: PartnerSearchDocument): SearchCase[] {
!document.description ||
!platformType ||
!platformIdentifier ||
- !linkDomain ||
!linkKey ||
- !shortLink ||
!destinationUrl
) {
throw new Error(
@@ -251,9 +243,7 @@ function createSearchCases(document: PartnerSearchDocument): SearchCase[] {
field: "platform identifier",
query: longestSearchToken(platformIdentifier),
},
- { field: "link domain", query: longestSearchToken(linkDomain) },
{ field: "link key", query: linkKey },
- { field: "short link", query: stripProtocol(shortLink) },
{
field: "destination URL",
query: longestSearchToken(destinationUrl),
diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.ts
index e829a164bd..e22e43dec6 100644
--- a/apps/web/tests/partners/get-partners-count-search.test.ts
+++ b/apps/web/tests/partners/get-partners-count-search.test.ts
@@ -11,6 +11,8 @@ const mocks = vi.hoisted(() => ({
partnerGroupBy: vi.fn(),
partnerTagGroupBy: vi.fn(),
applicationEventGroupBy: vi.fn(),
+ enrollmentFindUnique: vi.fn(),
+ linkFindUnique: vi.fn(),
}));
vi.mock("@/lib/prisma", () => ({
@@ -19,10 +21,12 @@ vi.mock("@/lib/prisma", () => ({
programEnrollment: {
count: mocks.count,
groupBy: mocks.enrollmentGroupBy,
+ findUnique: mocks.enrollmentFindUnique,
},
- partner: { groupBy: mocks.partnerGroupBy },
+ partner: { groupBy: mocks.partnerGroupBy, findUnique: vi.fn() },
programPartnerTag: { groupBy: mocks.partnerTagGroupBy },
programApplicationEvent: { groupBy: mocks.applicationEventGroupBy },
+ link: { findUnique: mocks.linkFindUnique },
},
}));
@@ -73,6 +77,29 @@ describe("getPartnersCount search", () => {
expect(mocks.count).not.toHaveBeenCalled();
});
+ it("counts an exact short-link result from its hits, not the aggregation", async () => {
+ // The aggregation would count every document sharing a token with the
+ // pasted link, which is most of the program.
+ mocks.linkFindUnique.mockResolvedValue({
+ programId: "prog_test",
+ partnerId: "pn_1",
+ });
+ mocks.enrollmentFindUnique.mockResolvedValue({ id: "pge_9" });
+ mocks.count.mockResolvedValue(1);
+ const searchProvider = createSearchProvider(12_000);
+
+ const count = await getPartnersCount<number>(
+ { programId: "prog_test", search: "go.acme.com/partnername" },
+ { searchProvider },
+ );
+
+ expect(count).toBe(1);
+ expect(searchProvider.countCandidates).not.toHaveBeenCalled();
+ expect(mocks.count).toHaveBeenCalledWith({
+ where: expect.objectContaining({ id: { in: ["pge_9"] } }),
+ });
+ });
+
it.each([
["a tenant ID", { tenantId: "tenant_1" }],
["explicit partner IDs", { partnerIds: ["pn_1"] }],
diff --git a/apps/web/tests/partners/partner-search-backfill.test.ts b/apps/web/tests/partners/partner-search-backfill.test.ts
index 62730b1294..cc208f4dd9 100644
--- a/apps/web/tests/partners/partner-search-backfill.test.ts
+++ b/apps/web/tests/partners/partner-search-backfill.test.ts
@@ -25,6 +25,9 @@ function createSource(id: string): PartnerSearchDocumentSource {
partnerId: `pn_${id}`,
status: "approved" as const,
groupId: null,
+ program: {
+ url: "https://example.com",
+ },
partner: {
name: "Rafi Hasan",
diff --git a/apps/web/tests/partners/partner-search-document.test.ts b/apps/web/tests/partners/partner-search-document.test.ts
index 51b1d53f3b..f0c4002540 100644
--- a/apps/web/tests/partners/partner-search-document.test.ts
+++ b/apps/web/tests/partners/partner-search-document.test.ts
@@ -10,6 +10,9 @@ const source: PartnerSearchDocumentSource = {
partnerId: "pn_test",
status: "approved" as const,
groupId: "grp_test",
+ program: {
+ url: "https://www.example.com/landing",
+ },
partner: {
name: "Rafi Hasan",
@@ -34,16 +37,12 @@ const source: PartnerSearchDocumentSource = {
},
links: [
{
- domain: "dub.sh",
key: "rafi",
- shortLink: "https://dub.sh/rafi",
- url: "https://example.com/referrals/rafi",
+ url: "https://example.com/referrals/rafi?utm_source=partner&token=abc",
},
{
- domain: "dub.sh",
key: "rafi-tools",
- shortLink: "https://dub.sh/rafi-tools",
- url: "https://rafi.dev/tools",
+ url: "https://rafi.dev/tools#pricing",
},
],
};
@@ -60,12 +59,12 @@ describe("serializePartnerSearchDocument", () => {
description: "Developer tools educator",
platformTypes: ["website", "twitter"],
platformIdentifiers: ["https://rafi.dev", "@rafi-on-x"],
- linkDomains: ["dub.sh"],
linkKeys: ["rafi", "rafi-tools"],
- shortLinks: ["https://dub.sh/rafi", "https://dub.sh/rafi-tools"],
destinationUrls: [
- "https://example.com/referrals/rafi",
- "https://rafi.dev/tools",
+ // The program's own host is dropped, the query string with it. A
+ // partner-specific host is kept: it identifies the partner.
+ "/referrals/rafi",
+ "rafi.dev/tools",
],
status: "approved",
groupId: "grp_test",
@@ -74,4 +73,32 @@ describe("serializePartnerSearchDocument", () => {
partnerTagIds: ["ptag_a"],
});
});
+
+ it("drops a destination that is only the program's root", () => {
+ const document = serializePartnerSearchDocument({
+ ...source,
+ links: [{ key: "root", url: "https://example.com/" }],
+ });
+
+ expect(document.destinationUrls).toEqual([]);
+ });
+
+ it("keeps the whole host when the program has no URL", () => {
+ const document = serializePartnerSearchDocument({
+ ...source,
+ program: { url: null },
+ links: [{ key: "rafi", url: "https://example.com/referrals/rafi" }],
+ });
+
+ expect(document.destinationUrls).toEqual(["example.com/referrals/rafi"]);
+ });
+
+ it("keeps an unparseable destination as is", () => {
+ const document = serializePartnerSearchDocument({
+ ...source,
+ links: [{ key: "raw", url: "not a url" }],
+ });
+
+ expect(document.destinationUrls).toEqual(["not a url"]);
+ });
});
diff --git a/apps/web/tests/partners/partner-search-find-candidates.test.ts b/apps/web/tests/partners/partner-search-find-candidates.test.ts
index e06bc59b64..0a24c236af 100644
--- a/apps/web/tests/partners/partner-search-find-candidates.test.ts
+++ b/apps/web/tests/partners/partner-search-find-candidates.test.ts
@@ -2,10 +2,18 @@ import type { PartnerSearchProvider } from "@/lib/api/partners/search";
import { findPartnerSearchCandidates } from "@/lib/api/partners/search";
import { beforeEach, describe, expect, it, vi } from "vitest";
-const mocks = vi.hoisted(() => ({ findUnique: vi.fn() }));
+const mocks = vi.hoisted(() => ({
+ findUnique: vi.fn(),
+ linkFindUnique: vi.fn(),
+ enrollmentFindUnique: vi.fn(),
+}));
vi.mock("@/lib/prisma", () => ({
- prisma: { partner: { findUnique: mocks.findUnique } },
+ prisma: {
+ partner: { findUnique: mocks.findUnique },
+ link: { findUnique: mocks.linkFindUnique },
+ programEnrollment: { findUnique: mocks.enrollmentFindUnique },
+ },
}));
function createProvider(
@@ -30,9 +38,75 @@ const query = (search: string) => ({
describe("findPartnerSearchCandidates", () => {
beforeEach(() => {
mocks.findUnique.mockReset();
+ mocks.linkFindUnique.mockReset();
+ mocks.enrollmentFindUnique.mockReset();
vi.spyOn(console, "error").mockImplementation(() => {});
});
+ describe("pasted short links", () => {
+ it.each([
+ ["with protocol", "https://go.acme.com/partnername"],
+ ["without protocol", "go.acme.com/partnername"],
+ ])("resolves %s to exactly that enrollment", async (_label, search) => {
+ mocks.linkFindUnique.mockResolvedValue({
+ programId: "prog_1",
+ partnerId: "pn_1",
+ });
+ mocks.enrollmentFindUnique.mockResolvedValue({ id: "pge_9" });
+ const provider = createProvider();
+
+ await expect(
+ findPartnerSearchCandidates(provider, query(search)),
+ ).resolves.toEqual({ hits: [{ id: "pge_9" }], exact: true });
+
+ expect(mocks.linkFindUnique).toHaveBeenCalledWith({
+ where: { shortLink: "https://go.acme.com/partnername" },
+ select: { programId: true, partnerId: true },
+ });
+ expect(provider.searchCandidates).not.toHaveBeenCalled();
+ });
+
+ it("returns no hits for a link from another program", async () => {
+ // Its domain tokens would only flood the ranked results.
+ mocks.linkFindUnique.mockResolvedValue({
+ programId: "prog_other",
+ partnerId: "pn_1",
+ });
+ const provider = createProvider();
+
+ await expect(
+ findPartnerSearchCandidates(provider, query("go.acme.com/theirs")),
+ ).resolves.toEqual({ hits: [], exact: true });
+
+ expect(provider.searchCandidates).not.toHaveBeenCalled();
+ });
+
+ it("falls through to the ranked search on a miss", async () => {
+ // A partial paste still matches the link key by prefix.
+ mocks.linkFindUnique.mockResolvedValue(null);
+ const provider = createProvider();
+
+ await expect(
+ findPartnerSearchCandidates(provider, query("go.acme.com/partnern")),
+ ).resolves.toEqual({ hits: [{ id: "pge_1" }] });
+
+ expect(provider.searchCandidates).toHaveBeenCalledOnce();
+ });
+
+ it.each([
+ ["a bare domain", "go.acme.com"],
+ ["a name", "drew moore"],
+ ["a key alone", "partnername"],
+ ])("does not treat %s as a short link", async (_label, search) => {
+ const provider = createProvider();
+
+ await findPartnerSearchCandidates(provider, query(search));
+
+ expect(mocks.linkFindUnique).not.toHaveBeenCalled();
+ expect(provider.searchCandidates).toHaveBeenCalledOnce();
+ });
+ });
+
it("answers a known email from the database, without calling the provider", async () => {
mocks.findUnique.mockResolvedValue({ id: "pn_1" });
const provider = createProvider();
diff --git a/apps/web/tests/partners/partner-search-sync.test.ts b/apps/web/tests/partners/partner-search-sync.test.ts
index 6973a2e49f..195418fd07 100644
--- a/apps/web/tests/partners/partner-search-sync.test.ts
+++ b/apps/web/tests/partners/partner-search-sync.test.ts
@@ -28,6 +28,9 @@ function createSource(
partnerId: `pn_${id}`,
status: "approved" as const,
groupId: null,
+ program: {
+ url: "https://example.com",
+ },
partner: {
name: "Rafi Hasan",
diff --git a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
index 7f08336bfa..1fe6cb043c 100644
--- a/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/turbopuffer-partner-search-provider.test.ts
@@ -20,10 +20,8 @@ const document: PartnerSearchDocument = {
description: "Affiliate marketer",
platformTypes: ["youtube"],
platformIdentifiers: ["@rafi"],
- linkDomains: ["dub.sh"],
- linkKeys: ["rafi"],
- shortLinks: ["https://dub.sh/rafi"],
- destinationUrls: ["https://example.com/referrals/rafi"],
+ linkKeys: ["rafi-link"],
+ destinationUrls: ["/referrals/rafi"],
status: "approved",
groupId: "grp_test",
country: "US",
@@ -97,7 +95,7 @@ describe("Turbopuffer partner search provider", () => {
);
// Description, platforms, and links stay out, so a name cannot be
// out-weighted by how much else a partner has.
- for (const value of ["affiliate marketer", "youtube", "@rafi", "dub.sh"]) {
+ for (const value of ["affiliate marketer", "youtube", "@rafi", "rafi-link"]) {
expect(row.identityText).not.toContain(value);
}
// ContainsAllTokens reads the BM25 index, so no text attribute needs a
@@ -123,8 +121,8 @@ describe("Turbopuffer partner search provider", () => {
"affiliate marketer",
"youtube",
"@rafi",
- "https://dub.sh/rafi",
- "https://example.com/referrals/rafi",
+ "rafi-link",
+ "/referrals/rafi",
]) {
expect(row.searchText).toContain(value);
}