The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Improve search performance with two phase DB hydration
Rafi37992453fe81
diff --git a/apps/web/lib/api/partners/get-partners.ts b/apps/web/lib/api/partners/get-partners.tsindex 43117e3f60..a71c324892 100644--- a/apps/web/lib/api/partners/get-partners.ts+++ b/apps/web/lib/api/partners/get-partners.ts@@ -1,5 +1,6 @@import { prisma } from "@/lib/prisma";import { toCentsNumber } from "@dub/utils";+import { Prisma } from "@prisma/client";import { buildProgramEnrollmentWhereForList } from "./program-enrollment-query";import {buildPartnerSearchCandidateQuery,@@ -55,55 +56,80 @@ export async function getPartners(...(candidateResult ? { search: undefined } : {}),});- const programEnrollments = await prisma.programEnrollment.findMany({- where: {- ...enrollmentWhere,- ...(candidateResult- ? { id: { in: candidateResult.hits.map(({ id }) => id) } }- : {}),- },- include: {- partner: {- include: {- programPartnerTags: {- where: {- programId,- },- include: {- partnerTag: true,- },+ // The candidate IDs already carry the provider's ranking, but the database+ // still has to apply every other filter, so they narrow the query rather than+ // define the result. See PARTNER_SEARCH_CANDIDATE_LIMIT for what that costs.+ const candidateWhere: Prisma.ProgramEnrollmentWhereInput = {+ ...enrollmentWhere,+ ...(candidateResult+ ? { id: { in: candidateResult.hits.map(({ id }) => id) } }+ : {}),+ };++ const include = {+ partner: {+ include: {+ programPartnerTags: {+ where: {+ programId,+ },+ include: {+ partnerTag: true,},- platforms: true,},+ platforms: true,},- links: true,- ...(includeGroup- ? {- partnerGroup: {- select: {- name: true,- },- },- }- : {}),},- ...(relevanceHits === null+ links: true,+ ...(includeGroup? {- take: pageSize,- skip: (page - 1) * pageSize,- orderBy: {- [databaseSortBy]: sortOrder,+ partnerGroup: {+ select: {+ name: true,+ },},}: {}),- });+ } satisfies Prisma.ProgramEnrollmentInclude;++ let partners: Prisma.ProgramEnrollmentGetPayload<{+ include: typeof include;+ }>[];++ if (relevanceHits) {+ // Relevance order cannot be expressed in SQL, so the page has to be chosen+ // in memory. Resolve which candidates survive the filters as bare IDs+ // first — hydrating all of them to show one page costs ~330ms at the+ // candidate limit against ~4ms for the page alone.+ const matches = await prisma.programEnrollment.findMany({+ where: candidateWhere,+ select: { id: true },+ });- const partners = relevanceHits- ? orderByPartnerSearchHits(programEnrollments, relevanceHits).slice(- (page - 1) * pageSize,- page * pageSize,- )- : programEnrollments;+ const pageIds = orderByPartnerSearchHits(matches, relevanceHits)+ .slice((page - 1) * pageSize, page * pageSize)+ .map(({ id }) => id);++ const pageEnrollments =+ pageIds.length > 0+ ? await prisma.programEnrollment.findMany({+ where: { id: { in: pageIds } },+ include,+ })+ : [];++ partners = orderByPartnerSearchHits(pageEnrollments, relevanceHits);+ } else {+ partners = await prisma.programEnrollment.findMany({+ where: candidateWhere,+ include,+ take: pageSize,+ skip: (page - 1) * pageSize,+ orderBy: {+ [databaseSortBy]: sortOrder,+ },+ });+ }return partners.map(({ partner, links, partnerGroup, ...programEnrollment }) => ({diff --git a/apps/web/lib/api/partners/search/find-candidates.ts b/apps/web/lib/api/partners/search/find-candidates.tsindex 10ae038865..d2413a7e0a 100644--- a/apps/web/lib/api/partners/search/find-candidates.ts+++ b/apps/web/lib/api/partners/search/find-candidates.ts@@ -24,7 +24,19 @@ export async function findPartnerSearchCandidates({ throwOnError = false }: FindPartnerSearchCandidatesOptions = {},): Promise<PartnerSearchResult | null> {try {- return await searchProvider.searchCandidates(query);+ const result = await searchProvider.searchCandidates(query);++ // A full page means the provider had more to give, so every count built+ // from these candidates is a floor rather than a total, and any filter the+ // database applies is choosing from a truncated set. Logged because there+ // is nowhere in the response to say so without changing its shape.+ if (result.hits.length >= query.limit) {+ console.warn(+ `[Partner Search] Query "${query.query}" filled the ${query.limit} candidate ceiling for program ${query.programId}. Results and counts are a floor, not a total.`,+ );+ }++ return result;} catch (error) {if (throwOnError) {throw error;diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.tsindex 077d666725..625c12e657 100644--- a/apps/web/lib/api/partners/search/types.ts+++ b/apps/web/lib/api/partners/search/types.ts@@ -1,6 +1,24 @@import { PlatformType } from "@prisma/client";-export const PARTNER_SEARCH_CANDIDATE_LIMIT = 100;+/**+ * How many ranked enrollment IDs a provider may return for one query.+ *+ * 999 because Upstash Redis Search rejects anything from 1000 up+ * (`ERR limit should be a positive number less than 1000`).+ *+ * KNOWN LIMITATION: filters run after this cut, not before.+ *+ * 1. the provider ranks matches and keeps the top 999, ignoring all filters+ * 2. the database applies status/group/country/tag/metric to those 999+ *+ * A query matching fewer than 999 is unaffected. Beyond that, both the rows and+ * the count are a floor. There is no field on the list or count response to say+ * so without breaking their shape, so findPartnerSearchCandidates logs it.+ *+ * The fix is indexing the filterable fields so the provider filters before it+ * ranks, which means keeping them in sync on every write — out of scope here.+ */+export const PARTNER_SEARCH_CANDIDATE_LIMIT = 999;export function validatePartnerSearchCandidateLimit(limit: number) {if (diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.tsindex c3b634a3d0..4dc5b3b735 100644--- a/apps/web/scripts/dev/benchmark-partner-search.ts+++ b/apps/web/scripts/dev/benchmark-partner-search.ts@@ -247,23 +247,35 @@ async function loadSearchCasePools(partnerCount: number,): Promise<SearchCasePool[]> {const stride = Math.max(1, Math.floor(partnerCount / sampleSize));- const enrollments = await prisma.programEnrollment.findMany({- where: {- programId,- partner: {- email: { not: null },- companyName: { not: null },- description: { not: null },- platforms: { some: {} },- },- links: { some: {} },+ const where = {+ programId,+ partner: {+ email: { not: null },+ companyName: { not: null },+ description: { not: null },+ platforms: { some: {} },},- select: partnerSearchDocumentSelect,+ links: { some: {} },+ };++ // Striding across the program means scanning it, so resolve bare IDs first.+ // Hydrating every row to keep one in `stride` exceeds MySQL's placeholder+ // limit once a program reaches 100K partners.+ const candidateIds = await prisma.programEnrollment.findMany({+ where,+ select: { id: true },orderBy: { id: "asc" },take: sampleSize * stride,});- const sampled = enrollments.filter((_, index) => index % stride === 0);+ const sampledIds = candidateIds+ .filter((_, index) => index % stride === 0)+ .map(({ id }) => id);++ const sampled = await prisma.programEnrollment.findMany({+ where: { id: { in: sampledIds } },+ select: partnerSearchDocumentSelect,+ });if (sampled.length === 0) {throw new Error(diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.tsindex 6de9b926ac..b5be43b324 100644--- a/apps/web/tests/partners/get-partners-count-search.test.ts+++ b/apps/web/tests/partners/get-partners-count-search.test.ts@@ -62,7 +62,7 @@ describe("getPartnersCount search", () => {expect(searchProvider.searchCandidates).toHaveBeenCalledWith({programId: "prog_test",query: "examp",- limit: 100,+ limit: 999,});expect(mocks.count).toHaveBeenCalledWith({where: expect.objectContaining({diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.tsindex d1d87f0822..86a50de135 100644--- a/apps/web/tests/partners/get-partners-search.test.ts+++ b/apps/web/tests/partners/get-partners-search.test.ts@@ -105,7 +105,7 @@ describe("getPartners search", () => {expect(searchProvider.searchCandidates).toHaveBeenCalledWith({programId: "prog_test",query: "examp",- limit: 100,+ limit: 999,});expect(mocks.findMany).toHaveBeenCalledWith(expect.objectContaining({@@ -122,11 +122,13 @@ describe("getPartners search", () => {expect(partners.map(({ id }) => id)).toEqual(["pn_1", "pn_2"]);});- it("restores relevance order before slicing a page", async () => {- mocks.findMany.mockResolvedValue([- enrollment("pge_3", "pn_3"),- enrollment("pge_1", "pn_1"),- ]);+ it("hydrates only the requested page, in relevance order", async () => {+ // Relevance order cannot be expressed in SQL, so the page is chosen in+ // memory. The first query resolves surviving candidates as bare IDs; only+ // the page itself is hydrated.+ mocks.findMany+ .mockResolvedValueOnce([{ id: "pge_3" }, { id: "pge_1" }])+ .mockResolvedValueOnce([enrollment("pge_3", "pn_3")]);const searchProvider = createSearchProvider([{ id: "pge_1" },{ id: "pge_2" },@@ -145,16 +147,39 @@ describe("getPartners search", () => {{ searchProvider },);- expect(mocks.findMany).toHaveBeenCalledWith(- expect.not.objectContaining({- take: expect.anything(),- skip: expect.anything(),- orderBy: expect.anything(),- }),- );+ const [idQuery] = mocks.findMany.mock.calls[0];+ expect(idQuery.select).toEqual({ id: true });+ expect(idQuery.include).toBeUndefined();++ // pge_1 ranks first and lands on page 1, so page 2 asks only for pge_3.+ const [pageQuery] = mocks.findMany.mock.calls[1];+ expect(pageQuery.where).toEqual({ id: { in: ["pge_3"] } });+ expect(pageQuery.include).toBeDefined();+ expect(pageQuery.take).toBeUndefined();+expect(partners.map(({ id }) => id)).toEqual(["pn_3"]);});+ it("skips the hydration query when the page is empty", async () => {+ mocks.findMany.mockResolvedValueOnce([{ id: "pge_1" }]);+ const searchProvider = createSearchProvider([{ id: "pge_1" }]);++ const partners = await getPartners(+ {+ programId: "prog_test",+ search: "examp",+ page: 5,+ pageSize: 25,+ sortBy: "relevance",+ sortOrder: "desc",+ },+ { searchProvider },+ );++ expect(partners).toEqual([]);+ expect(mocks.findMany).toHaveBeenCalledTimes(1);+ });+it("falls back to the database search path when the provider fails", async () => {const searchProvider = createSearchProvider();vi.mocked(searchProvider.searchCandidates).mockRejectedValue(diff --git a/apps/web/tests/partners/partner-search-query.test.ts b/apps/web/tests/partners/partner-search-query.test.tsindex c295ed7c79..6516ca8522 100644--- a/apps/web/tests/partners/partner-search-query.test.ts+++ b/apps/web/tests/partners/partner-search-query.test.ts@@ -15,7 +15,7 @@ describe("buildPartnerSearchCandidateQuery", () => {expect(buildPartnerSearchCandidateQuery(defaultInput)).toEqual({programId: "prog_test",query: "examp",- limit: 100,+ limit: 999,});});