Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Improve search performance with two phase DB hydration

Rafi37992453fe81
diff --git a/apps/web/lib/api/partners/get-partners.ts b/apps/web/lib/api/partners/get-partners.ts
index 43117e3f60..a71c324892 100644
--- a/apps/web/lib/api/partners/get-partners.ts
+++ b/apps/web/lib/api/partners/get-partners.ts
@@ -1,5 +1,6 @@
import { prisma } from "@/lib/prisma";
import { toCentsNumber } from "@dub/utils";
+import { Prisma } from "@prisma/client";
import { buildProgramEnrollmentWhereForList } from "./program-enrollment-query";
import {
buildPartnerSearchCandidateQuery,
@@ -55,55 +56,80 @@ export async function getPartners(
...(candidateResult ? { search: undefined } : {}),
});
- const programEnrollments = await prisma.programEnrollment.findMany({
- where: {
- ...enrollmentWhere,
- ...(candidateResult
- ? { id: { in: candidateResult.hits.map(({ id }) => id) } }
- : {}),
- },
- include: {
- partner: {
- include: {
- programPartnerTags: {
- where: {
- programId,
- },
- include: {
- partnerTag: true,
- },
+ // The candidate IDs already carry the provider's ranking, but the database
+ // still has to apply every other filter, so they narrow the query rather than
+ // define the result. See PARTNER_SEARCH_CANDIDATE_LIMIT for what that costs.
+ const candidateWhere: Prisma.ProgramEnrollmentWhereInput = {
+ ...enrollmentWhere,
+ ...(candidateResult
+ ? { id: { in: candidateResult.hits.map(({ id }) => id) } }
+ : {}),
+ };
+
+ const include = {
+ partner: {
+ include: {
+ programPartnerTags: {
+ where: {
+ programId,
+ },
+ include: {
+ partnerTag: true,
},
- platforms: true,
},
+ platforms: true,
},
- links: true,
- ...(includeGroup
- ? {
- partnerGroup: {
- select: {
- name: true,
- },
- },
- }
- : {}),
},
- ...(relevanceHits === null
+ links: true,
+ ...(includeGroup
? {
- take: pageSize,
- skip: (page - 1) * pageSize,
- orderBy: {
- [databaseSortBy]: sortOrder,
+ partnerGroup: {
+ select: {
+ name: true,
+ },
},
}
: {}),
- });
+ } satisfies Prisma.ProgramEnrollmentInclude;
+
+ let partners: Prisma.ProgramEnrollmentGetPayload<{
+ include: typeof include;
+ }>[];
+
+ if (relevanceHits) {
+ // Relevance order cannot be expressed in SQL, so the page has to be chosen
+ // in memory. Resolve which candidates survive the filters as bare IDs
+ // first — hydrating all of them to show one page costs ~330ms at the
+ // candidate limit against ~4ms for the page alone.
+ const matches = await prisma.programEnrollment.findMany({
+ where: candidateWhere,
+ select: { id: true },
+ });
- const partners = relevanceHits
- ? orderByPartnerSearchHits(programEnrollments, relevanceHits).slice(
- (page - 1) * pageSize,
- page * pageSize,
- )
- : programEnrollments;
+ const pageIds = orderByPartnerSearchHits(matches, relevanceHits)
+ .slice((page - 1) * pageSize, page * pageSize)
+ .map(({ id }) => id);
+
+ const pageEnrollments =
+ pageIds.length > 0
+ ? await prisma.programEnrollment.findMany({
+ where: { id: { in: pageIds } },
+ include,
+ })
+ : [];
+
+ partners = orderByPartnerSearchHits(pageEnrollments, relevanceHits);
+ } else {
+ partners = await prisma.programEnrollment.findMany({
+ where: candidateWhere,
+ include,
+ take: pageSize,
+ skip: (page - 1) * pageSize,
+ orderBy: {
+ [databaseSortBy]: sortOrder,
+ },
+ });
+ }
return partners.map(
({ partner, links, partnerGroup, ...programEnrollment }) => ({
diff --git a/apps/web/lib/api/partners/search/find-candidates.ts b/apps/web/lib/api/partners/search/find-candidates.ts
index 10ae038865..d2413a7e0a 100644
--- a/apps/web/lib/api/partners/search/find-candidates.ts
+++ b/apps/web/lib/api/partners/search/find-candidates.ts
@@ -24,7 +24,19 @@ export async function findPartnerSearchCandidates(
{ throwOnError = false }: FindPartnerSearchCandidatesOptions = {},
): Promise<PartnerSearchResult | null> {
try {
- return await searchProvider.searchCandidates(query);
+ const result = await searchProvider.searchCandidates(query);
+
+ // A full page means the provider had more to give, so every count built
+ // from these candidates is a floor rather than a total, and any filter the
+ // database applies is choosing from a truncated set. Logged because there
+ // is nowhere in the response to say so without changing its shape.
+ if (result.hits.length >= query.limit) {
+ console.warn(
+ `[Partner Search] Query "${query.query}" filled the ${query.limit} candidate ceiling for program ${query.programId}. Results and counts are a floor, not a total.`,
+ );
+ }
+
+ return result;
} catch (error) {
if (throwOnError) {
throw error;
diff --git a/apps/web/lib/api/partners/search/types.ts b/apps/web/lib/api/partners/search/types.ts
index 077d666725..625c12e657 100644
--- a/apps/web/lib/api/partners/search/types.ts
+++ b/apps/web/lib/api/partners/search/types.ts
@@ -1,6 +1,24 @@
import { PlatformType } from "@prisma/client";
-export const PARTNER_SEARCH_CANDIDATE_LIMIT = 100;
+/**
+ * How many ranked enrollment IDs a provider may return for one query.
+ *
+ * 999 because Upstash Redis Search rejects anything from 1000 up
+ * (`ERR limit should be a positive number less than 1000`).
+ *
+ * KNOWN LIMITATION: filters run after this cut, not before.
+ *
+ * 1. the provider ranks matches and keeps the top 999, ignoring all filters
+ * 2. the database applies status/group/country/tag/metric to those 999
+ *
+ * A query matching fewer than 999 is unaffected. Beyond that, both the rows and
+ * the count are a floor. There is no field on the list or count response to say
+ * so without breaking their shape, so findPartnerSearchCandidates logs it.
+ *
+ * The fix is indexing the filterable fields so the provider filters before it
+ * ranks, which means keeping them in sync on every write — out of scope here.
+ */
+export const PARTNER_SEARCH_CANDIDATE_LIMIT = 999;
export function validatePartnerSearchCandidateLimit(limit: number) {
if (
diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.ts
index c3b634a3d0..4dc5b3b735 100644
--- a/apps/web/scripts/dev/benchmark-partner-search.ts
+++ b/apps/web/scripts/dev/benchmark-partner-search.ts
@@ -247,23 +247,35 @@ async function loadSearchCasePools(
partnerCount: number,
): Promise<SearchCasePool[]> {
const stride = Math.max(1, Math.floor(partnerCount / sampleSize));
- const enrollments = await prisma.programEnrollment.findMany({
- where: {
- programId,
- partner: {
- email: { not: null },
- companyName: { not: null },
- description: { not: null },
- platforms: { some: {} },
- },
- links: { some: {} },
+ const where = {
+ programId,
+ partner: {
+ email: { not: null },
+ companyName: { not: null },
+ description: { not: null },
+ platforms: { some: {} },
},
- select: partnerSearchDocumentSelect,
+ links: { some: {} },
+ };
+
+ // Striding across the program means scanning it, so resolve bare IDs first.
+ // Hydrating every row to keep one in `stride` exceeds MySQL's placeholder
+ // limit once a program reaches 100K partners.
+ const candidateIds = await prisma.programEnrollment.findMany({
+ where,
+ select: { id: true },
orderBy: { id: "asc" },
take: sampleSize * stride,
});
- const sampled = enrollments.filter((_, index) => index % stride === 0);
+ const sampledIds = candidateIds
+ .filter((_, index) => index % stride === 0)
+ .map(({ id }) => id);
+
+ const sampled = await prisma.programEnrollment.findMany({
+ where: { id: { in: sampledIds } },
+ select: partnerSearchDocumentSelect,
+ });
if (sampled.length === 0) {
throw new Error(
diff --git a/apps/web/tests/partners/get-partners-count-search.test.ts b/apps/web/tests/partners/get-partners-count-search.test.ts
index 6de9b926ac..b5be43b324 100644
--- a/apps/web/tests/partners/get-partners-count-search.test.ts
+++ b/apps/web/tests/partners/get-partners-count-search.test.ts
@@ -62,7 +62,7 @@ describe("getPartnersCount search", () => {
expect(searchProvider.searchCandidates).toHaveBeenCalledWith({
programId: "prog_test",
query: "examp",
- limit: 100,
+ limit: 999,
});
expect(mocks.count).toHaveBeenCalledWith({
where: expect.objectContaining({
diff --git a/apps/web/tests/partners/get-partners-search.test.ts b/apps/web/tests/partners/get-partners-search.test.ts
index d1d87f0822..86a50de135 100644
--- a/apps/web/tests/partners/get-partners-search.test.ts
+++ b/apps/web/tests/partners/get-partners-search.test.ts
@@ -105,7 +105,7 @@ describe("getPartners search", () => {
expect(searchProvider.searchCandidates).toHaveBeenCalledWith({
programId: "prog_test",
query: "examp",
- limit: 100,
+ limit: 999,
});
expect(mocks.findMany).toHaveBeenCalledWith(
expect.objectContaining({
@@ -122,11 +122,13 @@ describe("getPartners search", () => {
expect(partners.map(({ id }) => id)).toEqual(["pn_1", "pn_2"]);
});
- it("restores relevance order before slicing a page", async () => {
- mocks.findMany.mockResolvedValue([
- enrollment("pge_3", "pn_3"),
- enrollment("pge_1", "pn_1"),
- ]);
+ it("hydrates only the requested page, in relevance order", async () => {
+ // Relevance order cannot be expressed in SQL, so the page is chosen in
+ // memory. The first query resolves surviving candidates as bare IDs; only
+ // the page itself is hydrated.
+ mocks.findMany
+ .mockResolvedValueOnce([{ id: "pge_3" }, { id: "pge_1" }])
+ .mockResolvedValueOnce([enrollment("pge_3", "pn_3")]);
const searchProvider = createSearchProvider([
{ id: "pge_1" },
{ id: "pge_2" },
@@ -145,16 +147,39 @@ describe("getPartners search", () => {
{ searchProvider },
);
- expect(mocks.findMany).toHaveBeenCalledWith(
- expect.not.objectContaining({
- take: expect.anything(),
- skip: expect.anything(),
- orderBy: expect.anything(),
- }),
- );
+ const [idQuery] = mocks.findMany.mock.calls[0];
+ expect(idQuery.select).toEqual({ id: true });
+ expect(idQuery.include).toBeUndefined();
+
+ // pge_1 ranks first and lands on page 1, so page 2 asks only for pge_3.
+ const [pageQuery] = mocks.findMany.mock.calls[1];
+ expect(pageQuery.where).toEqual({ id: { in: ["pge_3"] } });
+ expect(pageQuery.include).toBeDefined();
+ expect(pageQuery.take).toBeUndefined();
+
expect(partners.map(({ id }) => id)).toEqual(["pn_3"]);
});
+ it("skips the hydration query when the page is empty", async () => {
+ mocks.findMany.mockResolvedValueOnce([{ id: "pge_1" }]);
+ const searchProvider = createSearchProvider([{ id: "pge_1" }]);
+
+ const partners = await getPartners(
+ {
+ programId: "prog_test",
+ search: "examp",
+ page: 5,
+ pageSize: 25,
+ sortBy: "relevance",
+ sortOrder: "desc",
+ },
+ { searchProvider },
+ );
+
+ expect(partners).toEqual([]);
+ expect(mocks.findMany).toHaveBeenCalledTimes(1);
+ });
+
it("falls back to the database search path when the provider fails", async () => {
const searchProvider = createSearchProvider();
vi.mocked(searchProvider.searchCandidates).mockRejectedValue(
diff --git a/apps/web/tests/partners/partner-search-query.test.ts b/apps/web/tests/partners/partner-search-query.test.ts
index c295ed7c79..6516ca8522 100644
--- a/apps/web/tests/partners/partner-search-query.test.ts
+++ b/apps/web/tests/partners/partner-search-query.test.ts
@@ -15,7 +15,7 @@ describe("buildPartnerSearchCandidateQuery", () => {
expect(buildPartnerSearchCandidateQuery(defaultInput)).toEqual({
programId: "prog_test",
query: "examp",
- limit: 100,
+ limit: 999,
});
});