Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Improve the benchmark script.

Rafic760876a862c
diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.ts
index d893e8b29e..fc2533aa87 100644
--- a/apps/web/scripts/dev/benchmark-partner-search.ts
+++ b/apps/web/scripts/dev/benchmark-partner-search.ts
@@ -1,3 +1,26 @@
+/**
+ * Measures p99 latency of the relevance-ranked partner list for a program.
+ *
+ * Calls `getPartners` in process, not over HTTP, so the numbers exclude route
+ * handling, auth, and serialization — treat them as a floor for the endpoint.
+ *
+ * To avoid measuring a warm cache it samples `--sampleSize` partners strided
+ * across the program and exhausts a field's queries before repeating any. Fields
+ * whose values barely vary still collapse to a few queries, so each field's
+ * distinct-query count is reported and undersized pools are flagged; a p99 under
+ * that warning is optimistic.
+ *
+ * Exits non-zero when a field produced no successful request, when the error rate
+ * exceeds `--maxErrorRate`, or when a p99 reaches `--thresholdMs`.
+ *
+ * cd apps/web
+ * pnpm run script dev/benchmark-partner-search --programId=prog_123 [--requests=1000]
+ * [--sampleSize=100] [--concurrency=10] [--warmup=50] [--thresholdMs=1000]
+ *
+ * Requires PARTNER_SEARCH_PROVIDER and an index backfilled for the program:
+ * pnpm run script partners/backfill-partner-search --programId=prog_123
+ */
+
import { getPartners } from "@/lib/api/partners/get-partners";
import {
getPartnerSearchProvider,
@@ -8,7 +31,10 @@ import {
type PartnerSearchDocument,
} from "@/lib/api/partners/search";
import { prisma } from "@/lib/prisma";
-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";
+import {
+ parseNonNegativeInteger,
+ parsePositiveInteger,
+} from "@/scripts/utils/parse-cli-number";
import "dotenv-flow/config";
const DEFAULT_REQUESTS = 1_000;
@@ -17,6 +43,9 @@ const DEFAULT_CONCURRENCY = 10;
const DEFAULT_PAGE_SIZE = 25;
const DEFAULT_THRESHOLD_MS = 1_000;
const MINIMUM_REQUESTS = 1_000;
+const DEFAULT_SAMPLE_SIZE = 100;
+const MAX_SAMPLE_SIZE = 1_000;
+const DEFAULT_MAX_ERROR_RATE = 0;
interface BenchmarkArguments {
programId: string;
@@ -25,6 +54,8 @@ interface BenchmarkArguments {
concurrency: number;
pageSize: number;
thresholdMs: number;
+ sampleSize: number;
+ maxErrorRate: number;
}
interface SearchCase {
@@ -32,6 +63,11 @@ interface SearchCase {
query: string;
}
+interface SearchCasePool {
+ field: string;
+ queries: string[];
+}
+
interface BenchmarkResult {
field: string;
query: string;
@@ -39,6 +75,16 @@ interface BenchmarkResult {
error: string | null;
}
+function parsePercentage(value: string, flag: string): number {
+ const parsed = /^\d+(\.\d+)?$/.test(value) ? Number(value) : NaN;
+
+ if (!Number.isFinite(parsed) || parsed > 100) {
+ throw new Error(`${flag} must be between 0 and 100, received: "${value}"`);
+ }
+
+ return parsed;
+}
+
function parseArguments(args: string[]): BenchmarkArguments {
let programId: string | undefined;
let requests = DEFAULT_REQUESTS;
@@ -46,6 +92,8 @@ function parseArguments(args: string[]): BenchmarkArguments {
let concurrency = DEFAULT_CONCURRENCY;
let pageSize = DEFAULT_PAGE_SIZE;
let thresholdMs = DEFAULT_THRESHOLD_MS;
+ let sampleSize = DEFAULT_SAMPLE_SIZE;
+ let maxErrorRate = DEFAULT_MAX_ERROR_RATE;
for (const arg of args) {
if (arg.startsWith("--programId=")) {
@@ -56,10 +104,20 @@ function parseArguments(args: string[]): BenchmarkArguments {
"--requests",
);
} else if (arg.startsWith("--warmup=")) {
- warmupRequests = parsePositiveInteger(
+ warmupRequests = parseNonNegativeInteger(
arg.slice("--warmup=".length),
"--warmup",
);
+ } else if (arg.startsWith("--sampleSize=")) {
+ sampleSize = parsePositiveInteger(
+ arg.slice("--sampleSize=".length),
+ "--sampleSize",
+ );
+ } else if (arg.startsWith("--maxErrorRate=")) {
+ maxErrorRate = parsePercentage(
+ arg.slice("--maxErrorRate=".length),
+ "--maxErrorRate",
+ );
} else if (arg.startsWith("--concurrency=")) {
concurrency = parsePositiveInteger(
arg.slice("--concurrency=".length),
@@ -96,6 +154,9 @@ function parseArguments(args: string[]): BenchmarkArguments {
`--pageSize cannot exceed ${PARTNER_SEARCH_CANDIDATE_LIMIT}.`,
);
}
+ if (sampleSize > MAX_SAMPLE_SIZE) {
+ throw new Error(`--sampleSize cannot exceed ${MAX_SAMPLE_SIZE}.`);
+ }
return {
programId,
@@ -104,6 +165,8 @@ function parseArguments(args: string[]): BenchmarkArguments {
concurrency,
pageSize,
thresholdMs,
+ sampleSize,
+ maxErrorRate,
};
}
@@ -174,8 +237,13 @@ function createSearchCases(document: PartnerSearchDocument): SearchCase[] {
];
}
-async function loadSearchCases(programId: string): Promise<SearchCase[]> {
- const enrollment = await prisma.programEnrollment.findFirst({
+async function loadSearchCasePools(
+ programId: string,
+ sampleSize: number,
+ partnerCount: number,
+): Promise<SearchCasePool[]> {
+ const stride = Math.max(1, Math.floor(partnerCount / sampleSize));
+ const enrollments = await prisma.programEnrollment.findMany({
where: {
programId,
partner: {
@@ -187,15 +255,63 @@ async function loadSearchCases(programId: string): Promise<SearchCase[]> {
links: { some: {} },
},
select: partnerSearchDocumentSelect,
+ orderBy: { id: "asc" },
+ take: sampleSize * stride,
});
- if (!enrollment) {
+ const sampled = enrollments.filter((_, index) => index % stride === 0);
+
+ if (sampled.length === 0) {
throw new Error(
`No complete partner search document found for program ${programId}.`,
);
}
- return createSearchCases(serializePartnerSearchDocument(enrollment));
+ const queriesByField = new Map<string, Set<string>>();
+
+ for (const enrollment of sampled) {
+ const document = serializePartnerSearchDocument(enrollment);
+
+ for (const { field, query } of createSearchCases(document)) {
+ const queries = queriesByField.get(field) ?? new Set<string>();
+ queries.add(query);
+ queriesByField.set(field, queries);
+ }
+ }
+
+ return Array.from(queriesByField, ([field, queries]) => ({
+ field,
+ queries: Array.from(queries),
+ }));
+}
+
+function reportOverlappingPools(pools: SearchCasePool[]) {
+ const fieldsByQuerySet = new Map<string, string[]>();
+
+ for (const { field, queries } of pools) {
+ const key = JSON.stringify(queries);
+ fieldsByQuerySet.set(key, [...(fieldsByQuerySet.get(key) ?? []), field]);
+ }
+
+ for (const fields of fieldsByQuerySet.values()) {
+ if (fields.length > 1) {
+ console.warn(
+ `⚠️ These fields resolve to identical queries and measure the same work: ${fields.join(", ")}.`,
+ );
+ }
+ }
+}
+
+function reportRepeatedQueries(pools: SearchCasePool[], requests: number) {
+ const requestsPerField = Math.ceil(requests / pools.length);
+
+ for (const { field, queries } of pools) {
+ if (queries.length < requestsPerField) {
+ console.warn(
+ `⚠️ "${field}" has only ${queries.length.toLocaleString()} distinct ${queries.length === 1 ? "query" : "queries"} for ${requestsPerField.toLocaleString()} requests, so each repeats ~${Math.round(requestsPerField / queries.length)}x and is likely served warm.`,
+ );
+ }
+ }
}
function percentile(values: number[], quantile: number): number {
@@ -254,10 +370,20 @@ async function main() {
throw new Error(`Program ${options.programId} has 0 partners.`);
}
- const searchCases = await loadSearchCases(options.programId);
+ const pools = await loadSearchCasePools(
+ options.programId,
+ options.sampleSize,
+ partnerCount,
+ );
const runSearch = async (index: number): Promise<BenchmarkResult> => {
- const searchCase = searchCases[index % searchCases.length];
+ // Rotate fields on every request and advance that field's query once per
+ // full rotation, so consecutive requests never repeat a query until the
+ // pool is exhausted. Deterministic, so two runs issue the same work.
+ const pool = pools[index % pools.length];
+ const query =
+ pool.queries[Math.floor(index / pools.length) % pool.queries.length];
+ const searchCase: SearchCase = { field: pool.field, query };
const filters = {
programId: options.programId,
search: searchCase.query,
@@ -297,9 +423,15 @@ async function main() {
console.log(
`${options.requests.toLocaleString()} measured requests, ${options.warmupRequests.toLocaleString()} warm-up requests, concurrency ${options.concurrency}`,
);
+ const distinctQueries = pools.reduce(
+ (total, { queries }) => total + queries.length,
+ 0,
+ );
console.log(
- `Each request runs the relevance-ranked partner list path across ${searchCases.length} search cases.`,
+ `Each request runs the relevance-ranked partner list path across ${pools.length} search cases, drawing from ${distinctQueries.toLocaleString()} distinct queries sampled from ${options.sampleSize.toLocaleString()} partners.`,
);
+ reportOverlappingPools(pools);
+ reportRepeatedQueries(pools, options.requests);
const warmupResults = await runWithConcurrency(
options.warmupRequests,
@@ -317,7 +449,9 @@ async function main() {
const results = await runWithConcurrency(
options.requests,
options.concurrency,
- runSearch,
+ // Continue past the warm-up indices so the measured run does not reissue the
+ // queries the warm-up just cached.
+ (index) => runSearch(options.warmupRequests + index),
);
const elapsedMs = performance.now() - startedAt;
const successfulResults = results.filter(({ error }) => error === null);
@@ -329,7 +463,8 @@ async function main() {
latencies.length
: null;
const p99 = latencies.length > 0 ? percentile(latencies, 0.99) : null;
- const caseSummaries = searchCases.map(({ field, query }) => {
+ const caseSummaries = pools.map(({ field, queries }) => {
+ const query = `${queries[0]}${queries.length > 1 ? ` (+${queries.length - 1})` : ""}`;
const caseResults = results.filter((result) => result.field === field);
const caseErrors = caseResults.filter(({ error }) => error !== null).length;
const caseLatencies = caseResults
@@ -341,7 +476,8 @@ async function main() {
query,
samples: caseResults.length,
errors: caseErrors,
- errorRate: (caseErrors / caseResults.length) * 100,
+ errorRate:
+ caseResults.length > 0 ? (caseErrors / caseResults.length) * 100 : 0,
p50Ms: caseLatencies.length > 0 ? percentile(caseLatencies, 0.5) : null,
p95Ms: caseLatencies.length > 0 ? percentile(caseLatencies, 0.95) : null,
p99Ms: caseLatencies.length > 0 ? percentile(caseLatencies, 0.99) : null,
@@ -405,9 +541,23 @@ async function main() {
requestsPerSecond: ((options.requests * 1_000) / elapsedMs).toFixed(1),
});
- if (failedResults.length > 0) {
- console.error(
- `${failedResults.length.toLocaleString()} of ${results.length.toLocaleString()} measured requests failed (${((failedResults.length / results.length) * 100).toFixed(2)}% error rate).`,
+ const casesWithoutSamples = caseSummaries.filter(
+ ({ p99Ms }) => p99Ms === null,
+ );
+ if (casesWithoutSamples.length > 0) {
+ throw new Error(
+ `No successful requests for: ${casesWithoutSamples
+ .map(({ field }) => field)
+ .join(
+ ", ",
+ )}. There is no p99 to compare against the threshold — check that the search index is backfilled for this program.`,
+ );
+ }
+
+ const errorRate = (failedResults.length / results.length) * 100;
+ if (errorRate > options.maxErrorRate) {
+ throw new Error(
+ `${failedResults.length.toLocaleString()} of ${results.length.toLocaleString()} measured requests failed (${errorRate.toFixed(2)}% error rate, --maxErrorRate=${options.maxErrorRate}).`,
);
}
@@ -418,7 +568,7 @@ async function main() {
}
console.log(
- `Completed: every search case has p99 latency below ${options.thresholdMs}ms${failedResults.length > 0 ? ", with request errors reported above" : ""}.`,
+ `Completed: every search case has p99 latency below ${options.thresholdMs}ms${failedResults.length > 0 ? `, with a ${errorRate.toFixed(2)}% error rate within the --maxErrorRate=${options.maxErrorRate} allowance` : ""}.`,
);
}
diff --git a/apps/web/scripts/dev/debug-partner-search.ts b/apps/web/scripts/dev/debug-partner-search.ts
index 915b86da2e..84574f26e1 100644
--- a/apps/web/scripts/dev/debug-partner-search.ts
+++ b/apps/web/scripts/dev/debug-partner-search.ts
@@ -10,7 +10,7 @@ import {
type PartnerSearchHit,
} from "@/lib/api/partners/search";
import { prisma } from "@/lib/prisma";
-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";
+import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";
import { ProgramEnrollmentStatus } from "@prisma/client";
import "dotenv-flow/config";
diff --git a/apps/web/scripts/dev/seed-100k-partners.ts b/apps/web/scripts/dev/seed-100k-partners.ts
index 6f5027cb51..7ce2a0c269 100644
--- a/apps/web/scripts/dev/seed-100k-partners.ts
+++ b/apps/web/scripts/dev/seed-100k-partners.ts
@@ -24,7 +24,7 @@
import { createId } from "@/lib/api/create-id";
import { hashPassword } from "@/lib/auth/password";
import { prisma } from "@/lib/prisma";
-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";
+import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";
import { PlatformType, Prisma } from "@prisma/client";
import { createHash } from "crypto";
import "dotenv-flow/config";
diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.ts
index 83a7d29be3..fb349c9ddf 100644
--- a/apps/web/scripts/partners/backfill-partner-search.ts
+++ b/apps/web/scripts/partners/backfill-partner-search.ts
@@ -5,7 +5,7 @@ import {
type PartnerSearchBackfillProgress,
} from "@/lib/api/partners/search";
import { prisma } from "@/lib/prisma";
-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";
+import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";
import "dotenv-flow/config";
const DEFAULT_BATCH_SIZE = 500;
diff --git a/apps/web/scripts/utils/parse-positive-integer.ts b/apps/web/scripts/utils/parse-cli-number.ts
similarity index 56%
rename from apps/web/scripts/utils/parse-positive-integer.ts
rename to apps/web/scripts/utils/parse-cli-number.ts
index b7fe12fa50..0cef2f84be 100644
--- a/apps/web/scripts/utils/parse-positive-integer.ts
+++ b/apps/web/scripts/utils/parse-cli-number.ts
@@ -1,7 +1,7 @@
const DIGITS_ONLY = /^\d+$/;
/**
- * Parses a CLI flag value as a positive integer.
+ * Parses a CLI flag value as an integer of at least `minimum`.
*
* The digits-only check has to run before `Number()`, which on its own would
* silently accept hex (`0x10` → 16), scientific notation (`1e9` → 1000000000),
@@ -10,13 +10,18 @@ const DIGITS_ONLY = /^\d+$/;
* rows. `Number.isSafeInteger` then rejects digit strings past 2^53 - 1, which
* cannot round-trip.
*/
-export function parsePositiveInteger(value: string | undefined, flag: string) {
+function parseInteger(
+ value: string | undefined,
+ flag: string,
+ minimum: number,
+ expected: string,
+) {
const parsed =
value !== undefined && DIGITS_ONLY.test(value) ? Number(value) : NaN;
- if (!Number.isSafeInteger(parsed) || parsed <= 0) {
+ if (!Number.isSafeInteger(parsed) || parsed < minimum) {
throw new Error(
- `${flag} must be a positive integer, received: ${
+ `${flag} must be ${expected}, received: ${
value === undefined ? "(missing)" : `"${value}"`
}`,
);
@@ -24,3 +29,15 @@ export function parsePositiveInteger(value: string | undefined, flag: string) {
return parsed;
}
+
+export function parsePositiveInteger(value: string | undefined, flag: string) {
+ return parseInteger(value, flag, 1, "a positive integer");
+}
+
+/** For flags where zero is meaningful, such as disabling benchmark warm-up. */
+export function parseNonNegativeInteger(
+ value: string | undefined,
+ flag: string,
+) {
+ return parseInteger(value, flag, 0, "a non-negative integer");
+}
diff --git a/apps/web/tests/scripts/parse-positive-integer.test.ts b/apps/web/tests/scripts/parse-cli-number.test.ts
similarity index 55%
rename from apps/web/tests/scripts/parse-positive-integer.test.ts
rename to apps/web/tests/scripts/parse-cli-number.test.ts
index 2b168e39af..942b7f5653 100644
--- a/apps/web/tests/scripts/parse-positive-integer.test.ts
+++ b/apps/web/tests/scripts/parse-cli-number.test.ts
@@ -1,4 +1,7 @@
-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";
+import {
+ parseNonNegativeInteger,
+ parsePositiveInteger,
+} from "@/scripts/utils/parse-cli-number";
import { describe, expect, it } from "vitest";
describe("parsePositiveInteger", () => {
@@ -37,3 +40,31 @@ describe("parsePositiveInteger", () => {
);
});
});
+
+describe("parseNonNegativeInteger", () => {
+ it("accepts zero, which parsePositiveInteger rejects", () => {
+ expect(parseNonNegativeInteger("0", "--warmup")).toBe(0);
+ expect(() => parsePositiveInteger("0", "--warmup")).toThrow(
+ "--warmup must be a positive integer",
+ );
+ });
+
+ it("parses plain digit strings", () => {
+ expect(parseNonNegativeInteger("50", "--warmup")).toBe(50);
+ });
+
+ it.each([["0x10"], ["1e9"], ["5.0"], ["+5"], [" 5 "], ["-1"], [""], ["abc"]])(
+ "applies the same coercion guard to %j",
+ (value) => {
+ expect(() => parseNonNegativeInteger(value, "--warmup")).toThrow(
+ "--warmup must be a non-negative integer",
+ );
+ },
+ );
+
+ it("reports the offending value and the flag name", () => {
+ expect(() => parseNonNegativeInteger("abc", "--warmup")).toThrow(
+ '--warmup must be a non-negative integer, received: "abc"',
+ );
+ });
+});