The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Improve the benchmark script.
Rafic760876a862c
diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.tsindex d893e8b29e..fc2533aa87 100644--- a/apps/web/scripts/dev/benchmark-partner-search.ts+++ b/apps/web/scripts/dev/benchmark-partner-search.ts@@ -1,3 +1,26 @@+/**+ * Measures p99 latency of the relevance-ranked partner list for a program.+ *+ * Calls `getPartners` in process, not over HTTP, so the numbers exclude route+ * handling, auth, and serialization — treat them as a floor for the endpoint.+ *+ * To avoid measuring a warm cache it samples `--sampleSize` partners strided+ * across the program and exhausts a field's queries before repeating any. Fields+ * whose values barely vary still collapse to a few queries, so each field's+ * distinct-query count is reported and undersized pools are flagged; a p99 under+ * that warning is optimistic.+ *+ * Exits non-zero when a field produced no successful request, when the error rate+ * exceeds `--maxErrorRate`, or when a p99 reaches `--thresholdMs`.+ *+ * cd apps/web+ * pnpm run script dev/benchmark-partner-search --programId=prog_123 [--requests=1000]+ * [--sampleSize=100] [--concurrency=10] [--warmup=50] [--thresholdMs=1000]+ *+ * Requires PARTNER_SEARCH_PROVIDER and an index backfilled for the program:+ * pnpm run script partners/backfill-partner-search --programId=prog_123+ */+import { getPartners } from "@/lib/api/partners/get-partners";import {getPartnerSearchProvider,@@ -8,7 +31,10 @@ import {type PartnerSearchDocument,} from "@/lib/api/partners/search";import { prisma } from "@/lib/prisma";-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";+import {+ parseNonNegativeInteger,+ parsePositiveInteger,+} from "@/scripts/utils/parse-cli-number";import "dotenv-flow/config";const DEFAULT_REQUESTS = 1_000;@@ -17,6 +43,9 @@ const DEFAULT_CONCURRENCY = 10;const DEFAULT_PAGE_SIZE = 25;const DEFAULT_THRESHOLD_MS = 1_000;const MINIMUM_REQUESTS = 1_000;+const DEFAULT_SAMPLE_SIZE = 100;+const MAX_SAMPLE_SIZE = 1_000;+const DEFAULT_MAX_ERROR_RATE = 0;interface BenchmarkArguments {programId: string;@@ -25,6 +54,8 @@ interface BenchmarkArguments {concurrency: number;pageSize: number;thresholdMs: number;+ sampleSize: number;+ maxErrorRate: number;}interface SearchCase {@@ -32,6 +63,11 @@ interface SearchCase {query: string;}+interface SearchCasePool {+ field: string;+ queries: string[];+}+interface BenchmarkResult {field: string;query: string;@@ -39,6 +75,16 @@ interface BenchmarkResult {error: string | null;}+function parsePercentage(value: string, flag: string): number {+ const parsed = /^\d+(\.\d+)?$/.test(value) ? Number(value) : NaN;++ if (!Number.isFinite(parsed) || parsed > 100) {+ throw new Error(`${flag} must be between 0 and 100, received: "${value}"`);+ }++ return parsed;+}+function parseArguments(args: string[]): BenchmarkArguments {let programId: string | undefined;let requests = DEFAULT_REQUESTS;@@ -46,6 +92,8 @@ function parseArguments(args: string[]): BenchmarkArguments {let concurrency = DEFAULT_CONCURRENCY;let pageSize = DEFAULT_PAGE_SIZE;let thresholdMs = DEFAULT_THRESHOLD_MS;+ let sampleSize = DEFAULT_SAMPLE_SIZE;+ let maxErrorRate = DEFAULT_MAX_ERROR_RATE;for (const arg of args) {if (arg.startsWith("--programId=")) {@@ -56,10 +104,20 @@ function parseArguments(args: string[]): BenchmarkArguments {"--requests",);} else if (arg.startsWith("--warmup=")) {- warmupRequests = parsePositiveInteger(+ warmupRequests = parseNonNegativeInteger(arg.slice("--warmup=".length),"--warmup",);+ } else if (arg.startsWith("--sampleSize=")) {+ sampleSize = parsePositiveInteger(+ arg.slice("--sampleSize=".length),+ "--sampleSize",+ );+ } else if (arg.startsWith("--maxErrorRate=")) {+ maxErrorRate = parsePercentage(+ arg.slice("--maxErrorRate=".length),+ "--maxErrorRate",+ );} else if (arg.startsWith("--concurrency=")) {concurrency = parsePositiveInteger(arg.slice("--concurrency=".length),@@ -96,6 +154,9 @@ function parseArguments(args: string[]): BenchmarkArguments {`--pageSize cannot exceed ${PARTNER_SEARCH_CANDIDATE_LIMIT}.`,);}+ if (sampleSize > MAX_SAMPLE_SIZE) {+ throw new Error(`--sampleSize cannot exceed ${MAX_SAMPLE_SIZE}.`);+ }return {programId,@@ -104,6 +165,8 @@ function parseArguments(args: string[]): BenchmarkArguments {concurrency,pageSize,thresholdMs,+ sampleSize,+ maxErrorRate,};}@@ -174,8 +237,13 @@ function createSearchCases(document: PartnerSearchDocument): SearchCase[] {];}-async function loadSearchCases(programId: string): Promise<SearchCase[]> {- const enrollment = await prisma.programEnrollment.findFirst({+async function loadSearchCasePools(+ programId: string,+ sampleSize: number,+ partnerCount: number,+): Promise<SearchCasePool[]> {+ const stride = Math.max(1, Math.floor(partnerCount / sampleSize));+ const enrollments = await prisma.programEnrollment.findMany({where: {programId,partner: {@@ -187,15 +255,63 @@ async function loadSearchCases(programId: string): Promise<SearchCase[]> {links: { some: {} },},select: partnerSearchDocumentSelect,+ orderBy: { id: "asc" },+ take: sampleSize * stride,});- if (!enrollment) {+ const sampled = enrollments.filter((_, index) => index % stride === 0);++ if (sampled.length === 0) {throw new Error(`No complete partner search document found for program ${programId}.`,);}- return createSearchCases(serializePartnerSearchDocument(enrollment));+ const queriesByField = new Map<string, Set<string>>();++ for (const enrollment of sampled) {+ const document = serializePartnerSearchDocument(enrollment);++ for (const { field, query } of createSearchCases(document)) {+ const queries = queriesByField.get(field) ?? new Set<string>();+ queries.add(query);+ queriesByField.set(field, queries);+ }+ }++ return Array.from(queriesByField, ([field, queries]) => ({+ field,+ queries: Array.from(queries),+ }));+}++function reportOverlappingPools(pools: SearchCasePool[]) {+ const fieldsByQuerySet = new Map<string, string[]>();++ for (const { field, queries } of pools) {+ const key = JSON.stringify(queries);+ fieldsByQuerySet.set(key, [...(fieldsByQuerySet.get(key) ?? []), field]);+ }++ for (const fields of fieldsByQuerySet.values()) {+ if (fields.length > 1) {+ console.warn(+ `⚠️ These fields resolve to identical queries and measure the same work: ${fields.join(", ")}.`,+ );+ }+ }+}++function reportRepeatedQueries(pools: SearchCasePool[], requests: number) {+ const requestsPerField = Math.ceil(requests / pools.length);++ for (const { field, queries } of pools) {+ if (queries.length < requestsPerField) {+ console.warn(+ `⚠️ "${field}" has only ${queries.length.toLocaleString()} distinct ${queries.length === 1 ? "query" : "queries"} for ${requestsPerField.toLocaleString()} requests, so each repeats ~${Math.round(requestsPerField / queries.length)}x and is likely served warm.`,+ );+ }+ }}function percentile(values: number[], quantile: number): number {@@ -254,10 +370,20 @@ async function main() {throw new Error(`Program ${options.programId} has 0 partners.`);}- const searchCases = await loadSearchCases(options.programId);+ const pools = await loadSearchCasePools(+ options.programId,+ options.sampleSize,+ partnerCount,+ );const runSearch = async (index: number): Promise<BenchmarkResult> => {- const searchCase = searchCases[index % searchCases.length];+ // Rotate fields on every request and advance that field's query once per+ // full rotation, so consecutive requests never repeat a query until the+ // pool is exhausted. Deterministic, so two runs issue the same work.+ const pool = pools[index % pools.length];+ const query =+ pool.queries[Math.floor(index / pools.length) % pool.queries.length];+ const searchCase: SearchCase = { field: pool.field, query };const filters = {programId: options.programId,search: searchCase.query,@@ -297,9 +423,15 @@ async function main() {console.log(`${options.requests.toLocaleString()} measured requests, ${options.warmupRequests.toLocaleString()} warm-up requests, concurrency ${options.concurrency}`,);+ const distinctQueries = pools.reduce(+ (total, { queries }) => total + queries.length,+ 0,+ );console.log(- `Each request runs the relevance-ranked partner list path across ${searchCases.length} search cases.`,+ `Each request runs the relevance-ranked partner list path across ${pools.length} search cases, drawing from ${distinctQueries.toLocaleString()} distinct queries sampled from ${options.sampleSize.toLocaleString()} partners.`,);+ reportOverlappingPools(pools);+ reportRepeatedQueries(pools, options.requests);const warmupResults = await runWithConcurrency(options.warmupRequests,@@ -317,7 +449,9 @@ async function main() {const results = await runWithConcurrency(options.requests,options.concurrency,- runSearch,+ // Continue past the warm-up indices so the measured run does not reissue the+ // queries the warm-up just cached.+ (index) => runSearch(options.warmupRequests + index),);const elapsedMs = performance.now() - startedAt;const successfulResults = results.filter(({ error }) => error === null);@@ -329,7 +463,8 @@ async function main() {latencies.length: null;const p99 = latencies.length > 0 ? percentile(latencies, 0.99) : null;- const caseSummaries = searchCases.map(({ field, query }) => {+ const caseSummaries = pools.map(({ field, queries }) => {+ const query = `${queries[0]}${queries.length > 1 ? ` (+${queries.length - 1})` : ""}`;const caseResults = results.filter((result) => result.field === field);const caseErrors = caseResults.filter(({ error }) => error !== null).length;const caseLatencies = caseResults@@ -341,7 +476,8 @@ async function main() {query,samples: caseResults.length,errors: caseErrors,- errorRate: (caseErrors / caseResults.length) * 100,+ errorRate:+ caseResults.length > 0 ? (caseErrors / caseResults.length) * 100 : 0,p50Ms: caseLatencies.length > 0 ? percentile(caseLatencies, 0.5) : null,p95Ms: caseLatencies.length > 0 ? percentile(caseLatencies, 0.95) : null,p99Ms: caseLatencies.length > 0 ? percentile(caseLatencies, 0.99) : null,@@ -405,9 +541,23 @@ async function main() {requestsPerSecond: ((options.requests * 1_000) / elapsedMs).toFixed(1),});- if (failedResults.length > 0) {- console.error(- `${failedResults.length.toLocaleString()} of ${results.length.toLocaleString()} measured requests failed (${((failedResults.length / results.length) * 100).toFixed(2)}% error rate).`,+ const casesWithoutSamples = caseSummaries.filter(+ ({ p99Ms }) => p99Ms === null,+ );+ if (casesWithoutSamples.length > 0) {+ throw new Error(+ `No successful requests for: ${casesWithoutSamples+ .map(({ field }) => field)+ .join(+ ", ",+ )}. There is no p99 to compare against the threshold — check that the search index is backfilled for this program.`,+ );+ }++ const errorRate = (failedResults.length / results.length) * 100;+ if (errorRate > options.maxErrorRate) {+ throw new Error(+ `${failedResults.length.toLocaleString()} of ${results.length.toLocaleString()} measured requests failed (${errorRate.toFixed(2)}% error rate, --maxErrorRate=${options.maxErrorRate}).`,);}@@ -418,7 +568,7 @@ async function main() {}console.log(- `Completed: every search case has p99 latency below ${options.thresholdMs}ms${failedResults.length > 0 ? ", with request errors reported above" : ""}.`,+ `Completed: every search case has p99 latency below ${options.thresholdMs}ms${failedResults.length > 0 ? `, with a ${errorRate.toFixed(2)}% error rate within the --maxErrorRate=${options.maxErrorRate} allowance` : ""}.`,);}diff --git a/apps/web/scripts/dev/debug-partner-search.ts b/apps/web/scripts/dev/debug-partner-search.tsindex 915b86da2e..84574f26e1 100644--- a/apps/web/scripts/dev/debug-partner-search.ts+++ b/apps/web/scripts/dev/debug-partner-search.ts@@ -10,7 +10,7 @@ import {type PartnerSearchHit,} from "@/lib/api/partners/search";import { prisma } from "@/lib/prisma";-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";+import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";import { ProgramEnrollmentStatus } from "@prisma/client";import "dotenv-flow/config";diff --git a/apps/web/scripts/dev/seed-100k-partners.ts b/apps/web/scripts/dev/seed-100k-partners.tsindex 6f5027cb51..7ce2a0c269 100644--- a/apps/web/scripts/dev/seed-100k-partners.ts+++ b/apps/web/scripts/dev/seed-100k-partners.ts@@ -24,7 +24,7 @@import { createId } from "@/lib/api/create-id";import { hashPassword } from "@/lib/auth/password";import { prisma } from "@/lib/prisma";-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";+import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";import { PlatformType, Prisma } from "@prisma/client";import { createHash } from "crypto";import "dotenv-flow/config";diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.tsindex 83a7d29be3..fb349c9ddf 100644--- a/apps/web/scripts/partners/backfill-partner-search.ts+++ b/apps/web/scripts/partners/backfill-partner-search.ts@@ -5,7 +5,7 @@ import {type PartnerSearchBackfillProgress,} from "@/lib/api/partners/search";import { prisma } from "@/lib/prisma";-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";+import { parsePositiveInteger } from "@/scripts/utils/parse-cli-number";import "dotenv-flow/config";const DEFAULT_BATCH_SIZE = 500;diff --git a/apps/web/scripts/utils/parse-positive-integer.ts b/apps/web/scripts/utils/parse-cli-number.tssimilarity index 56%rename from apps/web/scripts/utils/parse-positive-integer.tsrename to apps/web/scripts/utils/parse-cli-number.tsindex b7fe12fa50..0cef2f84be 100644--- a/apps/web/scripts/utils/parse-positive-integer.ts+++ b/apps/web/scripts/utils/parse-cli-number.ts@@ -1,7 +1,7 @@const DIGITS_ONLY = /^\d+$/;/**- * Parses a CLI flag value as a positive integer.+ * Parses a CLI flag value as an integer of at least `minimum`.** The digits-only check has to run before `Number()`, which on its own would* silently accept hex (`0x10` → 16), scientific notation (`1e9` → 1000000000),@@ -10,13 +10,18 @@ const DIGITS_ONLY = /^\d+$/;* rows. `Number.isSafeInteger` then rejects digit strings past 2^53 - 1, which* cannot round-trip.*/-export function parsePositiveInteger(value: string | undefined, flag: string) {+function parseInteger(+ value: string | undefined,+ flag: string,+ minimum: number,+ expected: string,+) {const parsed =value !== undefined && DIGITS_ONLY.test(value) ? Number(value) : NaN;- if (!Number.isSafeInteger(parsed) || parsed <= 0) {+ if (!Number.isSafeInteger(parsed) || parsed < minimum) {throw new Error(- `${flag} must be a positive integer, received: ${+ `${flag} must be ${expected}, received: ${value === undefined ? "(missing)" : `"${value}"`}`,);@@ -24,3 +29,15 @@ export function parsePositiveInteger(value: string | undefined, flag: string) {return parsed;}++export function parsePositiveInteger(value: string | undefined, flag: string) {+ return parseInteger(value, flag, 1, "a positive integer");+}++/** For flags where zero is meaningful, such as disabling benchmark warm-up. */+export function parseNonNegativeInteger(+ value: string | undefined,+ flag: string,+) {+ return parseInteger(value, flag, 0, "a non-negative integer");+}diff --git a/apps/web/tests/scripts/parse-positive-integer.test.ts b/apps/web/tests/scripts/parse-cli-number.test.tssimilarity index 55%rename from apps/web/tests/scripts/parse-positive-integer.test.tsrename to apps/web/tests/scripts/parse-cli-number.test.tsindex 2b168e39af..942b7f5653 100644--- a/apps/web/tests/scripts/parse-positive-integer.test.ts+++ b/apps/web/tests/scripts/parse-cli-number.test.ts@@ -1,4 +1,7 @@-import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";+import {+ parseNonNegativeInteger,+ parsePositiveInteger,+} from "@/scripts/utils/parse-cli-number";import { describe, expect, it } from "vitest";describe("parsePositiveInteger", () => {@@ -37,3 +40,31 @@ describe("parsePositiveInteger", () => {);});});++describe("parseNonNegativeInteger", () => {+ it("accepts zero, which parsePositiveInteger rejects", () => {+ expect(parseNonNegativeInteger("0", "--warmup")).toBe(0);+ expect(() => parsePositiveInteger("0", "--warmup")).toThrow(+ "--warmup must be a positive integer",+ );+ });++ it("parses plain digit strings", () => {+ expect(parseNonNegativeInteger("50", "--warmup")).toBe(50);+ });++ it.each([["0x10"], ["1e9"], ["5.0"], ["+5"], [" 5 "], ["-1"], [""], ["abc"]])(+ "applies the same coercion guard to %j",+ (value) => {+ expect(() => parseNonNegativeInteger(value, "--warmup")).toThrow(+ "--warmup must be a non-negative integer",+ );+ },+ );++ it("reports the offending value and the flag name", () => {+ expect(() => parseNonNegativeInteger("abc", "--warmup")).toThrow(+ '--warmup must be a non-negative integer, received: "abc"',+ );+ });+});