Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Pace TypeSafe requests against its per-second limits (#46)
Jaafer Rahmani43e6a81deca0
diff --git a/packages/core/src/evaluator.ts b/packages/core/src/evaluator.tsindex 360ddbc..3468917 100644--- a/packages/core/src/evaluator.ts+++ b/packages/core/src/evaluator.ts@@ -51,7 +51,7 @@ export function createEvaluator(options: {let cooldownUntil = 0;const rateBudget =options.provider === "typesafe"- ? createRateBudget({ tokensPerSecond: 250_000, requestsPerMinute: 1_200 })+ ? createRateBudget({ tokensPerSecond: 100_000, requestsPerSecond: 40 }): undefined;const authenticationFailure = new AbortController();function assertActive() {diff --git a/packages/core/src/rate-budget.ts b/packages/core/src/rate-budget.tsindex a66b6dd..5ec183a 100644--- a/packages/core/src/rate-budget.ts+++ b/packages/core/src/rate-budget.ts@@ -1,6 +1,6 @@-/** Per-evaluator rolling budgets. Concurrency remains owned by the evaluator. */+/** Per-evaluator one-second rolling budgets. Concurrency remains owned by the evaluator. */export function createRateBudget(- limits: { tokensPerSecond: number; requestsPerMinute: number },+ limits: { tokensPerSecond: number; requestsPerSecond: number },now = () => performance.now(),) {const starts: Array<{ at: number; tokens: number }> = [];@@ -8,14 +8,13 @@ export function createRateBudget(if (!Number.isSafeInteger(tokens) || tokens < 0 || tokens > limits.tokensPerSecond)throw new Error("Token reservation exceeds the rate budget");const time = now();- while (starts.length && starts[0]!.at <= time - 60_000) starts.shift();+ while (starts.length && starts[0]!.at <= time - 1000) starts.shift();let wait =- starts.length >= limits.requestsPerMinute- ? starts[starts.length - limits.requestsPerMinute]!.at + 60_000 - time+ starts.length >= limits.requestsPerSecond+ ? starts[starts.length - limits.requestsPerSecond]!.at + 1000 - time: 0;- const recent = starts.filter((entry) => entry.at > time - 1000);- let total = recent.reduce((sum, entry) => sum + entry.tokens, 0) + tokens;- for (const entry of recent) {+ let total = starts.reduce((sum, entry) => sum + entry.tokens, 0) + tokens;+ for (const entry of starts) {if (total <= limits.tokensPerSecond) break;total -= entry.tokens;wait = Math.max(wait, entry.at + 1000 - time);diff --git a/packages/core/test/rate-budget.test.ts b/packages/core/test/rate-budget.test.tsindex c7b08ad..43c7313 100644--- a/packages/core/test/rate-budget.test.ts+++ b/packages/core/test/rate-budget.test.ts@@ -3,7 +3,7 @@ import { createRateBudget } from "../src/rate-budget";test("rolling budgets reconcile usage and enforce the stricter token or request window", () => {let now = 0;- const budget = createRateBudget({ tokensPerSecond: 100, requestsPerMinute: 3 }, () => now);+ const budget = createRateBudget({ tokensPerSecond: 100, requestsPerSecond: 3 }, () => now);const first = budget.reserve(80);expect(budget.waitMs(30)).toBe(1000);first.reconcile(undefined);@@ -13,9 +13,29 @@ test("rolling budgets reconcile usage and enforce the stricter token or requestbudget.reserve(30);now = 500;budget.reserve(40);- expect(budget.waitMs(20)).toBe(59_500);- now = 60_000;- expect(budget.waitMs(100)).toBe(0);+ expect(budget.waitMs(20)).toBe(500);+ now = 1000;+ expect(budget.waitMs(60)).toBe(0);+ expect(budget.waitMs(61)).toBe(500);+ now = 1500;budget.reserve(100);expect(budget.waitMs(1)).toBe(1000);});++test("request reservations expire individually at the one-second boundary", () => {+ let now = 0;+ const budget = createRateBudget({ tokensPerSecond: 100, requestsPerSecond: 3 }, () => now);+ budget.reserve(0);+ now = 200;+ budget.reserve(0);+ now = 500;+ budget.reserve(0);+ now = 999;+ expect(budget.waitMs(0)).toBe(1);+ now = 1000;+ expect(budget.waitMs(0)).toBe(0);+ budget.reserve(0);+ expect(budget.waitMs(0)).toBe(200);+ now = 1200;+ expect(budget.waitMs(0)).toBe(0);+});diff --git a/test/evaluator.test.ts b/test/evaluator.test.tsindex f444291..51fbbd4 100644--- a/test/evaluator.test.ts+++ b/test/evaluator.test.ts@@ -555,15 +555,17 @@ test("custom cache identity keeps endpoints and models separate from presets", a});test("native token pacing preserves concurrent small calls and excludes queue time from timeout", async () => {- const arrivals: Array<{ at: number; tokens: number }> = [];+ const arrivals: number[] = [];+ const starts: Array<{ at: number; tokens: number }> = [];+ const inputTokens = (state: string) => (state.length > 1000 ? 60_000 : 100);let active = 0;let peak = 0;const server = Bun.serve({port: 0,async fetch(request) {const body = (await request.json()) as { state: string };- const tokens = body.state.length > 1000 ? 60_000 : 100;- arrivals.push({ at: performance.now(), tokens });+ const tokens = inputTokens(body.state);+ arrivals.push(performance.now());peak = Math.max(peak, ++active);await new Promise((resolve) => setTimeout(resolve, 80));active--;@@ -579,12 +581,16 @@ test("native token pacing preserves concurrent small calls and excludes queue tiapiKey: "fixture",signal: new AbortController().signal,timeoutMs: 500,- fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),+ fetch: routeProviderFetch((input, init) => {+ const body = JSON.parse(init!.body as string) as { state: string };+ starts.push({ at: performance.now(), tokens: inputTokens(body.state) });+ return fetch(input, init);+ }, `http://127.0.0.1:${server.port}`),});const values = await Promise.all(Array.from({ length: 32 }, (_, i) =>evaluator.evaluate({- state: i < 24 ? "small" : "x".repeat(65_000),+ state: i < 29 ? "small" : "x".repeat(65_000),questions: { q: { type: "boolean", instructions: "Relevant?" } },}),),@@ -592,12 +598,58 @@ test("native token pacing preserves concurrent small calls and excludes queue tiexpect(values.every((value) => value.q === 0.8)).toBe(true);expect(peak).toBeGreaterThan(4);expect(arrivals).toHaveLength(32);- for (const start of arrivals) {- // Leave a few milliseconds for loopback transport scheduling jitter.- const tokens = arrivals+ expect(starts).toHaveLength(32);+ for (const start of starts) {+ // Pace client starts; transport delay can compress server arrival windows.+ const tokens = starts.filter((entry) => entry.at >= start.at && entry.at < start.at + 990).reduce((sum, entry) => sum + entry.tokens, 0);- expect(tokens).toBeLessThanOrEqual(250_000);+ expect(tokens).toBeLessThanOrEqual(100_000);+ }+ } finally {+ server.stop(true);+ }+}, 10_000);++test("TypeSafe admits at most 40 small requests per second through real HTTP", async () => {+ const arrivals: number[] = [];+ const starts: number[] = [];+ const server = Bun.serve({+ port: 0,+ fetch() {+ arrivals.push(performance.now());+ return Response.json({+ answers: { q: { type: "noul", noul: 0.8 } },+ usage: { input_tokens: 1, output_tokens: 1 },+ });+ },+ });+ try {+ const evaluator = createEvaluator({+ provider: "typesafe",+ apiKey: "fixture",+ signal: new AbortController().signal,+ concurrency: 64,+ fetch: routeProviderFetch((input, init) => {+ starts.push(performance.now());+ return fetch(input, init);+ }, `http://127.0.0.1:${server.port}`),+ });+ const values = await Promise.all(+ Array.from({ length: 80 }, () =>+ evaluator.evaluate({+ state: "small",+ questions: { q: { type: "boolean", instructions: "Relevant?" } },+ }),+ ),+ );+ expect(values.every((value) => value.q === 0.8)).toBe(true);+ expect(arrivals).toHaveLength(80);+ expect(starts).toHaveLength(80);+ expect(starts[39]! - starts[0]!).toBeLessThan(900);+ for (const start of starts) {+ // Unit tests own the exact boundary; allow SDK dispatch scheduling jitter.+ expect(starts.filter((at) => at >= start && at < start + 900).length).toBeLessThanOrEqual(40);}} finally {server.stop(true);@@ -632,4 +684,4 @@ test("token-queued requests revalidate source after waiting", async () => {await Promise.all(running);expect(await queued).toMatchObject({ kind: "source-invalid" });expect(calls).toBe(3);-});+}, 10_000);