Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Pace TypeSafe requests against its per-second limits (#46)

Jaafer Rahmani43e6a81deca0
diff --git a/packages/core/src/evaluator.ts b/packages/core/src/evaluator.ts
index 360ddbc..3468917 100644
--- a/packages/core/src/evaluator.ts
+++ b/packages/core/src/evaluator.ts
@@ -51,7 +51,7 @@ export function createEvaluator(options: {
let cooldownUntil = 0;
const rateBudget =
options.provider === "typesafe"
- ? createRateBudget({ tokensPerSecond: 250_000, requestsPerMinute: 1_200 })
+ ? createRateBudget({ tokensPerSecond: 100_000, requestsPerSecond: 40 })
: undefined;
const authenticationFailure = new AbortController();
function assertActive() {
diff --git a/packages/core/src/rate-budget.ts b/packages/core/src/rate-budget.ts
index a66b6dd..5ec183a 100644
--- a/packages/core/src/rate-budget.ts
+++ b/packages/core/src/rate-budget.ts
@@ -1,6 +1,6 @@
-/** Per-evaluator rolling budgets. Concurrency remains owned by the evaluator. */
+/** Per-evaluator one-second rolling budgets. Concurrency remains owned by the evaluator. */
export function createRateBudget(
- limits: { tokensPerSecond: number; requestsPerMinute: number },
+ limits: { tokensPerSecond: number; requestsPerSecond: number },
now = () => performance.now(),
) {
const starts: Array<{ at: number; tokens: number }> = [];
@@ -8,14 +8,13 @@ export function createRateBudget(
if (!Number.isSafeInteger(tokens) || tokens < 0 || tokens > limits.tokensPerSecond)
throw new Error("Token reservation exceeds the rate budget");
const time = now();
- while (starts.length && starts[0]!.at <= time - 60_000) starts.shift();
+ while (starts.length && starts[0]!.at <= time - 1000) starts.shift();
let wait =
- starts.length >= limits.requestsPerMinute
- ? starts[starts.length - limits.requestsPerMinute]!.at + 60_000 - time
+ starts.length >= limits.requestsPerSecond
+ ? starts[starts.length - limits.requestsPerSecond]!.at + 1000 - time
: 0;
- const recent = starts.filter((entry) => entry.at > time - 1000);
- let total = recent.reduce((sum, entry) => sum + entry.tokens, 0) + tokens;
- for (const entry of recent) {
+ let total = starts.reduce((sum, entry) => sum + entry.tokens, 0) + tokens;
+ for (const entry of starts) {
if (total <= limits.tokensPerSecond) break;
total -= entry.tokens;
wait = Math.max(wait, entry.at + 1000 - time);
diff --git a/packages/core/test/rate-budget.test.ts b/packages/core/test/rate-budget.test.ts
index c7b08ad..43c7313 100644
--- a/packages/core/test/rate-budget.test.ts
+++ b/packages/core/test/rate-budget.test.ts
@@ -3,7 +3,7 @@ import { createRateBudget } from "../src/rate-budget";
test("rolling budgets reconcile usage and enforce the stricter token or request window", () => {
let now = 0;
- const budget = createRateBudget({ tokensPerSecond: 100, requestsPerMinute: 3 }, () => now);
+ const budget = createRateBudget({ tokensPerSecond: 100, requestsPerSecond: 3 }, () => now);
const first = budget.reserve(80);
expect(budget.waitMs(30)).toBe(1000);
first.reconcile(undefined);
@@ -13,9 +13,29 @@ test("rolling budgets reconcile usage and enforce the stricter token or request
budget.reserve(30);
now = 500;
budget.reserve(40);
- expect(budget.waitMs(20)).toBe(59_500);
- now = 60_000;
- expect(budget.waitMs(100)).toBe(0);
+ expect(budget.waitMs(20)).toBe(500);
+ now = 1000;
+ expect(budget.waitMs(60)).toBe(0);
+ expect(budget.waitMs(61)).toBe(500);
+ now = 1500;
budget.reserve(100);
expect(budget.waitMs(1)).toBe(1000);
});
+
+test("request reservations expire individually at the one-second boundary", () => {
+ let now = 0;
+ const budget = createRateBudget({ tokensPerSecond: 100, requestsPerSecond: 3 }, () => now);
+ budget.reserve(0);
+ now = 200;
+ budget.reserve(0);
+ now = 500;
+ budget.reserve(0);
+ now = 999;
+ expect(budget.waitMs(0)).toBe(1);
+ now = 1000;
+ expect(budget.waitMs(0)).toBe(0);
+ budget.reserve(0);
+ expect(budget.waitMs(0)).toBe(200);
+ now = 1200;
+ expect(budget.waitMs(0)).toBe(0);
+});
diff --git a/test/evaluator.test.ts b/test/evaluator.test.ts
index f444291..51fbbd4 100644
--- a/test/evaluator.test.ts
+++ b/test/evaluator.test.ts
@@ -555,15 +555,17 @@ test("custom cache identity keeps endpoints and models separate from presets", a
});
test("native token pacing preserves concurrent small calls and excludes queue time from timeout", async () => {
- const arrivals: Array<{ at: number; tokens: number }> = [];
+ const arrivals: number[] = [];
+ const starts: Array<{ at: number; tokens: number }> = [];
+ const inputTokens = (state: string) => (state.length > 1000 ? 60_000 : 100);
let active = 0;
let peak = 0;
const server = Bun.serve({
port: 0,
async fetch(request) {
const body = (await request.json()) as { state: string };
- const tokens = body.state.length > 1000 ? 60_000 : 100;
- arrivals.push({ at: performance.now(), tokens });
+ const tokens = inputTokens(body.state);
+ arrivals.push(performance.now());
peak = Math.max(peak, ++active);
await new Promise((resolve) => setTimeout(resolve, 80));
active--;
@@ -579,12 +581,16 @@ test("native token pacing preserves concurrent small calls and excludes queue ti
apiKey: "fixture",
signal: new AbortController().signal,
timeoutMs: 500,
- fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
+ fetch: routeProviderFetch((input, init) => {
+ const body = JSON.parse(init!.body as string) as { state: string };
+ starts.push({ at: performance.now(), tokens: inputTokens(body.state) });
+ return fetch(input, init);
+ }, `http://127.0.0.1:${server.port}`),
});
const values = await Promise.all(
Array.from({ length: 32 }, (_, i) =>
evaluator.evaluate({
- state: i < 24 ? "small" : "x".repeat(65_000),
+ state: i < 29 ? "small" : "x".repeat(65_000),
questions: { q: { type: "boolean", instructions: "Relevant?" } },
}),
),
@@ -592,12 +598,58 @@ test("native token pacing preserves concurrent small calls and excludes queue ti
expect(values.every((value) => value.q === 0.8)).toBe(true);
expect(peak).toBeGreaterThan(4);
expect(arrivals).toHaveLength(32);
- for (const start of arrivals) {
- // Leave a few milliseconds for loopback transport scheduling jitter.
- const tokens = arrivals
+ expect(starts).toHaveLength(32);
+ for (const start of starts) {
+ // Pace client starts; transport delay can compress server arrival windows.
+ const tokens = starts
.filter((entry) => entry.at >= start.at && entry.at < start.at + 990)
.reduce((sum, entry) => sum + entry.tokens, 0);
- expect(tokens).toBeLessThanOrEqual(250_000);
+ expect(tokens).toBeLessThanOrEqual(100_000);
+ }
+ } finally {
+ server.stop(true);
+ }
+}, 10_000);
+
+test("TypeSafe admits at most 40 small requests per second through real HTTP", async () => {
+ const arrivals: number[] = [];
+ const starts: number[] = [];
+ const server = Bun.serve({
+ port: 0,
+ fetch() {
+ arrivals.push(performance.now());
+ return Response.json({
+ answers: { q: { type: "noul", noul: 0.8 } },
+ usage: { input_tokens: 1, output_tokens: 1 },
+ });
+ },
+ });
+ try {
+ const evaluator = createEvaluator({
+ provider: "typesafe",
+ apiKey: "fixture",
+ signal: new AbortController().signal,
+ concurrency: 64,
+ fetch: routeProviderFetch((input, init) => {
+ starts.push(performance.now());
+ return fetch(input, init);
+ }, `http://127.0.0.1:${server.port}`),
+ });
+ const values = await Promise.all(
+ Array.from({ length: 80 }, () =>
+ evaluator.evaluate({
+ state: "small",
+ questions: { q: { type: "boolean", instructions: "Relevant?" } },
+ }),
+ ),
+ );
+ expect(values.every((value) => value.q === 0.8)).toBe(true);
+ expect(arrivals).toHaveLength(80);
+ expect(starts).toHaveLength(80);
+ expect(starts[39]! - starts[0]!).toBeLessThan(900);
+ for (const start of starts) {
+ // Unit tests own the exact boundary; allow SDK dispatch scheduling jitter.
+ expect(starts.filter((at) => at >= start && at < start + 900).length).toBeLessThanOrEqual(40);
}
} finally {
server.stop(true);
@@ -632,4 +684,4 @@ test("token-queued requests revalidate source after waiting", async () => {
await Promise.all(running);
expect(await queued).toMatchObject({ kind: "source-invalid" });
expect(calls).toBe(3);
-});
+}, 10_000);