Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Pace TypeSafe tokens while preserving request concurrency

David Zhang09346e16c43d
diff --git a/apps/cli/package.json b/apps/cli/package.json
index 6aeef6c..e67ac25 100644
--- a/apps/cli/package.json
+++ b/apps/cli/package.json
@@ -1,6 +1,6 @@
{
"name": "@dzhng/jevgrep",
- "version": "0.4.3",
+ "version": "0.4.4",
"description": "Content-aware source retrieval for coding agents",
"license": "MIT",
"repository": {
diff --git a/packages/core/src/evaluator.ts b/packages/core/src/evaluator.ts
index 1ef62fd..ab76975 100644
--- a/packages/core/src/evaluator.ts
+++ b/packages/core/src/evaluator.ts
@@ -4,6 +4,7 @@ import { providers, type ProviderId } from "./providers";
import { type createEvaluationCache, type CacheInput } from "./cache";
import { setTimeout as delay } from "node:timers/promises";
import { stripVTControlCharacters } from "node:util";
+import { createRateBudget, estimatedInputTokens } from "./rate-budget";
export type EvaluationRequest = {
state: Parameters<typeof evaluate>[0]["state"];
@@ -40,11 +41,16 @@ export function createEvaluator(options: {
}) {
const preset = providers[options.provider];
const concurrency = options.concurrency ?? 32;
+ const timeoutMs = options.timeoutMs ?? (options.provider === "typesafe" ? 60_000 : 15_000);
if (!Number.isSafeInteger(concurrency) || concurrency < 1)
throw new Error("Concurrency must be a positive integer");
let requests = 0;
let cacheHits = 0;
let cooldownUntil = 0;
+ const rateBudget =
+ options.provider === "typesafe"
+ ? createRateBudget({ tokensPerSecond: 250_000, requestsPerMinute: 1_200 })
+ : undefined;
const authenticationFailure = new AbortController();
function assertActive() {
if (options.signal.aborted) throw new EvaluationFailure("cancelled");
@@ -137,6 +143,7 @@ export function createEvaluator(options: {
return cached;
}
const navigation = policy?.navigation === true;
+ const reservedTokens = rateBudget ? estimatedInputTokens(request) : 0;
const multiple = Object.keys(request.questions).length > 1;
let attemptLimit = navigation && multiple ? 1 : 2;
for (let attempt = 0; attempt < attemptLimit; attempt++) {
@@ -146,18 +153,29 @@ export function createEvaluator(options: {
const release = await acquire();
try {
assertActive();
- while (cooldownUntil > Date.now()) {
- try {
- await delay(Math.min(60_000, cooldownUntil - Date.now()), undefined, {
- signal: AbortSignal.any([options.signal, authenticationFailure.signal]),
- });
- } catch {
- assertActive();
- throw new EvaluationFailure("cancelled");
+ // Wait before validating source and before starting the network timeout.
+ // Recheck after validation: another worker may have consumed the budget.
+ for (;;) {
+ const wait = Math.max(
+ cooldownUntil - Date.now(),
+ rateBudget?.waitMs(reservedTokens) ?? 0,
+ );
+ if (wait > 0) {
+ try {
+ await delay(Math.min(60_000, wait), undefined, { signal: stopped });
+ } catch {
+ assertActive();
+ throw new EvaluationFailure("cancelled");
+ }
+ continue;
}
+ await policy?.beforeAttempt?.();
+ assertActive();
+ if (cooldownUntil > Date.now() || (rateBudget?.waitMs(reservedTokens) ?? 0) > 0)
+ continue;
+ break;
}
- await policy?.beforeAttempt?.();
- assertActive();
+ const reservation = rateBudget?.reserve(reservedTokens);
try {
const result = await evaluate({
model: provider.evaluationModel(preset.model),
@@ -166,9 +184,10 @@ export function createEvaluator(options: {
abortSignal: AbortSignal.any([
options.signal,
authenticationFailure.signal,
- AbortSignal.timeout(options.timeoutMs ?? 15_000),
+ AbortSignal.timeout(timeoutMs),
]),
});
+ reservation?.reconcile(result.usage.inputTokens);
const scores = Object.fromEntries(
Object.keys(request.questions).map((id) => {
const answer = result.answers[id];
@@ -216,7 +235,7 @@ export function createEvaluator(options: {
const description = diagnostic
? `HTTP ${diagnostic.statusCode}${diagnostic.message ? `: ${diagnostic.message}` : ""}`
: name === "TimeoutError"
- ? `Request timed out after ${options.timeoutMs ?? 15_000} ms`
+ ? `Request timed out after ${timeoutMs} ms`
: APICallError.isInstance(error) && error.statusCode === undefined
? "Network request failed (connection unavailable or reset)"
: "Invalid or incomplete provider response";
diff --git a/packages/core/src/rate-budget.ts b/packages/core/src/rate-budget.ts
new file mode 100644
index 0000000..a66b6dd
--- /dev/null
+++ b/packages/core/src/rate-budget.ts
@@ -0,0 +1,52 @@
+/** Per-evaluator rolling budgets. Concurrency remains owned by the evaluator. */
+export function createRateBudget(
+ limits: { tokensPerSecond: number; requestsPerMinute: number },
+ now = () => performance.now(),
+) {
+ const starts: Array<{ at: number; tokens: number }> = [];
+ function waitMs(tokens: number) {
+ if (!Number.isSafeInteger(tokens) || tokens < 0 || tokens > limits.tokensPerSecond)
+ throw new Error("Token reservation exceeds the rate budget");
+ const time = now();
+ while (starts.length && starts[0]!.at <= time - 60_000) starts.shift();
+ let wait =
+ starts.length >= limits.requestsPerMinute
+ ? starts[starts.length - limits.requestsPerMinute]!.at + 60_000 - time
+ : 0;
+ const recent = starts.filter((entry) => entry.at > time - 1000);
+ let total = recent.reduce((sum, entry) => sum + entry.tokens, 0) + tokens;
+ for (const entry of recent) {
+ if (total <= limits.tokensPerSecond) break;
+ total -= entry.tokens;
+ wait = Math.max(wait, entry.at + 1000 - time);
+ }
+ return wait;
+ }
+ return {
+ waitMs,
+ reserve(tokens: number) {
+ if (waitMs(tokens) > 0) throw new Error("Rate budget is not available");
+ const entry = { at: now(), tokens };
+ starts.push(entry);
+ return {
+ reconcile(inputTokens: number | undefined) {
+ if (inputTokens !== undefined && Number.isSafeInteger(inputTokens) && inputTokens >= 0)
+ entry.tokens = inputTokens;
+ },
+ };
+ },
+ };
+}
+
+/** Conservative byte reservation, bounded by Jev's maximum accepted input size.
+ * This is not a tokenizer; returned usage replaces the estimate when available.
+ */
+export function estimatedInputTokens(request: {
+ state: unknown;
+ questions: Record<string, unknown>;
+}) {
+ return Math.min(
+ 65_536,
+ Buffer.byteLength(JSON.stringify(request)) + 512 + 32 * Object.keys(request.questions).length,
+ );
+}
diff --git a/packages/core/test/rate-budget.test.ts b/packages/core/test/rate-budget.test.ts
new file mode 100644
index 0000000..c7b08ad
--- /dev/null
+++ b/packages/core/test/rate-budget.test.ts
@@ -0,0 +1,21 @@
+import { expect, test } from "bun:test";
+import { createRateBudget } from "../src/rate-budget";
+
+test("rolling budgets reconcile usage and enforce the stricter token or request window", () => {
+ let now = 0;
+ const budget = createRateBudget({ tokensPerSecond: 100, requestsPerMinute: 3 }, () => now);
+ const first = budget.reserve(80);
+ expect(budget.waitMs(30)).toBe(1000);
+ first.reconcile(undefined);
+ expect(budget.waitMs(30)).toBe(1000);
+ first.reconcile(20);
+ expect(budget.waitMs(30)).toBe(0);
+ budget.reserve(30);
+ now = 500;
+ budget.reserve(40);
+ expect(budget.waitMs(20)).toBe(59_500);
+ now = 60_000;
+ expect(budget.waitMs(100)).toBe(0);
+ budget.reserve(100);
+ expect(budget.waitMs(1)).toBe(1000);
+});
diff --git a/specs/done/provider-support/README.md b/specs/done/provider-support/README.md
index dc84c30..05aaf85 100644
--- a/specs/done/provider-support/README.md
+++ b/specs/done/provider-support/README.md
@@ -25,6 +25,12 @@ This is a simpler setup contract, not a multi-account credential registry.
- The application owns actual HTTP attempts, retries, shared rate-limit cooldown
and cancellation. SDK retries remain disabled. The 50,000-attempt guard protects
against runaway work; it is not an optimization budget.
+- TypeSafe throughput is paced by estimated input tokens and request starts,
+ independently of the concurrent-request ceiling. Reservations use conservative
+ payload bytes until reported usage is available. Queue waits happen before the
+ network timeout and source is revalidated after waiting. These budgets belong
+ to one search; other processes sharing the account can still cause rate limits,
+ so provider cooldowns remain authoritative.
- Source freshness is checked before attempts and cached evidence use. An invalid
provider answer is failure or incomplete discovery, never negative evidence.
- Exact answer-cache identity separates provider, endpoint, model and protocol,
diff --git a/test/evaluator.test.ts b/test/evaluator.test.ts
index b3cee26..25d830d 100644
--- a/test/evaluator.test.ts
+++ b/test/evaluator.test.ts
@@ -434,3 +434,83 @@ for (const kind of ["cancelled", "authentication"] as const)
server.stop(true);
}
});
+
+test("native token pacing preserves concurrent small calls and excludes queue time from timeout", async () => {
+ const arrivals: Array<{ at: number; tokens: number }> = [];
+ let active = 0;
+ let peak = 0;
+ const server = Bun.serve({
+ port: 0,
+ async fetch(request) {
+ const body = (await request.json()) as { state: string };
+ const tokens = body.state.length > 1000 ? 60_000 : 100;
+ arrivals.push({ at: performance.now(), tokens });
+ peak = Math.max(peak, ++active);
+ await new Promise((resolve) => setTimeout(resolve, 80));
+ active--;
+ return Response.json({
+ answers: { q: { type: "noul", noul: 0.8 } },
+ usage: { input_tokens: tokens, output_tokens: 1 },
+ });
+ },
+ });
+ try {
+ const evaluator = createEvaluator({
+ provider: "typesafe",
+ apiKey: "fixture",
+ signal: new AbortController().signal,
+ timeoutMs: 500,
+ fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
+ });
+ const values = await Promise.all(
+ Array.from({ length: 32 }, (_, i) =>
+ evaluator.evaluate({
+ state: i < 24 ? "small" : "x".repeat(65_000),
+ questions: { q: { type: "boolean", instructions: "Relevant?" } },
+ }),
+ ),
+ );
+ expect(values.every((value) => value.q === 0.8)).toBe(true);
+ expect(peak).toBeGreaterThan(4);
+ expect(arrivals).toHaveLength(32);
+ for (const start of arrivals) {
+ // Leave a few milliseconds for loopback transport scheduling jitter.
+ const tokens = arrivals
+ .filter((entry) => entry.at >= start.at && entry.at < start.at + 990)
+ .reduce((sum, entry) => sum + entry.tokens, 0);
+ expect(tokens).toBeLessThanOrEqual(250_000);
+ }
+ } finally {
+ server.stop(true);
+ }
+}, 10_000);
+
+test("token-queued requests revalidate source after waiting", async () => {
+ let calls = 0;
+ let stale = false;
+ const evaluator = createEvaluator({
+ provider: "typesafe",
+ apiKey: "fixture",
+ signal: new AbortController().signal,
+ fetch: async () => {
+ calls++;
+ stale = true;
+ return Response.json({ answers: { q: { type: "noul", noul: 0.8 } } });
+ },
+ });
+ const request = {
+ state: "x".repeat(65_000),
+ questions: { q: { type: "boolean" as const, instructions: "Relevant?" } },
+ };
+ const running = Array.from({ length: 3 }, () => evaluator.evaluate(request));
+ const queued = evaluator
+ .evaluate(request, {
+ beforeAttempt: async () => {
+ if (stale) throw new EvaluationFailure("source-invalid");
+ },
+ })
+ .catch((error) => error);
+ await Promise.all(running);
+ expect(await queued).toMatchObject({ kind: "source-invalid" });
+ expect(calls).toBe(3);
+});