Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Pace TypeSafe tokens while preserving request concurrency
David Zhang09346e16c43d
diff --git a/apps/cli/package.json b/apps/cli/package.jsonindex 6aeef6c..e67ac25 100644--- a/apps/cli/package.json+++ b/apps/cli/package.json@@ -1,6 +1,6 @@{"name": "@dzhng/jevgrep",- "version": "0.4.3",+ "version": "0.4.4","description": "Content-aware source retrieval for coding agents","license": "MIT","repository": {diff --git a/packages/core/src/evaluator.ts b/packages/core/src/evaluator.tsindex 1ef62fd..ab76975 100644--- a/packages/core/src/evaluator.ts+++ b/packages/core/src/evaluator.ts@@ -4,6 +4,7 @@ import { providers, type ProviderId } from "./providers";import { type createEvaluationCache, type CacheInput } from "./cache";import { setTimeout as delay } from "node:timers/promises";import { stripVTControlCharacters } from "node:util";+import { createRateBudget, estimatedInputTokens } from "./rate-budget";export type EvaluationRequest = {state: Parameters<typeof evaluate>[0]["state"];@@ -40,11 +41,16 @@ export function createEvaluator(options: {}) {const preset = providers[options.provider];const concurrency = options.concurrency ?? 32;+ const timeoutMs = options.timeoutMs ?? (options.provider === "typesafe" ? 60_000 : 15_000);if (!Number.isSafeInteger(concurrency) || concurrency < 1)throw new Error("Concurrency must be a positive integer");let requests = 0;let cacheHits = 0;let cooldownUntil = 0;+ const rateBudget =+ options.provider === "typesafe"+ ? createRateBudget({ tokensPerSecond: 250_000, requestsPerMinute: 1_200 })+ : undefined;const authenticationFailure = new AbortController();function assertActive() {if (options.signal.aborted) throw new EvaluationFailure("cancelled");@@ -137,6 +143,7 @@ export function createEvaluator(options: {return cached;}const navigation = policy?.navigation === true;+ const reservedTokens = rateBudget ? estimatedInputTokens(request) : 0;const multiple = Object.keys(request.questions).length > 1;let attemptLimit = navigation && multiple ? 1 : 2;for (let attempt = 0; attempt < attemptLimit; attempt++) {@@ -146,18 +153,29 @@ export function createEvaluator(options: {const release = await acquire();try {assertActive();- while (cooldownUntil > Date.now()) {- try {- await delay(Math.min(60_000, cooldownUntil - Date.now()), undefined, {- signal: AbortSignal.any([options.signal, authenticationFailure.signal]),- });- } catch {- assertActive();- throw new EvaluationFailure("cancelled");+ // Wait before validating source and before starting the network timeout.+ // Recheck after validation: another worker may have consumed the budget.+ for (;;) {+ const wait = Math.max(+ cooldownUntil - Date.now(),+ rateBudget?.waitMs(reservedTokens) ?? 0,+ );+ if (wait > 0) {+ try {+ await delay(Math.min(60_000, wait), undefined, { signal: stopped });+ } catch {+ assertActive();+ throw new EvaluationFailure("cancelled");+ }+ continue;}+ await policy?.beforeAttempt?.();+ assertActive();+ if (cooldownUntil > Date.now() || (rateBudget?.waitMs(reservedTokens) ?? 0) > 0)+ continue;+ break;}- await policy?.beforeAttempt?.();- assertActive();+ const reservation = rateBudget?.reserve(reservedTokens);try {const result = await evaluate({model: provider.evaluationModel(preset.model),@@ -166,9 +184,10 @@ export function createEvaluator(options: {abortSignal: AbortSignal.any([options.signal,authenticationFailure.signal,- AbortSignal.timeout(options.timeoutMs ?? 15_000),+ AbortSignal.timeout(timeoutMs),]),});+ reservation?.reconcile(result.usage.inputTokens);const scores = Object.fromEntries(Object.keys(request.questions).map((id) => {const answer = result.answers[id];@@ -216,7 +235,7 @@ export function createEvaluator(options: {const description = diagnostic? `HTTP ${diagnostic.statusCode}${diagnostic.message ? `: ${diagnostic.message}` : ""}`: name === "TimeoutError"- ? `Request timed out after ${options.timeoutMs ?? 15_000} ms`+ ? `Request timed out after ${timeoutMs} ms`: APICallError.isInstance(error) && error.statusCode === undefined? "Network request failed (connection unavailable or reset)": "Invalid or incomplete provider response";diff --git a/packages/core/src/rate-budget.ts b/packages/core/src/rate-budget.tsnew file mode 100644index 0000000..a66b6dd--- /dev/null+++ b/packages/core/src/rate-budget.ts@@ -0,0 +1,52 @@+/** Per-evaluator rolling budgets. Concurrency remains owned by the evaluator. */+export function createRateBudget(+ limits: { tokensPerSecond: number; requestsPerMinute: number },+ now = () => performance.now(),+) {+ const starts: Array<{ at: number; tokens: number }> = [];+ function waitMs(tokens: number) {+ if (!Number.isSafeInteger(tokens) || tokens < 0 || tokens > limits.tokensPerSecond)+ throw new Error("Token reservation exceeds the rate budget");+ const time = now();+ while (starts.length && starts[0]!.at <= time - 60_000) starts.shift();+ let wait =+ starts.length >= limits.requestsPerMinute+ ? starts[starts.length - limits.requestsPerMinute]!.at + 60_000 - time+ : 0;+ const recent = starts.filter((entry) => entry.at > time - 1000);+ let total = recent.reduce((sum, entry) => sum + entry.tokens, 0) + tokens;+ for (const entry of recent) {+ if (total <= limits.tokensPerSecond) break;+ total -= entry.tokens;+ wait = Math.max(wait, entry.at + 1000 - time);+ }+ return wait;+ }+ return {+ waitMs,+ reserve(tokens: number) {+ if (waitMs(tokens) > 0) throw new Error("Rate budget is not available");+ const entry = { at: now(), tokens };+ starts.push(entry);+ return {+ reconcile(inputTokens: number | undefined) {+ if (inputTokens !== undefined && Number.isSafeInteger(inputTokens) && inputTokens >= 0)+ entry.tokens = inputTokens;+ },+ };+ },+ };+}++/** Conservative byte reservation, bounded by Jev's maximum accepted input size.+ * This is not a tokenizer; returned usage replaces the estimate when available.+ */+export function estimatedInputTokens(request: {+ state: unknown;+ questions: Record<string, unknown>;+}) {+ return Math.min(+ 65_536,+ Buffer.byteLength(JSON.stringify(request)) + 512 + 32 * Object.keys(request.questions).length,+ );+}diff --git a/packages/core/test/rate-budget.test.ts b/packages/core/test/rate-budget.test.tsnew file mode 100644index 0000000..c7b08ad--- /dev/null+++ b/packages/core/test/rate-budget.test.ts@@ -0,0 +1,21 @@+import { expect, test } from "bun:test";+import { createRateBudget } from "../src/rate-budget";++test("rolling budgets reconcile usage and enforce the stricter token or request window", () => {+ let now = 0;+ const budget = createRateBudget({ tokensPerSecond: 100, requestsPerMinute: 3 }, () => now);+ const first = budget.reserve(80);+ expect(budget.waitMs(30)).toBe(1000);+ first.reconcile(undefined);+ expect(budget.waitMs(30)).toBe(1000);+ first.reconcile(20);+ expect(budget.waitMs(30)).toBe(0);+ budget.reserve(30);+ now = 500;+ budget.reserve(40);+ expect(budget.waitMs(20)).toBe(59_500);+ now = 60_000;+ expect(budget.waitMs(100)).toBe(0);+ budget.reserve(100);+ expect(budget.waitMs(1)).toBe(1000);+});diff --git a/specs/done/provider-support/README.md b/specs/done/provider-support/README.mdindex dc84c30..05aaf85 100644--- a/specs/done/provider-support/README.md+++ b/specs/done/provider-support/README.md@@ -25,6 +25,12 @@ This is a simpler setup contract, not a multi-account credential registry.- The application owns actual HTTP attempts, retries, shared rate-limit cooldownand cancellation. SDK retries remain disabled. The 50,000-attempt guard protectsagainst runaway work; it is not an optimization budget.+- TypeSafe throughput is paced by estimated input tokens and request starts,+ independently of the concurrent-request ceiling. Reservations use conservative+ payload bytes until reported usage is available. Queue waits happen before the+ network timeout and source is revalidated after waiting. These budgets belong+ to one search; other processes sharing the account can still cause rate limits,+ so provider cooldowns remain authoritative.- Source freshness is checked before attempts and cached evidence use. An invalidprovider answer is failure or incomplete discovery, never negative evidence.- Exact answer-cache identity separates provider, endpoint, model and protocol,diff --git a/test/evaluator.test.ts b/test/evaluator.test.tsindex b3cee26..25d830d 100644--- a/test/evaluator.test.ts+++ b/test/evaluator.test.ts@@ -434,3 +434,83 @@ for (const kind of ["cancelled", "authentication"] as const)server.stop(true);}});++test("native token pacing preserves concurrent small calls and excludes queue time from timeout", async () => {+ const arrivals: Array<{ at: number; tokens: number }> = [];+ let active = 0;+ let peak = 0;+ const server = Bun.serve({+ port: 0,+ async fetch(request) {+ const body = (await request.json()) as { state: string };+ const tokens = body.state.length > 1000 ? 60_000 : 100;+ arrivals.push({ at: performance.now(), tokens });+ peak = Math.max(peak, ++active);+ await new Promise((resolve) => setTimeout(resolve, 80));+ active--;+ return Response.json({+ answers: { q: { type: "noul", noul: 0.8 } },+ usage: { input_tokens: tokens, output_tokens: 1 },+ });+ },+ });+ try {+ const evaluator = createEvaluator({+ provider: "typesafe",+ apiKey: "fixture",+ signal: new AbortController().signal,+ timeoutMs: 500,+ fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),+ });+ const values = await Promise.all(+ Array.from({ length: 32 }, (_, i) =>+ evaluator.evaluate({+ state: i < 24 ? "small" : "x".repeat(65_000),+ questions: { q: { type: "boolean", instructions: "Relevant?" } },+ }),+ ),+ );+ expect(values.every((value) => value.q === 0.8)).toBe(true);+ expect(peak).toBeGreaterThan(4);+ expect(arrivals).toHaveLength(32);+ for (const start of arrivals) {+ // Leave a few milliseconds for loopback transport scheduling jitter.+ const tokens = arrivals+ .filter((entry) => entry.at >= start.at && entry.at < start.at + 990)+ .reduce((sum, entry) => sum + entry.tokens, 0);+ expect(tokens).toBeLessThanOrEqual(250_000);+ }+ } finally {+ server.stop(true);+ }+}, 10_000);++test("token-queued requests revalidate source after waiting", async () => {+ let calls = 0;+ let stale = false;+ const evaluator = createEvaluator({+ provider: "typesafe",+ apiKey: "fixture",+ signal: new AbortController().signal,+ fetch: async () => {+ calls++;+ stale = true;+ return Response.json({ answers: { q: { type: "noul", noul: 0.8 } } });+ },+ });+ const request = {+ state: "x".repeat(65_000),+ questions: { q: { type: "boolean" as const, instructions: "Relevant?" } },+ };+ const running = Array.from({ length: 3 }, () => evaluator.evaluate(request));+ const queued = evaluator+ .evaluate(request, {+ beforeAttempt: async () => {+ if (stale) throw new EvaluationFailure("source-invalid");+ },+ })+ .catch((error) => error);+ await Promise.all(running);+ expect(await queued).toMatchObject({ kind: "source-invalid" });+ expect(calls).toBe(3);+});