Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
test: verify native Jev provider protocol and routing
David Zhang06daef973186
diff --git a/bun.lock b/bun.lockindex f7c9306..b6db4bc 100644--- a/bun.lock+++ b/bun.lock@@ -13,7 +13,7 @@},"apps/cli": {"name": "@dzhng/jevgrep",- "version": "0.1.0",+ "version": "0.2.0","bin": {"jg": "./dist/bin/index.js",},@@ -33,6 +33,7 @@"name": "@repo/core","version": "0.0.0","dependencies": {+ "@ai-sdk/typesafe-ai": "3.0.8","ai": "7.0.107","ignore": "7.0.5","pyodide": "0.25.1",@@ -55,9 +56,11 @@"packages": {"@ai-sdk/gateway": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.17", "@ai-sdk/provider-utils": "5.0.45", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-6wJmX/D5OO7tvDJbtFdXdBuRt8AkCnW+FLK+W35UHe/tHgolAa4hclXtJcMSYnlTDy/t5DNRgdesVaoMXtwzow=="],- "@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw=="],+ "@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-+GZJIgz1jk86pwEbb3f1BD2bdoSKyWE4Jg4YUc7NMnMozbWemSKYZcw2F4nMaO5qwsL5A8RmioAKW81YktRpIQ=="],- "@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.17", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ=="],+ "@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.18", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-T+/H8DCvqJoCqLhltVatS7ffA779Iyk/ZZrgNXkrgv51E0n2Cd6eIdCmtwhCGIKqtbTtwCvTmQdAMv5Z8rOGHQ=="],++ "@ai-sdk/typesafe-ai": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.18", "@ai-sdk/provider-utils": "5.0.49" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-nP5NJGCwZ7jDE2dImurTU9igbj1OUeFAXLvZKcM05KlBpLxhOddBe3sMftAhQq3Zq+B8NqLx1dcW4vQVD2l32w=="],"@clack/core": ["@clack/[email protected]", "", { "dependencies": { "fast-wrap-ansi": "^0.2.0", "sisteransi": "^1.0.5" } }, "sha512-iHTrHA8MtVuLl2TfZySmcKv1qO2PoyC9Z7pfSDozEuV5vtY3/wcOPKJXlqJ5Oq2Cx5DDGQGAMVx6HZfRRoVEbQ=="],@@ -207,6 +210,14 @@"zod": ["[email protected]", "", {}, "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q=="],+ "@ai-sdk/gateway/@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw=="],++ "@ai-sdk/gateway/@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.17", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ=="],++ "ai/@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw=="],++ "ai/@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.17", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ=="],+"bun-types/@types/node": ["@types/[email protected]", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-X1P21scMv4zGKLYqjdGjaKa7COa0RKVYYZZN/NfvLQ1JegxFhdhpZG/Lyn8AXx6CDUavKAd11v6BvfpkDByK8g=="],"bun-types/@types/node/undici-types": ["[email protected]", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="],diff --git a/packages/core/package.json b/packages/core/package.jsonindex dae5ae4..80d7f39 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -12,6 +12,7 @@"lint": "oxlint src"},"dependencies": {+ "@ai-sdk/typesafe-ai": "3.0.8","ai": "7.0.107","ignore": "7.0.5","pyodide": "0.25.1",diff --git a/packages/core/test-node/provider-protocol.test.mjs b/packages/core/test-node/provider-protocol.test.mjsnew file mode 100644index 0000000..d1cfc30--- /dev/null+++ b/packages/core/test-node/provider-protocol.test.mjs@@ -0,0 +1,226 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { createServer } from "node:http";+import { experimental_evaluate as evaluate } from "ai";+import { createTypeSafeAi } from "@ai-sdk/typesafe-ai";+import { routeProviderFetch } from "../../../test/fixtures/provider-route.mjs";++const state = {+ query: "event recording",+ declarations: [{ id: "u0", source: "function record() {}" }],+};+const questions = {+ q1: { type: "boolean", instructions: "Does u0 record events?" },+ q0: { type: "boolean", instructions: "Does u0 delete events?" },+};+const presets = [+ ["https://ai-gateway.vercel.sh/typesafe/v1", "typesafe-ai/jev"],+ ["https://api.typesafe.ai/v1", "jev-1.13.0"],+ ["https://openrouter.ai/api/v1", "jev-1.13"],+];+async function fixture(t, handler) {+ let calls = 0;+ const server = createServer((req, res) => {+ calls++;+ void handler(req, res);+ });+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));+ t.after(async () => {+ server.closeAllConnections();+ await new Promise((resolve) => server.close(resolve));+ });+ return { origin: `http://127.0.0.1:${server.address().port}`, calls: () => calls };+}+function model(baseURL, modelId, origin) {+ return createTypeSafeAi({+ apiKey: "synthetic-key",+ baseURL,+ fetch: routeProviderFetch(fetch, origin),+ }).evaluationModel(modelId);+}+for (const [baseURL, modelId] of presets) {+ test(`structured state and ordered questions survive ${baseURL}`, async (t) => {+ let observed;+ const http = await fixture(t, async (req, res) => {+ const chunks = [];+ for await (const chunk of req) chunks.push(chunk);+ observed = { path: req.url, headers: req.headers, body: JSON.parse(Buffer.concat(chunks)) };+ res.setHeader("content-type", "application/json");+ res.end(+ JSON.stringify({+ model: modelId,+ answers: { q1: { type: "noul", noul: 0.83 }, q0: { type: "noul", noul: 0.12 } },+ usage: { input_tokens: 12, output_tokens: 2 },+ }),+ );+ });+ const result = await evaluate({+ model: model(baseURL, modelId, http.origin),+ state,+ questions,+ maxRetries: 0,+ });+ assert.deepEqual(observed.body, {+ model: modelId,+ state,+ questions: {+ q1: { type: "noul", instructions: questions.q1.instructions },+ q0: { type: "noul", instructions: questions.q0.instructions },+ },+ });+ assert.deepEqual(Object.keys(observed.body.questions), ["q1", "q0"]);+ assert.equal(observed.path, `${new URL(baseURL).pathname}/systemone`);+ assert.equal(observed.headers.authorization, "Bearer synthetic-key");+ assert.equal(observed.headers["x-jevgrep-original-url"], `${baseURL}/systemone`);+ assert.match(observed.headers["content-type"], /^application\/json/);+ assert.deepEqual(result.answers, {+ q1: { type: "boolean", probability: 0.83 },+ q0: { type: "boolean", probability: 0.12 },+ });+ assert.equal(http.calls(), 1);+ });+}++for (const status of [401, 403, 408, 409, 429, 500, 503]) {+ test(`native status ${status} remains observable without SDK retry`, async (t) => {+ const http = await fixture(t, (_req, res) => {+ res.writeHead(status, { "content-type": "application/json", "retry-after": "2" });+ res.end(JSON.stringify({ message: "synthetic rejection", error_type: "fixture" }));+ });+ await assert.rejects(+ evaluate({ model: model(...presets[0], http.origin), state, questions, maxRetries: 0 }),+ (error) => {+ assert.equal(error.name, "AI_APICallError");+ assert.equal(error.statusCode, status);+ assert.equal(error.responseHeaders["retry-after"], "2");+ return true;+ },+ );+ assert.equal(http.calls(), 1);+ });+}+for (const [label, body] of [+ ["malformed JSON", "{broken"],+ ["missing answer", JSON.stringify({ answers: {} })],+ [+ "out of range",+ JSON.stringify({+ answers: { q1: { type: "noul", noul: 2 }, q0: { type: "noul", noul: 0.12 } },+ }),+ ],+]) {+ test(`${label} is rejected rather than clamped`, async (t) => {+ const http = await fixture(t, (_req, res) => {+ res.setHeader("content-type", "application/json");+ res.end(body);+ });+ await assert.rejects(+ evaluate({ model: model(...presets[0], http.origin), state, questions, maxRetries: 0 }),+ (error) => {+ assert.equal(+ error.name,+ label === "malformed JSON" ? "AI_APICallError" : "AI_InvalidResponseDataError",+ );+ if (label === "malformed JSON") {+ assert.equal(error.statusCode, 200);+ assert.equal(error.isRetryable, false);+ }+ t.diagnostic(`${label}: ${error.name}`);+ return true;+ },+ );+ assert.equal(http.calls(), 1);+ });+}+test("disconnect remains a status-less transport failure", async (t) => {+ const http = await fixture(t, (_req, res) => res.destroy());+ await assert.rejects(+ evaluate({ model: model(...presets[0], http.origin), state, questions, maxRetries: 0 }),+ (error) => {+ assert.equal(error.name, "AI_APICallError");+ assert.equal(error.statusCode, undefined);+ assert.equal(error.cause?.code, "UND_ERR_SOCKET");+ assert.equal(error.isRetryable, true);+ return true;+ },+ );+ assert.equal(http.calls(), 1);+});+for (const kind of ["timeout", "cancel"]) {+ test(`${kind} cancels actual HTTP without hidden retries`, { timeout: 5000 }, async (t) => {+ let received;+ const started = new Promise((resolve) => {+ received = resolve;+ });+ const http = await fixture(t, () => received());+ const controller = new AbortController();+ const result = evaluate({+ model: model(...presets[0], http.origin),+ state,+ questions,+ maxRetries: 0,+ abortSignal: controller.signal,+ });+ const rejected = assert.rejects(result, (error) => {+ assert.equal(error.name, kind === "timeout" ? "TimeoutError" : "AbortError");+ return true;+ });+ await started;+ if (kind === "cancel") controller.abort();+ else {+ const deadline = AbortSignal.timeout(30);+ deadline.addEventListener("abort", () => controller.abort(deadline.reason), { once: true });+ }+ await rejected;+ assert.equal(http.calls(), 1);+ });+}++test("routing rejects unknown destinations before HTTP", async (t) => {+ const http = await fixture(t, (_req, res) => res.end("{}"));+ const routed = routeProviderFetch(fetch, http.origin);+ await assert.rejects(+ routed("https://unapproved.invalid/v1/systemone"),+ /Unexpected provider destination/,+ );+ assert.equal(http.calls(), 0);+});++test("Node preload routes SDK-shaped fetches and rejects any other host", async (t) => {+ const { execFile } = await import("node:child_process");+ const { promisify } = await import("node:util");+ const { fileURLToPath } = await import("node:url");+ const run = promisify(execFile);+ const bodies = [];+ const http = await fixture(t, async (req, res) => {+ const chunks = [];+ for await (const chunk of req) chunks.push(chunk);+ bodies.push(Buffer.concat(chunks).toString());+ res.end("fixture-ok");+ });+ const preload = fileURLToPath(+ new URL("../../../test/fixtures/provider-route.mjs", import.meta.url),+ );+ const options = { env: { JEVGREP_TEST_PROVIDER_ORIGIN: http.origin }, timeout: 5000 };+ const args = ["--import", preload, "--input-type=module", "-e"];+ const result = await run(+ process.execPath,+ [+ ...args,+ `console.log(await (await fetch('https://api.typesafe.ai/v1/systemone', {method: 'POST', body: 'synthetic'})).text())`,+ ],+ options,+ );+ assert.equal(result.stdout, "fixture-ok\n");+ assert.equal(result.stderr, "");+ assert.deepEqual(bodies, ["synthetic"]);+ await assert.rejects(+ run(+ process.execPath,+ [...args, `await fetch('https://unapproved.invalid/v1/systemone')`],+ options,+ ),+ /Unexpected provider destination/,+ );+ assert.equal(http.calls(), 1);+});diff --git a/scripts/test-suite.sh b/scripts/test-suite.shindex 890bcbd..92ad1ec 100644--- a/scripts/test-suite.sh+++ b/scripts/test-suite.sh@@ -1,6 +1,7 @@#!/usr/bin/env bashset -euo pipefailbun test apps/cli/test packages/core/test test/reference test/gateway.test.ts test/retrieval.test.ts test/retrieval-freshness.test.ts+node --test packages/core/test-node/*.test.mjsnode --experimental-strip-types --test test/parser/*.test.tsnode --test test/release.test.mjspython3 -B -m unittest discover -s evals/implementation/swebench -p test_installed.pydiff --git a/specs/provider-support/README.md b/specs/provider-support/README.mdindex 1a69c2a..8db4b60 100644--- a/specs/provider-support/README.md+++ b/specs/provider-support/README.md@@ -6,12 +6,11 @@ ordinary searches keep the same interface and retrieval behavior.## Next Agent Prompt-You are implementing this plan, last updated 2026-09-26. **Status: specification-only; no provider implementation or live verification has happened.** Start at-[slice 1](slices/01-protocol.md): reproduce the pinned adapter against controlled-HTTP before changing production. Read [the evidence](research.md) and-[the preservation gates](parity.md). The completed [decision map](map.md) supplies-user rationale; this README and its slices now own implementation order.+You are implementing this plan, last updated 2026-09-26. Slice 1 is complete:+18 Node protocol tests pass locally and in Docker, and all 5 frozen replay tests+remain green. Start [slice 2](slices/02-evaluator.md): replace the evaluator while+preserving production parity. Read [protocol evidence](assets/protocol-verification.md)+and [preservation gates](parity.md). No live provider calls have occurred.You have no unresolved product question. Published-package behavior and liveprovider access need verification, not invented compatibility code. Missing@@ -19,7 +18,7 @@ credentials must not block offline implementation; report the missing live checkhonestly. Do not collect secrets in chat or overwrite the user's own saved setupfor tests. Use isolated HOME/config/cache locations.-- [ ] [1. Protocol reproduction and controlled routing](slices/01-protocol.md)+- [x] [1. Protocol reproduction and controlled routing](slices/01-protocol.md)- [ ] [2. Shared evaluator and production parity](slices/02-evaluator.md)- [ ] [3. Saved-provider auth, installed journeys, and onboarding](slices/03-auth.md)- [ ] [4. Maintained eval routing and final verification](slices/04-tooling.md)diff --git a/specs/provider-support/assets/protocol-verification.md b/specs/provider-support/assets/protocol-verification.mdnew file mode 100644index 0000000..a143323--- /dev/null+++ b/specs/provider-support/assets/protocol-verification.md@@ -0,0 +1,48 @@+# Adapter protocol reproduction++2026-09-26, slice 1 complete. Protocol suite: 18/18 passing on local Node and+Node 22 in network-isolated Docker. Frozen reference replay: 5/5 passing with+all recorded digests, exact requests and complete stdout unchanged. Type checks+pass. Independent Codex review found a possible timeout-test hang; the deadline+now starts after request arrival and the test has a finite outer timeout. The+corrected suite passed in both environments. Shape, diff and documentation review+found no remaining issue.++The published `@ai-sdk/[email protected]` implementation matches the request and+answer conversion documented in the research record. `[email protected]` remains fixed;+its provider 4.0.17/provider-utils 5.0.45 dependencies remain separately resolved.+The new adapter uses provider 4.0.18/provider-utils 5.0.49. Their differing versions+do not require upgrading the historical oracle.++Observed through Node HTTP, SDK retries disabled:++| Scenario | Observation |+| --------------------------------------------------- | ---------------------------------------------------------------------------------- |+| Each preset, nested state and two ordered questions | One POST, exact state/instructions, `noul` wire types and unchanged probabilities |+| 401/403/408/409/429/500/503 | `AI_APICallError`, original HTTP status and response headers retained; one attempt |+| Malformed JSON | `AI_APICallError`, status 200, non-retryable |+| Missing or out-of-range answer | `AI_InvalidResponseDataError`; rejected, not clamped |+| Socket disconnect | `AI_APICallError`, no HTTP status, `UND_ERR_SOCKET` cause, retryable |+| Deadline / external abort | `TimeoutError` / `AbortError`, no additional attempt |+| Unknown destination | Harness rejects before HTTP |+| Real Node preload | SDK-shaped fetch redirected with original body; unexpected host rejected |++The test-only redirection preserves SDK URL/init/body rather than rebuilding its+body as a Request stream. A streamed-body prototype failed the 401 probe with+Node's `expected non-null body source` error, obscuring authentication failure;+that prototype was removed. No product transport workaround is needed.++The adapter archive includes its license. Its provider-utils 5.0.49 dependency+omits a license, like 5.0.45. The upstream repository LICENSE at tag revision+`ee3169b3c4880e2abe4d0d7c781243bb81822ec4` is byte-identical to the existing retained+5.0.45 license. Slice 2 must extend the exact-version notice mapping before the+adapter becomes part of the product bundle.++Reproduce with `node --test packages/core/test-node/provider-protocol.test.mjs`;+the same command runs inside `scripts/test-reference.sh`, and the normal suite+now includes it. No credentials or live model calls were used.++Docker dependency downloads required explicit build proxy arguments using the+existing Docker Desktop proxy (`http.docker.internal:3128`); direct registry+connections timed out. The completed image was then exercised by the unmodified+network-isolated test script. No product configuration was changed.diff --git a/specs/provider-support/choices.md b/specs/provider-support/choices.mdnew file mode 100644index 0000000..4e8969e--- /dev/null+++ b/specs/provider-support/choices.md@@ -0,0 +1,10 @@+# Provider support choices++No unplanned product choice has been made in slice 1. The SDK pin, endpoint/model+presets, external routing seam and immutable reference were specified. Probe file+names and test layout are delegated implementation discretion.++The routing fixture preserves the SDK's original body instead of converting it+to a stream. This is a correctness fix within the specified transport contract,+not an additional product behavior. Further passes append only decisions not+already made or delegated by the plan.diff --git a/specs/provider-support/slices/01-protocol.md b/specs/provider-support/slices/01-protocol.mdindex 329c11a..29c6eab 100644--- a/specs/provider-support/slices/01-protocol.md+++ b/specs/provider-support/slices/01-protocol.md@@ -1,7 +1,7 @@# 1. Reproduce the provider protocol**Unlock:** prove the chosen SDK and controlled routing before touching retrieval.-No dependency on another slice. Status: not started.+No dependency on another slice. Status: complete; see [verification](../assets/protocol-verification.md).## Seam and artifactdiff --git a/test/fixtures/provider-route.mjs b/test/fixtures/provider-route.mjsnew file mode 100644index 0000000..e8a3e8b--- /dev/null+++ b/test/fixtures/provider-route.mjs@@ -0,0 +1,18 @@+// Harness-only transport: never shipped or read by the CLI itself.+const endpoints = new Set([+ "https://ai-gateway.vercel.sh/typesafe/v1/systemone",+ "https://api.typesafe.ai/v1/systemone",+ "https://openrouter.ai/api/v1/systemone",+]);+export function routeProviderFetch(originalFetch, origin) {+ return async (input, init) => {+ const url = new URL(input);+ if (!endpoints.has(url.href)) throw new Error(`Unexpected provider destination: ${url.href}`);+ const headers = new Headers(init?.headers);+ headers.set("x-jevgrep-original-url", url.href);+ return originalFetch(new URL(url.pathname, origin), { ...init, headers });+ };+}+if (process.env.JEVGREP_TEST_PROVIDER_ORIGIN) {+ globalThis.fetch = routeProviderFetch(globalThis.fetch, process.env.JEVGREP_TEST_PROVIDER_ORIGIN);+}