Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

test: verify native Jev provider protocol and routing

David Zhang06daef973186
diff --git a/bun.lock b/bun.lock
index f7c9306..b6db4bc 100644
--- a/bun.lock
+++ b/bun.lock
@@ -13,7 +13,7 @@
},
"apps/cli": {
"name": "@dzhng/jevgrep",
- "version": "0.1.0",
+ "version": "0.2.0",
"bin": {
"jg": "./dist/bin/index.js",
},
@@ -33,6 +33,7 @@
"name": "@repo/core",
"version": "0.0.0",
"dependencies": {
+ "@ai-sdk/typesafe-ai": "3.0.8",
"ai": "7.0.107",
"ignore": "7.0.5",
"pyodide": "0.25.1",
@@ -55,9 +56,11 @@
"packages": {
"@ai-sdk/gateway": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.17", "@ai-sdk/provider-utils": "5.0.45", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-6wJmX/D5OO7tvDJbtFdXdBuRt8AkCnW+FLK+W35UHe/tHgolAa4hclXtJcMSYnlTDy/t5DNRgdesVaoMXtwzow=="],
- "@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw=="],
+ "@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-+GZJIgz1jk86pwEbb3f1BD2bdoSKyWE4Jg4YUc7NMnMozbWemSKYZcw2F4nMaO5qwsL5A8RmioAKW81YktRpIQ=="],
- "@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.17", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ=="],
+ "@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.18", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-T+/H8DCvqJoCqLhltVatS7ffA779Iyk/ZZrgNXkrgv51E0n2Cd6eIdCmtwhCGIKqtbTtwCvTmQdAMv5Z8rOGHQ=="],
+
+ "@ai-sdk/typesafe-ai": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.18", "@ai-sdk/provider-utils": "5.0.49" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-nP5NJGCwZ7jDE2dImurTU9igbj1OUeFAXLvZKcM05KlBpLxhOddBe3sMftAhQq3Zq+B8NqLx1dcW4vQVD2l32w=="],
"@clack/core": ["@clack/[email protected]", "", { "dependencies": { "fast-wrap-ansi": "^0.2.0", "sisteransi": "^1.0.5" } }, "sha512-iHTrHA8MtVuLl2TfZySmcKv1qO2PoyC9Z7pfSDozEuV5vtY3/wcOPKJXlqJ5Oq2Cx5DDGQGAMVx6HZfRRoVEbQ=="],
@@ -207,6 +210,14 @@
"zod": ["[email protected]", "", {}, "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q=="],
+ "@ai-sdk/gateway/@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw=="],
+
+ "@ai-sdk/gateway/@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.17", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ=="],
+
+ "ai/@ai-sdk/provider": ["@ai-sdk/[email protected]", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-VYMBxIQdcHqbIf1j+YZlI9Ati6LZ4wJe0GGd4z4a5H/KxTggjeOiyaVYTnfF7LHZK5jMQ+rofmzz4QPqf++NUw=="],
+
+ "ai/@ai-sdk/provider-utils": ["@ai-sdk/[email protected]", "", { "dependencies": { "@ai-sdk/provider": "4.0.17", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-gLuaCups8OCIRz3eO6WWz5op7VLYNNtFzyH0sWUr0FE9v7cTxR0qxDpQdrKnGcql+9WWttobjnZsJVsuEbsPQQ=="],
+
"bun-types/@types/node": ["@types/[email protected]", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-X1P21scMv4zGKLYqjdGjaKa7COa0RKVYYZZN/NfvLQ1JegxFhdhpZG/Lyn8AXx6CDUavKAd11v6BvfpkDByK8g=="],
"bun-types/@types/node/undici-types": ["[email protected]", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="],
diff --git a/packages/core/package.json b/packages/core/package.json
index dae5ae4..80d7f39 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -12,6 +12,7 @@
"lint": "oxlint src"
},
"dependencies": {
+ "@ai-sdk/typesafe-ai": "3.0.8",
"ai": "7.0.107",
"ignore": "7.0.5",
"pyodide": "0.25.1",
diff --git a/packages/core/test-node/provider-protocol.test.mjs b/packages/core/test-node/provider-protocol.test.mjs
new file mode 100644
index 0000000..d1cfc30
--- /dev/null
+++ b/packages/core/test-node/provider-protocol.test.mjs
@@ -0,0 +1,226 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { experimental_evaluate as evaluate } from "ai";
+import { createTypeSafeAi } from "@ai-sdk/typesafe-ai";
+import { routeProviderFetch } from "../../../test/fixtures/provider-route.mjs";
+
+const state = {
+ query: "event recording",
+ declarations: [{ id: "u0", source: "function record() {}" }],
+};
+const questions = {
+ q1: { type: "boolean", instructions: "Does u0 record events?" },
+ q0: { type: "boolean", instructions: "Does u0 delete events?" },
+};
+const presets = [
+ ["https://ai-gateway.vercel.sh/typesafe/v1", "typesafe-ai/jev"],
+ ["https://api.typesafe.ai/v1", "jev-1.13.0"],
+ ["https://openrouter.ai/api/v1", "jev-1.13"],
+];
+async function fixture(t, handler) {
+ let calls = 0;
+ const server = createServer((req, res) => {
+ calls++;
+ void handler(req, res);
+ });
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+ t.after(async () => {
+ server.closeAllConnections();
+ await new Promise((resolve) => server.close(resolve));
+ });
+ return { origin: `http://127.0.0.1:${server.address().port}`, calls: () => calls };
+}
+function model(baseURL, modelId, origin) {
+ return createTypeSafeAi({
+ apiKey: "synthetic-key",
+ baseURL,
+ fetch: routeProviderFetch(fetch, origin),
+ }).evaluationModel(modelId);
+}
+for (const [baseURL, modelId] of presets) {
+ test(`structured state and ordered questions survive ${baseURL}`, async (t) => {
+ let observed;
+ const http = await fixture(t, async (req, res) => {
+ const chunks = [];
+ for await (const chunk of req) chunks.push(chunk);
+ observed = { path: req.url, headers: req.headers, body: JSON.parse(Buffer.concat(chunks)) };
+ res.setHeader("content-type", "application/json");
+ res.end(
+ JSON.stringify({
+ model: modelId,
+ answers: { q1: { type: "noul", noul: 0.83 }, q0: { type: "noul", noul: 0.12 } },
+ usage: { input_tokens: 12, output_tokens: 2 },
+ }),
+ );
+ });
+ const result = await evaluate({
+ model: model(baseURL, modelId, http.origin),
+ state,
+ questions,
+ maxRetries: 0,
+ });
+ assert.deepEqual(observed.body, {
+ model: modelId,
+ state,
+ questions: {
+ q1: { type: "noul", instructions: questions.q1.instructions },
+ q0: { type: "noul", instructions: questions.q0.instructions },
+ },
+ });
+ assert.deepEqual(Object.keys(observed.body.questions), ["q1", "q0"]);
+ assert.equal(observed.path, `${new URL(baseURL).pathname}/systemone`);
+ assert.equal(observed.headers.authorization, "Bearer synthetic-key");
+ assert.equal(observed.headers["x-jevgrep-original-url"], `${baseURL}/systemone`);
+ assert.match(observed.headers["content-type"], /^application\/json/);
+ assert.deepEqual(result.answers, {
+ q1: { type: "boolean", probability: 0.83 },
+ q0: { type: "boolean", probability: 0.12 },
+ });
+ assert.equal(http.calls(), 1);
+ });
+}
+
+for (const status of [401, 403, 408, 409, 429, 500, 503]) {
+ test(`native status ${status} remains observable without SDK retry`, async (t) => {
+ const http = await fixture(t, (_req, res) => {
+ res.writeHead(status, { "content-type": "application/json", "retry-after": "2" });
+ res.end(JSON.stringify({ message: "synthetic rejection", error_type: "fixture" }));
+ });
+ await assert.rejects(
+ evaluate({ model: model(...presets[0], http.origin), state, questions, maxRetries: 0 }),
+ (error) => {
+ assert.equal(error.name, "AI_APICallError");
+ assert.equal(error.statusCode, status);
+ assert.equal(error.responseHeaders["retry-after"], "2");
+ return true;
+ },
+ );
+ assert.equal(http.calls(), 1);
+ });
+}
+for (const [label, body] of [
+ ["malformed JSON", "{broken"],
+ ["missing answer", JSON.stringify({ answers: {} })],
+ [
+ "out of range",
+ JSON.stringify({
+ answers: { q1: { type: "noul", noul: 2 }, q0: { type: "noul", noul: 0.12 } },
+ }),
+ ],
+]) {
+ test(`${label} is rejected rather than clamped`, async (t) => {
+ const http = await fixture(t, (_req, res) => {
+ res.setHeader("content-type", "application/json");
+ res.end(body);
+ });
+ await assert.rejects(
+ evaluate({ model: model(...presets[0], http.origin), state, questions, maxRetries: 0 }),
+ (error) => {
+ assert.equal(
+ error.name,
+ label === "malformed JSON" ? "AI_APICallError" : "AI_InvalidResponseDataError",
+ );
+ if (label === "malformed JSON") {
+ assert.equal(error.statusCode, 200);
+ assert.equal(error.isRetryable, false);
+ }
+ t.diagnostic(`${label}: ${error.name}`);
+ return true;
+ },
+ );
+ assert.equal(http.calls(), 1);
+ });
+}
+test("disconnect remains a status-less transport failure", async (t) => {
+ const http = await fixture(t, (_req, res) => res.destroy());
+ await assert.rejects(
+ evaluate({ model: model(...presets[0], http.origin), state, questions, maxRetries: 0 }),
+ (error) => {
+ assert.equal(error.name, "AI_APICallError");
+ assert.equal(error.statusCode, undefined);
+ assert.equal(error.cause?.code, "UND_ERR_SOCKET");
+ assert.equal(error.isRetryable, true);
+ return true;
+ },
+ );
+ assert.equal(http.calls(), 1);
+});
+for (const kind of ["timeout", "cancel"]) {
+ test(`${kind} cancels actual HTTP without hidden retries`, { timeout: 5000 }, async (t) => {
+ let received;
+ const started = new Promise((resolve) => {
+ received = resolve;
+ });
+ const http = await fixture(t, () => received());
+ const controller = new AbortController();
+ const result = evaluate({
+ model: model(...presets[0], http.origin),
+ state,
+ questions,
+ maxRetries: 0,
+ abortSignal: controller.signal,
+ });
+ const rejected = assert.rejects(result, (error) => {
+ assert.equal(error.name, kind === "timeout" ? "TimeoutError" : "AbortError");
+ return true;
+ });
+ await started;
+ if (kind === "cancel") controller.abort();
+ else {
+ const deadline = AbortSignal.timeout(30);
+ deadline.addEventListener("abort", () => controller.abort(deadline.reason), { once: true });
+ }
+ await rejected;
+ assert.equal(http.calls(), 1);
+ });
+}
+
+test("routing rejects unknown destinations before HTTP", async (t) => {
+ const http = await fixture(t, (_req, res) => res.end("{}"));
+ const routed = routeProviderFetch(fetch, http.origin);
+ await assert.rejects(
+ routed("https://unapproved.invalid/v1/systemone"),
+ /Unexpected provider destination/,
+ );
+ assert.equal(http.calls(), 0);
+});
+
+test("Node preload routes SDK-shaped fetches and rejects any other host", async (t) => {
+ const { execFile } = await import("node:child_process");
+ const { promisify } = await import("node:util");
+ const { fileURLToPath } = await import("node:url");
+ const run = promisify(execFile);
+ const bodies = [];
+ const http = await fixture(t, async (req, res) => {
+ const chunks = [];
+ for await (const chunk of req) chunks.push(chunk);
+ bodies.push(Buffer.concat(chunks).toString());
+ res.end("fixture-ok");
+ });
+ const preload = fileURLToPath(
+ new URL("../../../test/fixtures/provider-route.mjs", import.meta.url),
+ );
+ const options = { env: { JEVGREP_TEST_PROVIDER_ORIGIN: http.origin }, timeout: 5000 };
+ const args = ["--import", preload, "--input-type=module", "-e"];
+ const result = await run(
+ process.execPath,
+ [
+ ...args,
+ `console.log(await (await fetch('https://api.typesafe.ai/v1/systemone', {method: 'POST', body: 'synthetic'})).text())`,
+ ],
+ options,
+ );
+ assert.equal(result.stdout, "fixture-ok\n");
+ assert.equal(result.stderr, "");
+ assert.deepEqual(bodies, ["synthetic"]);
+ await assert.rejects(
+ run(
+ process.execPath,
+ [...args, `await fetch('https://unapproved.invalid/v1/systemone')`],
+ options,
+ ),
+ /Unexpected provider destination/,
+ );
+ assert.equal(http.calls(), 1);
+});
diff --git a/scripts/test-suite.sh b/scripts/test-suite.sh
index 890bcbd..92ad1ec 100644
--- a/scripts/test-suite.sh
+++ b/scripts/test-suite.sh
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
bun test apps/cli/test packages/core/test test/reference test/gateway.test.ts test/retrieval.test.ts test/retrieval-freshness.test.ts
+node --test packages/core/test-node/*.test.mjs
node --experimental-strip-types --test test/parser/*.test.ts
node --test test/release.test.mjs
python3 -B -m unittest discover -s evals/implementation/swebench -p test_installed.py
diff --git a/specs/provider-support/README.md b/specs/provider-support/README.md
index 1a69c2a..8db4b60 100644
--- a/specs/provider-support/README.md
+++ b/specs/provider-support/README.md
@@ -6,12 +6,11 @@ ordinary searches keep the same interface and retrieval behavior.
## Next Agent Prompt
-You are implementing this plan, last updated 2026-09-26. **Status: specification
-only; no provider implementation or live verification has happened.** Start at
-[slice 1](slices/01-protocol.md): reproduce the pinned adapter against controlled
-HTTP before changing production. Read [the evidence](research.md) and
-[the preservation gates](parity.md). The completed [decision map](map.md) supplies
-user rationale; this README and its slices now own implementation order.
+You are implementing this plan, last updated 2026-09-26. Slice 1 is complete:
+18 Node protocol tests pass locally and in Docker, and all 5 frozen replay tests
+remain green. Start [slice 2](slices/02-evaluator.md): replace the evaluator while
+preserving production parity. Read [protocol evidence](assets/protocol-verification.md)
+and [preservation gates](parity.md). No live provider calls have occurred.
You have no unresolved product question. Published-package behavior and live
provider access need verification, not invented compatibility code. Missing
@@ -19,7 +18,7 @@ credentials must not block offline implementation; report the missing live check
honestly. Do not collect secrets in chat or overwrite the user's own saved setup
for tests. Use isolated HOME/config/cache locations.
-- [ ] [1. Protocol reproduction and controlled routing](slices/01-protocol.md)
+- [x] [1. Protocol reproduction and controlled routing](slices/01-protocol.md)
- [ ] [2. Shared evaluator and production parity](slices/02-evaluator.md)
- [ ] [3. Saved-provider auth, installed journeys, and onboarding](slices/03-auth.md)
- [ ] [4. Maintained eval routing and final verification](slices/04-tooling.md)
diff --git a/specs/provider-support/assets/protocol-verification.md b/specs/provider-support/assets/protocol-verification.md
new file mode 100644
index 0000000..a143323
--- /dev/null
+++ b/specs/provider-support/assets/protocol-verification.md
@@ -0,0 +1,48 @@
+# Adapter protocol reproduction
+
+2026-09-26, slice 1 complete. Protocol suite: 18/18 passing on local Node and
+Node 22 in network-isolated Docker. Frozen reference replay: 5/5 passing with
+all recorded digests, exact requests and complete stdout unchanged. Type checks
+pass. Independent Codex review found a possible timeout-test hang; the deadline
+now starts after request arrival and the test has a finite outer timeout. The
+corrected suite passed in both environments. Shape, diff and documentation review
+found no remaining issue.
+
+The published `@ai-sdk/[email protected]` implementation matches the request and
+answer conversion documented in the research record. `[email protected]` remains fixed;
+its provider 4.0.17/provider-utils 5.0.45 dependencies remain separately resolved.
+The new adapter uses provider 4.0.18/provider-utils 5.0.49. Their differing versions
+do not require upgrading the historical oracle.
+
+Observed through Node HTTP, SDK retries disabled:
+
+| Scenario | Observation |
+| --------------------------------------------------- | ---------------------------------------------------------------------------------- |
+| Each preset, nested state and two ordered questions | One POST, exact state/instructions, `noul` wire types and unchanged probabilities |
+| 401/403/408/409/429/500/503 | `AI_APICallError`, original HTTP status and response headers retained; one attempt |
+| Malformed JSON | `AI_APICallError`, status 200, non-retryable |
+| Missing or out-of-range answer | `AI_InvalidResponseDataError`; rejected, not clamped |
+| Socket disconnect | `AI_APICallError`, no HTTP status, `UND_ERR_SOCKET` cause, retryable |
+| Deadline / external abort | `TimeoutError` / `AbortError`, no additional attempt |
+| Unknown destination | Harness rejects before HTTP |
+| Real Node preload | SDK-shaped fetch redirected with original body; unexpected host rejected |
+
+The test-only redirection preserves SDK URL/init/body rather than rebuilding its
+body as a Request stream. A streamed-body prototype failed the 401 probe with
+Node's `expected non-null body source` error, obscuring authentication failure;
+that prototype was removed. No product transport workaround is needed.
+
+The adapter archive includes its license. Its provider-utils 5.0.49 dependency
+omits a license, like 5.0.45. The upstream repository LICENSE at tag revision
+`ee3169b3c4880e2abe4d0d7c781243bb81822ec4` is byte-identical to the existing retained
+5.0.45 license. Slice 2 must extend the exact-version notice mapping before the
+adapter becomes part of the product bundle.
+
+Reproduce with `node --test packages/core/test-node/provider-protocol.test.mjs`;
+the same command runs inside `scripts/test-reference.sh`, and the normal suite
+now includes it. No credentials or live model calls were used.
+
+Docker dependency downloads required explicit build proxy arguments using the
+existing Docker Desktop proxy (`http.docker.internal:3128`); direct registry
+connections timed out. The completed image was then exercised by the unmodified
+network-isolated test script. No product configuration was changed.
diff --git a/specs/provider-support/choices.md b/specs/provider-support/choices.md
new file mode 100644
index 0000000..4e8969e
--- /dev/null
+++ b/specs/provider-support/choices.md
@@ -0,0 +1,10 @@
+# Provider support choices
+
+No unplanned product choice has been made in slice 1. The SDK pin, endpoint/model
+presets, external routing seam and immutable reference were specified. Probe file
+names and test layout are delegated implementation discretion.
+
+The routing fixture preserves the SDK's original body instead of converting it
+to a stream. This is a correctness fix within the specified transport contract,
+not an additional product behavior. Further passes append only decisions not
+already made or delegated by the plan.
diff --git a/specs/provider-support/slices/01-protocol.md b/specs/provider-support/slices/01-protocol.md
index 329c11a..29c6eab 100644
--- a/specs/provider-support/slices/01-protocol.md
+++ b/specs/provider-support/slices/01-protocol.md
@@ -1,7 +1,7 @@
# 1. Reproduce the provider protocol
**Unlock:** prove the chosen SDK and controlled routing before touching retrieval.
-No dependency on another slice. Status: not started.
+No dependency on another slice. Status: complete; see [verification](../assets/protocol-verification.md).
## Seam and artifact
diff --git a/test/fixtures/provider-route.mjs b/test/fixtures/provider-route.mjs
new file mode 100644
index 0000000..e8a3e8b
--- /dev/null
+++ b/test/fixtures/provider-route.mjs
@@ -0,0 +1,18 @@
+// Harness-only transport: never shipped or read by the CLI itself.
+const endpoints = new Set([
+ "https://ai-gateway.vercel.sh/typesafe/v1/systemone",
+ "https://api.typesafe.ai/v1/systemone",
+ "https://openrouter.ai/api/v1/systemone",
+]);
+export function routeProviderFetch(originalFetch, origin) {
+ return async (input, init) => {
+ const url = new URL(input);
+ if (!endpoints.has(url.href)) throw new Error(`Unexpected provider destination: ${url.href}`);
+ const headers = new Headers(init?.headers);
+ headers.set("x-jevgrep-original-url", url.href);
+ return originalFetch(new URL(url.pathname, origin), { ...init, headers });
+ };
+}
+if (process.env.JEVGREP_TEST_PROVIDER_ORIGIN) {
+ globalThis.fetch = routeProviderFetch(globalThis.fetch, process.env.JEVGREP_TEST_PROVIDER_ORIGIN);
+}