Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

21.2 KB
Raw Blame
import { routeProviderFetch } from "./fixtures/provider-route.mjs";
import { expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createEvaluationCache } from "../packages/core/src/cache";
import { createEvaluator, EvaluationFailure } from "../packages/core/src/evaluator";

test("Jev uses native state and validated boolean probabilities through real HTTP", async () => {
  const state = {
    query: "find event recording",
    items: [{ path: "events.ts", source: "recordEvent()" }],
  };
  const server = Bun.serve({
    port: 0,
    async fetch(request) {
      expect(await request.json()).toEqual({
        model: "typesafe-ai/jev",
        state,
        questions: { useful: { type: "noul", instructions: "Is the source useful?" } },
      });
      return Response.json({ answers: { useful: { type: "noul", noul: 0.8 } } });
    },
  });
  try {
    const evaluator = createEvaluator({
      apiKey: "fixture",
      provider: "vercel",
      fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
      signal: new AbortController().signal,
    });
    const result = await evaluator.evaluate({
      state,
      questions: { useful: { type: "boolean", instructions: "Is the source useful?" } },
    });
    expect(result).toEqual({ useful: 0.8 });
    expect(evaluator.requests).toBe(1);
  } finally {
    server.stop(true);
  }
});

test("transient failures retry within the shared request guard and never become scores", async () => {
  let calls = 0;
  const server = Bun.serve({
    port: 0,
    fetch() {
      calls++;
      return calls === 1
        ? Response.json({ error: "retry" }, { status: 503 })
        : Response.json({ answers: { q: { type: "noul", noul: 0.2 } } });
    },
  });
  try {
    const evaluator = createEvaluator({
      apiKey: "fixture",
      provider: "vercel",
      fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
      signal: new AbortController().signal,
      requestLimit: 2,
    });
    const request = {
      state: "test",
      questions: { q: { type: "boolean" as const, instructions: "Relevant?" } },
    };
    expect(await evaluator.evaluate(request)).toEqual({ q: 0.2 });
    await expect(evaluator.evaluate(request)).rejects.toMatchObject({ kind: "request-limit" });
    expect(calls).toBe(2);
    expect(evaluator.requests).toBe(2);
  } finally {
    server.stop(true);
  }
});

test("cancellation of a rate-limited request prevents further attempts", async () => {
  const controller = new AbortController();
  let calls = 0;
  let firstResponse!: () => void;
  const received = new Promise<void>((resolve) => {
    firstResponse = resolve;
  });
  const server = Bun.serve({
    port: 0,
    fetch() {
      calls++;
      firstResponse();
      return Response.json(
        { error: "rate limited" },
        { status: 429, headers: { "retry-after": "30" } },
      );
    },
  });
  try {
    const evaluator = createEvaluator({
      apiKey: "fixture",
      provider: "vercel",
      fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
      signal: controller.signal,
    });
    const result = evaluator.evaluate({
      state: "test",
      questions: { q: { type: "boolean", instructions: "Relevant?" } },
    });
    await received;
    controller.abort();
    await expect(result).rejects.toMatchObject({ kind: "cancelled" });
    expect(calls).toBe(1);
    expect(evaluator.requests).toBe(1);
  } finally {
    controller.abort();
    server.stop(true);
  }
}, 2000);

test("Retry-After delays a retry before the provider can recover", async () => {
  const received: number[] = [];
  const server = Bun.serve({
    port: 0,
    fetch() {
      received.push(performance.now());
      return received.length === 1
        ? Response.json(
            { error: "rate limited" },
            { status: 429, headers: { "retry-after": "0.1" } },
          )
        : Response.json({ answers: { q: { type: "noul", noul: 0.8 } } });
    },
  });
  try {
    const evaluator = createEvaluator({
      apiKey: "fixture",
      provider: "vercel",
      fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
      signal: new AbortController().signal,
    });
    expect(
      await evaluator.evaluate({
        state: "test",
        questions: { q: { type: "boolean", instructions: "Relevant?" } },
      }),
    ).toEqual({ q: 0.8 });
    expect(received.length).toBe(2);
    expect(received[1]! - received[0]!).toBeGreaterThanOrEqual(95);
    expect(evaluator.requests).toBe(2);
  } finally {
    server.stop(true);
  }
});

test("stalled HTTP attempts time out without exceeding the evaluator attempt limit", async () => {
  let calls = 0;
  const server = Bun.serve({
    port: 0,
    fetch() {
      calls++;
      return new Promise<Response>(() => {});
    },
  });
  try {
    const evaluator = createEvaluator({
      apiKey: "fixture",
      provider: "vercel",
      fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
      signal: new AbortController().signal,
      timeoutMs: 100,
    });
    await expect(
      evaluator.evaluate({
        state: "test",
        questions: { q: { type: "boolean", instructions: "Relevant?" } },
      }),
    ).rejects.toMatchObject({
      kind: "provider",
      message: expect.stringContaining("timed out after 100 ms"),
    });
    expect(calls).toBe(2);
    expect(evaluator.requests).toBe(2);
  } finally {
    server.stop(true);
  }
}, 2000);

test("authentication failure stops other in-flight and subsequent query requests", async () => {
  let calls = 0;
  const server = Bun.serve({
    port: 0,
    fetch() {
      calls++;
      if (calls === 1) return new Promise<Response>(() => {});
      if (calls === 2) return Response.json({ error: "unauthorized" }, { status: 401 });
      return Response.json({ answers: { q: { type: "noul", noul: 0.8 } } });
    },
  });
  try {
    const evaluator = createEvaluator({
      apiKey: "fixture",
      provider: "vercel",
      fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
      signal: new AbortController().signal,
      timeoutMs: 1000,
    });
    const request = {
      state: "test",
      questions: { q: { type: "boolean" as const, instructions: "Relevant?" } },
    };
    const outcomes = await Promise.allSettled([
      evaluator.evaluate(request),
      evaluator.evaluate(request),
    ]);
    expect(outcomes).toMatchObject([
      { status: "rejected", reason: { kind: "authentication" } },
      { status: "rejected", reason: { kind: "authentication" } },
    ]);
    await expect(evaluator.evaluate(request)).rejects.toMatchObject({ kind: "authentication" });
    expect(calls).toBe(2);
  } finally {
    server.stop(true);
  }
}, 2000);

test("authentication failure interrupts a sibling Retry-After wait", async () => {
  let calls = 0;
  let releaseAuthentication!: () => void;
  const authenticationReady = new Promise<void>((resolve) => {
    releaseAuthentication = resolve;
  });
  const server = Bun.serve({
    port: 0,
    async fetch() {
      calls++;
      if (calls === 1) {
        await authenticationReady;
        return Response.json({ error: "unauthorized" }, { status: 401 });
      }
      setTimeout(releaseAuthentication, 50);
      return Response.json({ error: "limited" }, { status: 429, headers: { "retry-after": "30" } });
    },
  });
  try {
    const evaluator = createEvaluator({
      apiKey: "fixture",
      provider: "vercel",
      fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
      signal: new AbortController().signal,
    });
    const request = {
      state: "test",
      questions: { q: { type: "boolean" as const, instructions: "Relevant?" } },
    };
    const outcomes = await Promise.allSettled([
      evaluator.evaluate(request),
      evaluator.evaluate(request),
    ]);
    expect(outcomes).toMatchObject([
      { status: "rejected", reason: { kind: "authentication" } },
      { status: "rejected", reason: { kind: "authentication" } },
    ]);
    expect(calls).toBe(2);
  } finally {
    server.stop(true);
  }
}, 2000);

for (const status of [409, 422, 503]) {
  test(`navigation HTTP ${status} preserves retry and split policy`, async () => {
    let calls = 0;
    const server = Bun.serve({
      port: 0,
      fetch() {
        calls++;
        return Response.json({ message: "synthetic failure" }, { status });
      },
    });
    try {
      const evaluator = createEvaluator({
        provider: "openrouter",
        apiKey: "fixture",
        fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
        signal: new AbortController().signal,
      });
      await expect(
        evaluator.evaluate(
          {
            state: { source: "code" },
            questions: {
              q0: { type: "boolean", instructions: "Relevant?" },
              q1: { type: "boolean", instructions: "Related?" },
            },
          },
          { navigation: true },
        ),
      ).rejects.toMatchObject({ kind: "provider", splitEligible: status === 503 });
      expect(calls).toBe(1);
      expect(evaluator.requests).toBe(1);
    } finally {
      server.stop(true);
    }
  });
}

test("native HTTP-date Retry-After is honored before another attempt", async () => {
  const retryAt = new Date(Math.ceil(Date.now() / 1000) * 1000 + 1000);
  const arrivals: number[] = [];
  const server = Bun.serve({
    port: 0,
    fetch() {
      arrivals.push(Date.now());
      return arrivals.length === 1
        ? Response.json(
            { message: "rate limited" },
            { status: 429, headers: { "retry-after": retryAt.toUTCString() } },
          )
        : Response.json({ answers: { q: { type: "noul", noul: 0.8 } } });
    },
  });
  try {
    const evaluator = createEvaluator({
      provider: "typesafe",
      apiKey: "fixture",
      fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
      signal: new AbortController().signal,
    });
    expect(
      await evaluator.evaluate({
        state: "code",
        questions: { q: { type: "boolean", instructions: "Relevant?" } },
      }),
    ).toEqual({ q: 0.8 });
    expect(arrivals.length).toBe(2);
    expect(arrivals[1]!).toBeGreaterThanOrEqual(retryAt.getTime());
  } finally {
    server.stop(true);
  }
}, 5000);

test("queued requests validate fresh source only after a provider slot is available", async () => {
  let release!: () => void;
  const gate = new Promise<void>((resolve) => {
    release = resolve;
  });
  let seen!: () => void;
  const received = new Promise<void>((resolve) => {
    seen = resolve;
  });
  let calls = 0;
  let stale = false;
  const server = Bun.serve({
    port: 0,
    async fetch() {
      calls++;
      seen();
      await gate;
      return Response.json({ answers: { q: { type: "noul", noul: 0.8 } } });
    },
  });
  try {
    const evaluator = createEvaluator({
      apiKey: "fixture",
      provider: "vercel",
      concurrency: 1,
      signal: new AbortController().signal,
      fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
    });
    const request = {
      state: "test",
      questions: { q: { type: "boolean" as const, instructions: "Relevant?" } },
    };
    const first = evaluator.evaluate(request);
    await received;
    const queued = evaluator
      .evaluate(request, {
        beforeAttempt: async () => {
          if (stale) throw new EvaluationFailure("source-invalid");
        },
      })
      .catch((error) => error);
    await new Promise<void>((resolve) => setImmediate(resolve));
    stale = true;
    release();
    expect(await first).toEqual({ q: 0.8 });
    expect(await queued).toMatchObject({ kind: "source-invalid" });
    expect(calls).toBe(1);
  } finally {
    release();
    server.stop(true);
  }
});

for (const kind of ["cancelled", "authentication"] as const)
  test(`${kind} rejects every queued request without sending it`, async () => {
    const controller = new AbortController();
    let release!: () => void;
    const gate = new Promise<void>((resolve) => {
      release = resolve;
    });
    let seen!: () => void;
    const received = new Promise<void>((resolve) => {
      seen = resolve;
    });
    let calls = 0;
    const server = Bun.serve({
      port: 0,
      async fetch() {
        calls++;
        seen();
        await gate;
        return kind === "authentication"
          ? Response.json({ error: "invalid key" }, { status: 401 })
          : Response.json({ answers: { q: { type: "noul", noul: 0.8 } } });
      },
    });
    try {
      const evaluator = createEvaluator({
        apiKey: "fixture",
        provider: "vercel",
        concurrency: 1,
        signal: controller.signal,
        fetch: routeProviderFetch(fetch, `http://127.0.0.1:${server.port}`),
      });
      const results = Promise.allSettled(
        Array.from({ length: 20 }, (_, i) =>
          evaluator.evaluate({
            state: String(i),
            questions: { q: { type: "boolean", instructions: "Relevant?" } },
          }),
        ),
      );
      await received;
      if (kind === "authentication") release();
      else controller.abort();
      const settled = await results;
      for (const result of settled)
        expect(result).toMatchObject({ status: "rejected", reason: { kind } });
      expect(calls).toBe(1);
      expect(evaluator.requests).toBe(1);
    } finally {
      release();
      server.stop(true);
    }
  });

test("a custom provider uses its saved base URL and model through the same protocol", async () => {
  const observed: Array<{ path: string; authorization: string; body: Record<string, unknown> }> =
    [];
  const server = Bun.serve({
    port: 0,
    async fetch(request) {
      observed.push({
        path: new URL(request.url).pathname,
        authorization: request.headers.get("authorization") ?? "",
        body: (await request.json()) as Record<string, unknown>,
      });
      return Response.json({ answers: { q: { type: "noul", noul: 0.7 } } });
    },
  });
  try {
    const evaluator = createEvaluator({
      apiKey: "fixture-key",
      provider: "custom",
      baseURL: `http://127.0.0.1:${server.port}/gateway/v1`,
      model: "gateway/jev-2",
      signal: new AbortController().signal,
    });
    expect(
      await evaluator.evaluate({
        state: "test",
        questions: { q: { type: "boolean", instructions: "Relevant?" } },
      }),
    ).toEqual({ q: 0.7 });
    expect(observed).toEqual([
      {
        path: "/gateway/v1/systemone",
        authorization: "Bearer fixture-key",
        body: {
          model: "gateway/jev-2",
          state: "test",
          questions: { q: { type: "noul", instructions: "Relevant?" } },
        },
      },
    ]);
  } finally {
    server.stop(true);
  }
});

test("custom cache identity keeps endpoints and models separate from presets", async () => {
  const models: string[] = [];
  const server = Bun.serve({
    port: 0,
    async fetch(request) {
      const body = (await request.json()) as { model: string };
      models.push(body.model);
      return Response.json({ answers: { q: { type: "noul", noul: 0.6 } } });
    },
  });
  const directory = await mkdtemp(join(tmpdir(), "jevgrep-custom-cache-"));
  try {
    const request = {
      state: "test",
      questions: { q: { type: "boolean" as const, instructions: "Relevant?" } },
    };
    const signal = new AbortController().signal;
    const origin = `http://127.0.0.1:${server.port}`;
    const evaluator = (options: Record<string, unknown>) =>
      createEvaluator({
        apiKey: "fixture-key",
        signal,
        cache: createEvaluationCache({ directory }),
        ...options,
      } as Parameters<typeof createEvaluator>[0]);

    expect(
      await evaluator({
        provider: "vercel",
        fetch: routeProviderFetch(fetch, origin),
      }).evaluate(request),
    ).toEqual({ q: 0.6 });
    expect(
      await evaluator({
        provider: "custom",
        baseURL: `${origin}/gateway/v1`,
        model: "gateway/jev-2",
      }).evaluate(request),
    ).toEqual({ q: 0.6 });
    expect(models).toEqual(["typesafe-ai/jev", "gateway/jev-2"]);

    expect(
      await evaluator({
        provider: "custom",
        baseURL: `${origin}/gateway/v1`,
        model: "gateway/jev-2",
      }).evaluate(request),
    ).toEqual({ q: 0.6 });
    expect(models).toEqual(["typesafe-ai/jev", "gateway/jev-2"]);

    expect(
      await evaluator({
        provider: "custom",
        baseURL: `${origin}/gateway/v1`,
        model: "gateway/jev-3",
      }).evaluate(request),
    ).toEqual({ q: 0.6 });
    expect(
      await evaluator({
        provider: "custom",
        baseURL: `${origin}/other/v1`,
        model: "gateway/jev-2",
      }).evaluate(request),
    ).toEqual({ q: 0.6 });
    expect(models).toEqual(["typesafe-ai/jev", "gateway/jev-2", "gateway/jev-3", "gateway/jev-2"]);
  } finally {
    server.stop(true);
    await rm(directory, { recursive: true, force: true });
  }
});

test("native token pacing preserves concurrent small calls and excludes queue time from timeout", async () => {
  const arrivals: number[] = [];
  const starts: Array<{ at: number; tokens: number }> = [];
  const inputTokens = (state: string) => (state.length > 1000 ? 60_000 : 100);
  let active = 0;
  let peak = 0;
  const server = Bun.serve({
    port: 0,
    async fetch(request) {
      const body = (await request.json()) as { state: string };
      const tokens = inputTokens(body.state);
      arrivals.push(performance.now());
      peak = Math.max(peak, ++active);
      await new Promise((resolve) => setTimeout(resolve, 80));
      active--;
      return Response.json({
        answers: { q: { type: "noul", noul: 0.8 } },
        usage: { input_tokens: tokens, output_tokens: 1 },
      });
    },
  });
  try {
    const evaluator = createEvaluator({
      provider: "typesafe",
      apiKey: "fixture",
      signal: new AbortController().signal,
      timeoutMs: 500,
      fetch: routeProviderFetch((input, init) => {
        const body = JSON.parse(init!.body as string) as { state: string };
        starts.push({ at: performance.now(), tokens: inputTokens(body.state) });
        return fetch(input, init);
      }, `http://127.0.0.1:${server.port}`),
    });
    const values = await Promise.all(
      Array.from({ length: 32 }, (_, i) =>
        evaluator.evaluate({
          state: i < 29 ? "small" : "x".repeat(65_000),
          questions: { q: { type: "boolean", instructions: "Relevant?" } },
        }),
      ),
    );
    expect(values.every((value) => value.q === 0.8)).toBe(true);
    expect(peak).toBeGreaterThan(4);
    expect(arrivals).toHaveLength(32);
    expect(starts).toHaveLength(32);
    for (const start of starts) {
      // Pace client starts; transport delay can compress server arrival windows.
      const tokens = starts
        .filter((entry) => entry.at >= start.at && entry.at < start.at + 990)
        .reduce((sum, entry) => sum + entry.tokens, 0);
      expect(tokens).toBeLessThanOrEqual(100_000);
    }
  } finally {
    server.stop(true);
  }
}, 10_000);

test("TypeSafe admits at most 40 small requests per second through real HTTP", async () => {
  const arrivals: number[] = [];
  const starts: number[] = [];
  const server = Bun.serve({
    port: 0,
    fetch() {
      arrivals.push(performance.now());
      return Response.json({
        answers: { q: { type: "noul", noul: 0.8 } },
        usage: { input_tokens: 1, output_tokens: 1 },
      });
    },
  });
  try {
    const evaluator = createEvaluator({
      provider: "typesafe",
      apiKey: "fixture",
      signal: new AbortController().signal,
      concurrency: 64,
      fetch: routeProviderFetch((input, init) => {
        starts.push(performance.now());
        return fetch(input, init);
      }, `http://127.0.0.1:${server.port}`),
    });
    const values = await Promise.all(
      Array.from({ length: 80 }, () =>
        evaluator.evaluate({
          state: "small",
          questions: { q: { type: "boolean", instructions: "Relevant?" } },
        }),
      ),
    );
    expect(values.every((value) => value.q === 0.8)).toBe(true);
    expect(arrivals).toHaveLength(80);
    expect(starts).toHaveLength(80);
    expect(starts[39]! - starts[0]!).toBeLessThan(900);
    for (const start of starts) {
      // Unit tests own the exact boundary; allow SDK dispatch scheduling jitter.
      expect(starts.filter((at) => at >= start && at < start + 900).length).toBeLessThanOrEqual(40);
    }
  } finally {
    server.stop(true);
  }
}, 10_000);

test("token-queued requests revalidate source after waiting", async () => {
  let calls = 0;
  let stale = false;
  const evaluator = createEvaluator({
    provider: "typesafe",
    apiKey: "fixture",
    signal: new AbortController().signal,
    fetch: async () => {
      calls++;
      stale = true;
      return Response.json({ answers: { q: { type: "noul", noul: 0.8 } } });
    },
  });
  const request = {
    state: "x".repeat(65_000),
    questions: { q: { type: "boolean" as const, instructions: "Relevant?" } },
  };
  const running = Array.from({ length: 3 }, () => evaluator.evaluate(request));
  const queued = evaluator
    .evaluate(request, {
      beforeAttempt: async () => {
        if (stale) throw new EvaluationFailure("source-invalid");
      },
    })
    .catch((error) => error);
  await Promise.all(running);
  expect(await queued).toMatchObject({ kind: "source-invalid" });
  expect(calls).toBe(3);
}, 10_000);