Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

19.2 KB
Raw Blame
/**
 * probe-live unit tests.
 *
 * Fix "error · 400 hides auth failures": composed-handler REMAPS terminal
 * upstream errors (401/403/terminal-429) to HTTP 400 so Claude Code surfaces
 * them instead of silently retrying. The probe must classify a remapped auth
 * failure as auth-failed and display the UPSTREAM code, via the structured
 * `error.upstream_status` field the remap site attaches.
 *
 * Fixture policy: the remapped bodies below are produced by the SAME functions
 * the remap site in composed-handler.ts calls (buildSurfacedErrorMessage +
 * wrapAnthropicError with upstreamStatus) — real producer output, not a
 * hand-crafted divergent shape.
 */

import { describe, expect, test } from "bun:test";
import {
  buildSurfacedErrorMessage,
  wrapAnthropicError,
} from "../handlers/shared/anthropic-error.js";
import {
  PROBE_MAX_TOKENS,
  type ProbeResult,
  classifyHttpError,
  describeProbeState,
  probeLink,
} from "./probe-live.js";

/** The exact body the composed-handler remap path sends for a terminal error. */
function remappedTerminalBody(upstreamStatus: number, providerMessage: string): string {
  const surfaced = buildSurfacedErrorMessage({
    providerDisplayName: "OpenCode Zen",
    status: upstreamStatus,
    hint: "Check API key / OAuth credentials.",
    providerMessage,
  });
  return JSON.stringify(wrapAnthropicError(400, surfaced, "invalid_request_error", upstreamStatus));
}

describe("classifyHttpError — remapped terminal errors (upstream_status)", () => {
  test("400 carrying upstream_status 401 classifies as auth-failed with the UPSTREAM code", () => {
    const body = remappedTerminalBody(401, "invalid api key");
    const result = classifyHttpError(400, body, 123);
    expect(result.state).toBe("auth-failed");
    expect(result.httpStatus).toBe(401);
    // The TUI line must read "auth failed · 401", not "error · 400".
    expect(describeProbeState(result)).toContain("auth failed · 401");
    expect(result.errorMessage).toContain("HTTP 401");
  });

  test("400 carrying upstream_status 403 classifies as auth-failed · 403", () => {
    const body = remappedTerminalBody(403, "forbidden");
    const result = classifyHttpError(400, body, 55);
    expect(result.state).toBe("auth-failed");
    expect(result.httpStatus).toBe(403);
  });

  test("a plain 400 (no upstream_status) still classifies as generic error · 400", () => {
    // Real non-remapped producer shape: the proxy's own invalid_request_error.
    const body = JSON.stringify(wrapAnthropicError(400, "missing required field: model"));
    const result = classifyHttpError(400, body, 10);
    expect(result.state).toBe("error");
    expect(result.httpStatus).toBe(400);
    expect(describeProbeState(result)).toContain("error · 400");
  });

  test("a remapped terminal 429 (upstream_status 429) classifies as out-of-credit · 429", () => {
    // Real case: Moonshot "suspended due to insufficient balance" / Z.AI code
    // 1113 — a billing exhaustion 429 the proxy remaps to 400. Must NOT read
    // as an opaque "error · 400" (what the user saw for kimi/glm), and must
    // NOT be "rate-limited" (the TUI treats throttling as healthy).
    const body = remappedTerminalBody(429, "insufficient balance");
    const result = classifyHttpError(400, body, 10);
    expect(result.state).toBe("out-of-credit");
    expect(result.httpStatus).toBe(429);
    expect(describeProbeState(result)).toContain("out of credit · 429");
  });

  test("a direct 402 Payment Required classifies as out-of-credit · 402", () => {
    const result = classifyHttpError(
      402,
      JSON.stringify({ error: { message: "subscription expired" } }),
      9
    );
    expect(result.state).toBe("out-of-credit");
    expect(result.httpStatus).toBe(402);
    expect(result.errorMessage).toBe("subscription expired");
  });

  test("a genuine (unremapped) 429 stays rate-limited — transient throttling is healthy", () => {
    const result = classifyHttpError(429, JSON.stringify({ error: { message: "slow down" } }), 8);
    expect(result.state).toBe("rate-limited");
  });

  test("a genuine upstream 401 is unaffected", () => {
    const result = classifyHttpError(
      401,
      JSON.stringify({ error: { message: "unauthorized" } }),
      7
    );
    expect(result.state).toBe("auth-failed");
    expect(result.httpStatus).toBe(401);
  });

  test("non-JSON body on a 400 does not throw and stays generic", () => {
    const result = classifyHttpError(400, "<html>bad gateway-ish text</html>", 5);
    expect(result.state).toBe("error");
    expect(result.httpStatus).toBe(400);
  });

  test("a proxy 503 connection failure classifies as network-error", () => {
    const body = JSON.stringify({
      type: "error",
      error: {
        type: "connection_error",
        message:
          "Cannot resolve chatgpt.com for OpenAI Codex. This is a DNS/network problem on your machine — check your internet connection, VPN, or DNS resolver (e.g. Tailscale MagicDNS) — not OpenAI Codex.",
      },
    });
    const result = classifyHttpError(503, body, 1385);
    // A proxy-side connection failure is a LOCAL network problem, not an upstream server error.
    expect(result.state).toBe("network-error");
    expect(result.errorMessage).toContain("Cannot resolve chatgpt.com");
  });

  test("a genuine transient upstream 503 stays server-error", () => {
    const body = JSON.stringify({
      type: "error",
      error: {
        type: "overloaded_error",
        message: "temporary overload",
      },
    });
    const result = classifyHttpError(503, body, 42);
    expect(result.state).toBe("server-error");
    expect(result.httpStatus).toBe(503);
  });
});

describe("probeLink — OAuth login hints", () => {
  const permissionDeniedBody =
    '{"error":{"status":"PERMISSION_DENIED","message":"Permission \'aiplatform.endpoints.predict\' denied on resource"}}';
  const unauthenticatedBody =
    '{"error":{"status":"UNAUTHENTICATED","message":"Request had invalid authentication credentials"}}';

  async function probeHttpError(
    provider: string,
    hasCredentials: boolean,
    status: number,
    body: string
  ): Promise<ProbeResult> {
    const server = Bun.serve({
      port: 0,
      fetch() {
        return new Response(body, {
          status,
          headers: { "Content-Type": "application/json" },
        });
      },
    });

    try {
      return await probeLink(
        `http://localhost:${server.port}`,
        { provider, modelSpec: `${provider}@test-model`, hasCredentials },
        1000
      );
    } finally {
      server.stop(true);
    }
  }

  test("antigravity suppresses the login hint on 403 without hiding the entitlement failure", async () => {
    const result = await probeHttpError("antigravity", false, 403, permissionDeniedBody);
    expect(result.actionHint).toBeUndefined();
    expect(result.state).toBe("auth-failed");
    expect(result.errorMessage).toContain("Permission 'aiplatform.endpoints.predict' denied");
  });

  test("antigravity keeps the login hint on 401", async () => {
    const result = await probeHttpError("antigravity", false, 401, unauthenticatedBody);
    expect(result.actionHint).toBe("run: claudish login antigravity");
    expect(result.state).toBe("auth-failed");
  });

  test("vertex suppresses the login hint on 403", async () => {
    const result = await probeHttpError("vertex", false, 403, permissionDeniedBody);
    expect(result.actionHint).toBeUndefined();
  });

  test("vertex keeps the login hint on 401", async () => {
    const result = await probeHttpError("vertex", false, 401, unauthenticatedBody);
    expect(result.actionHint).toBe("run: gcloud auth application-default login");
  });

  test("a non-OAuth provider does not get a login hint on 403", async () => {
    const result = await probeHttpError("openrouter", true, 403, permissionDeniedBody);
    expect(result.actionHint).toBeUndefined();
  });
});

describe("describeProbeState — upstream error details", () => {
  test("auth-failed includes the upstream region opt-in message", () => {
    const line = describeProbeState({
      state: "auth-failed",
      httpStatus: 403,
      latencyMs: 711,
      errorMessage:
        "The latest version of this model is only available hosted in China and requires explicit opt in: https://opencode.ai/workspace/wrk_01KK0EAEMKQZZN68V8YN1WAJ33/go",
    });
    expect(line).toContain("auth failed · 403");
    expect(line).toContain("only available hosted in China");
  });

  test("auth-failed without an error message keeps the status-only line", () => {
    expect(describeProbeState({ state: "auth-failed", httpStatus: 401, latencyMs: 120 })).toBe(
      "auth failed · 401 · 120ms"
    );
  });

  const detailCases: Array<{
    result: ProbeResult;
    statusOnly: string;
  }> = [
    {
      result: { state: "model-not-found", httpStatus: 404, latencyMs: 130 },
      statusOnly: "model not found · 404 · 130ms",
    },
    {
      result: { state: "out-of-credit", httpStatus: 402, latencyMs: 140 },
      statusOnly: "out of credit · 402 · 140ms",
    },
    {
      result: { state: "server-error", httpStatus: 503, latencyMs: 150 },
      statusOnly: "server error · 503 · 150ms",
    },
    {
      result: { state: "network-error", latencyMs: 160 },
      statusOnly: "network error · 160ms",
    },
  ];

  for (const { result, statusOnly } of detailCases) {
    test(`${result.state} appends a message and otherwise stays unchanged`, () => {
      expect(describeProbeState(result)).toBe(statusOnly);
      expect(describeProbeState({ ...result, errorMessage: "upstream detail" })).toBe(
        `${statusOnly} — upstream detail`
      );
    });
  }

  test("live never includes an error detail suffix", () => {
    expect(
      describeProbeState({ state: "live", latencyMs: 90, errorMessage: "must be ignored" })
    ).toBe("live · 90ms");
  });
});

describe("probe budget", () => {
  test("describeProbeState surfaces errorMessage for a code-less error (contentless 200)", () => {
    // Regression: a reasoning model that spends its whole budget before any
    // visible text returns HTTP 200 (no status) with a self-explaining
    // errorMessage. The TUI's describeProbeState dropped it → bare "error · Nms".
    const result: ProbeResult = {
      state: "error",
      latencyMs: 3340,
      errorMessage: "no visible output within probe budget (512 tokens consumed, none visible)",
    };
    const line = describeProbeState(result);
    expect(line).toContain("error · 3340ms");
    expect(line).toContain("no visible output within probe budget");
  });

  test("describeProbeState error without a message stays a bare error", () => {
    expect(describeProbeState({ state: "error", latencyMs: 10, httpStatus: 400 })).toBe(
      "error · 400 · 10ms"
    );
  });

  // Explicit contentless-truncation E2E: a reasoning model that burns the ENTIRE
  // probe budget on hidden reasoning yields HTTP 200 + finish_reason "length" +
  // zero visible text (verified live against OpenAI gpt-5-nano). The probe must
  // report the mapped explicit max_tokens signal, not "stream ended without content".
  //
  // Fixture policy: the Claude-side stream the probe consumes is produced by
  // the REAL openai-sse parser (createStreamingResponseHandler) — the actual
  // production producer. Only the upstream OpenAI chunks are synthesized,
  // mirroring the verified reproduction shape (empty content deltas,
  // finish_reason "length", usage with completion_tokens == the full cap).
  test("contentless 200 with explicit length finish reports max_tokens truncation", async () => {
    const { createStreamingResponseHandler } = await import("../handlers/shared/openai-compat.js");
    const { DefaultAPIFormat } = await import("../adapters/base-api-format.js");

    // Upstream OpenAI SSE: role delta with empty content → finish "length" →
    // usage chunk consuming the whole (new) probe budget on reasoning.
    const upstreamChunks = [
      `data: {"id":"chatcmpl-probe","object":"chat.completion.chunk","created":1,"model":"gpt-5-nano","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}`,
      `data: {"id":"chatcmpl-probe","object":"chat.completion.chunk","created":1,"model":"gpt-5-nano","choices":[{"index":0,"delta":{},"finish_reason":"length"}]}`,
      `data: {"id":"chatcmpl-probe","object":"chat.completion.chunk","created":1,"model":"gpt-5-nano","choices":[],"usage":{"prompt_tokens":25,"completion_tokens":${PROBE_MAX_TOKENS},"total_tokens":${25 + PROBE_MAX_TOKENS},"completion_tokens_details":{"reasoning_tokens":${PROBE_MAX_TOKENS}}}}`,
      "data: [DONE]",
    ].join("\n\n");

    const server = Bun.serve({
      port: 0,
      fetch() {
        const upstream = new Response(
          new ReadableStream({
            start(controller) {
              controller.enqueue(new TextEncoder().encode(`${upstreamChunks}\n\n`));
              controller.close();
            },
          }),
          { status: 200, headers: { "Content-Type": "text/event-stream" } }
        );
        // Minimal Hono-context shim (same shape format-translation.test.ts uses).
        const ctx: any = {
          body: (stream: ReadableStream, init?: any) => new Response(stream, init),
        };
        return createStreamingResponseHandler(
          ctx,
          upstream,
          new DefaultAPIFormat("gpt-5-nano"),
          "gpt-5-nano",
          null,
          undefined,
          undefined
        );
      },
    });

    try {
      const result = await probeLink(
        `http://127.0.0.1:${server.port}`,
        { provider: "openai", modelSpec: "oai@gpt-5-nano", hasCredentials: true },
        10000
      );
      expect(result.state).toBe("error");
      expect(result.errorMessage).toContain("no visible output within probe budget");
      expect(result.errorMessage).toContain("finish: max_tokens");
    } finally {
      server.stop(true);
    }
    // Explicit timeout > the probeLink budget (10_000ms) below. Without it the
    // test inherits Bun's 5_000ms default, which is SHORTER than the probe's own
    // budget — so under full-suite load (this passes in isolation) the probe
    // legitimately needs >5s to reach its terminal state and the test times out
    // before the operation it asserts on can finish.
  }, 20_000);

  test("contentless 200 with usage-only budget exhaustion reports tokens consumed", async () => {
    const { createStreamingResponseHandler } = await import("../handlers/shared/openai-compat.js");
    const { DefaultAPIFormat } = await import("../adapters/base-api-format.js");

    // Pins the fallback for providers that report usage but no truncation
    // finish_reason: the whole probe budget was consumed by hidden reasoning.
    const upstreamChunks = [
      `data: {"id":"chatcmpl-probe","object":"chat.completion.chunk","created":1,"model":"gpt-5-nano","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}`,
      `data: {"id":"chatcmpl-probe","object":"chat.completion.chunk","created":1,"model":"gpt-5-nano","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}`,
      `data: {"id":"chatcmpl-probe","object":"chat.completion.chunk","created":1,"model":"gpt-5-nano","choices":[],"usage":{"prompt_tokens":25,"completion_tokens":${PROBE_MAX_TOKENS},"total_tokens":${25 + PROBE_MAX_TOKENS},"completion_tokens_details":{"reasoning_tokens":${PROBE_MAX_TOKENS}}}}`,
      "data: [DONE]",
    ].join("\n\n");

    const server = Bun.serve({
      port: 0,
      fetch() {
        const upstream = new Response(
          new ReadableStream({
            start(controller) {
              controller.enqueue(new TextEncoder().encode(`${upstreamChunks}\n\n`));
              controller.close();
            },
          }),
          { status: 200, headers: { "Content-Type": "text/event-stream" } }
        );
        // Minimal Hono-context shim (same shape format-translation.test.ts uses).
        const ctx: any = {
          body: (stream: ReadableStream, init?: any) => new Response(stream, init),
        };
        return createStreamingResponseHandler(
          ctx,
          upstream,
          new DefaultAPIFormat("gpt-5-nano"),
          "gpt-5-nano",
          null,
          undefined,
          undefined
        );
      },
    });

    try {
      const result = await probeLink(
        `http://127.0.0.1:${server.port}`,
        { provider: "openai", modelSpec: "oai@gpt-5-nano", hasCredentials: true },
        10000
      );
      expect(result.state).toBe("error");
      expect(result.errorMessage).toContain("no visible output within probe budget");
      expect(result.errorMessage).toContain(`${PROBE_MAX_TOKENS} tokens consumed`);
    } finally {
      server.stop(true);
    }
    // Explicit timeout > the probeLink budget (10_000ms) below. Without it the
    // test inherits Bun's 5_000ms default, which is SHORTER than the probe's own
    // budget — so under full-suite load (this passes in isolation) the probe
    // legitimately needs >5s to reach its terminal state and the test times out
    // before the operation it asserts on can finish.
  }, 20_000);

  test("probe request forces minimal reasoning effort (reasoning models emit visible text)", async () => {
    // Verified live: gpt-5-nano with default effort spends the whole probe
    // budget on hidden reasoning → 0 visible text ~60% of runs. With
    // output_config.effort "minimal" it emits text deterministically (10/10).
    // The proxy maps this to reasoning_effort; here we assert the probe SENDS
    // it (the adapter mapping is covered by the api-format tests).
    let capturedBody: any = null;
    const server = Bun.serve({
      port: 0,
      async fetch(req) {
        capturedBody = await req.json();
        return new Response(
          'data: {"type":"content_block_delta","delta":{"text":"one"}}\n\ndata: [DONE]\n\n',
          {
            status: 200,
            headers: { "Content-Type": "text/event-stream" },
          }
        );
      },
    });
    try {
      await probeLink(
        `http://127.0.0.1:${server.port}`,
        { provider: "openai", modelSpec: "oai@gpt-5-nano", hasCredentials: true },
        10000
      );
      expect(capturedBody?.output_config?.effort).toBe("minimal");
    } finally {
      server.stop(true);
    }
  });

  test("a genuinely dead contentless stream keeps the plain message", async () => {
    // No stop reason, no usage — nothing indicates budget truncation.
    const server = Bun.serve({
      port: 0,
      fetch() {
        return new Response("data: [DONE]\n\n", {
          status: 200,
          headers: { "Content-Type": "text/event-stream" },
        });
      },
    });
    try {
      const result = await probeLink(
        `http://127.0.0.1:${server.port}`,
        { provider: "openai", modelSpec: "oai@gpt-5-nano", hasCredentials: true },
        10000
      );
      expect(result.state).toBe("error");
      expect(result.errorMessage).toBe("stream ended without content");
    } finally {
      server.stop(true);
    }
    // Explicit timeout > the probeLink budget (10_000ms). See the sibling
    // budget-truncation test above — Bun's 5_000ms default is shorter than the
    // probe's own budget, so this fails under full-suite load in isolation.
  }, 20_000);
});

// Type-level pin: ProbeResult.httpStatus carries the DISPLAYED code, which for
// remapped errors is the upstream status — keep the field optional-number.
const _pin: ProbeResult = { state: "error", latencyMs: 0, httpStatus: 401 };
void _pin;