Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

25.5 KB
Raw Blame
/**
 * E2E tests for the provider fallback mechanism.
 *
 * These tests use REAL API tokens and hit actual provider endpoints.
 * They start a real claudish proxy server and send Anthropic-format
 * /v1/messages requests with bare model names (no provider@ prefix)
 * to validate fallback chain behavior end-to-end.
 *
 * Required env vars (tests skip gracefully if not set):
 *   MINIMAX_API_KEY or OPENCODE_API_KEY or OPENROUTER_API_KEY
 *
 * Run: bun test packages/cli/src/handlers/fallback-handler.test.ts
 */

import { afterAll, describe, expect, test } from "bun:test";
import { createProxyServer } from "../proxy-server.js";
import { hasAnyCredential } from "../test-helpers/credential-gate.js";
import type { ProxyServer } from "../types.js";

// ---------------------------------------------------------------------------
// Test infrastructure
// ---------------------------------------------------------------------------

const TEST_PORT = 18900 + Math.floor(Math.random() * 100);

let proxyServer: ProxyServer | null = null;

async function ensureProxy(): Promise<number> {
  if (proxyServer) return TEST_PORT;

  proxyServer = await createProxyServer(
    TEST_PORT,
    process.env.OPENROUTER_API_KEY,
    undefined, // no default model — let fallback decide
    false,
    process.env.ANTHROPIC_API_KEY,
    undefined,
    { quiet: true }
  );
  return TEST_PORT;
}

afterAll(async () => {
  if (proxyServer) {
    await proxyServer.shutdown();
    proxyServer = null;
  }
});

/**
 * Send a minimal /v1/messages request to the proxy.
 * Returns { ok, status, body } where body is parsed from JSON or SSE.
 */
async function sendMessage(
  port: number,
  model: string,
  prompt = "Say hello in 5 words"
): Promise<{ ok: boolean; status: number; body: any }> {
  const res = await fetch(`http://127.0.0.1:${port}/v1/messages`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model,
      max_tokens: 64,
      stream: false,
      messages: [{ role: "user", content: prompt }],
    }),
  });

  const contentType = res.headers.get("content-type") || "";
  let body: any;

  if (contentType.includes("text/event-stream")) {
    // SSE response — parse event stream for content
    const text = await res.text();
    const lines = text.split("\n");
    let lastData: any = null;
    const textParts: string[] = [];
    let hasError = false;
    let errorData: any = null;

    for (const line of lines) {
      // SSE spec: "data:" with optional space — handle both "data: {...}" and "data:{...}"
      const isDataLine = line.startsWith("data: ") || line.startsWith("data:");
      if (isDataLine) {
        const data = (line.startsWith("data: ") ? line.slice(6) : line.slice(5)).trim();
        if (data === "[DONE]") continue;
        try {
          const parsed = JSON.parse(data);
          lastData = parsed;

          // Anthropic SSE: content_block_delta with text
          if (parsed.type === "content_block_delta" && parsed.delta?.text) {
            textParts.push(parsed.delta.text);
          }
          // Anthropic SSE: message_start with content array
          if (parsed.type === "message_start" && parsed.message?.content?.length > 0) {
            for (const block of parsed.message.content) {
              if (block.text) textParts.push(block.text);
            }
          }
          // OpenAI SSE: choices[].delta.content
          if (parsed.choices?.[0]?.delta?.content) {
            textParts.push(parsed.choices[0].delta.content);
          }
          // Error events
          if (parsed.type === "error" || parsed.error) {
            hasError = true;
            errorData = parsed;
          }
        } catch {
          // Skip non-JSON data lines
        }
      }
    }

    if (textParts.length > 0) {
      body = {
        content: [{ type: "text", text: textParts.join("") }],
        _raw_sse: true,
      };
      return { ok: true, status: res.status, body };
    }
    if (hasError && errorData) {
      return { ok: false, status: res.status, body: errorData };
    }
    if (lastData?.type === "message_stop" || lastData?.type === "message_delta") {
      // Anthropic SSE completed but no text extracted — treat as success (empty response)
      body = { content: [{ type: "text", text: "" }], _raw_sse: true };
      return { ok: true, status: res.status, body };
    }
    body = lastData || { _raw_text: text.slice(0, 500) };
    return { ok: false, status: res.status, body };
  }
  // JSON response
  try {
    body = await res.json();
  } catch {
    body = { _raw_text: await res.text() };
  }
  if (body?.type === "message" && Array.isArray(body.content) && body.content.length === 0) {
    // Both response envelopes must classify an empty-but-successful turn the same way.
    body.content = [{ type: "text", text: "" }];
    return { ok: true, status: res.status, body };
  }
  return { ok: res.ok, status: res.status, body };
}

/**
 * Can claudish serve ANY fallback-capable provider?
 *
 * Asked via claudish's OWN credential authority (env → aliases →
 * ~/.claudish/config.json apiKeys → 1Password), NOT raw process.env — a key
 * stored any supported way makes claudish able to serve the request, so these
 * tests must run rather than silently skip. Resolved once at module load.
 */
const HAS_ANY_CREDENTIALS = await hasAnyCredential([
  "minimax",
  "minimax-coding",
  "opencode-zen",
  "openrouter",
  "litellm",
  "google",
  "kimi",
  "kimi-coding",
  "openai",
]);

function hasAnyCredentials(): boolean {
  return HAS_ANY_CREDENTIALS;
}

// ---------------------------------------------------------------------------
// (Group 1 — unit tests of getFallbackChain — removed in commit 5 of the
// model-catalog and routing redesign. The fallback chain construction logic
// moved from auto-route.ts (now ~50 lines) into routing-rules.ts route().
// Equivalent matrix coverage lives in providers/routing-rules.test.ts and
// providers/route-candidates.test.ts.)
// ---------------------------------------------------------------------------

// ---------------------------------------------------------------------------
// Group 2: Real API — fallback produces a valid response or structured error
// ---------------------------------------------------------------------------

describe("Group 2: Real API — fallback response structure", () => {
  test.skipIf(!hasAnyCredentials())(
    "minimax-m2.5 without prefix returns success or structured fallback error",
    async () => {
      const port = await ensureProxy();

      const { ok, body } = await sendMessage(port, "minimax-m2.5");

      if (ok) {
        // Some provider in the chain succeeded
        expect(body.content).toBeDefined();
        expect(body.content.length).toBeGreaterThan(0);
      } else if (body.error?.type === "all_providers_failed") {
        // All providers failed — structured fallback error
        expect(body.error.attempts).toBeInstanceOf(Array);
        expect(body.error.attempts.length).toBeGreaterThan(0);

        for (const attempt of body.error.attempts) {
          expect(attempt.provider).toBeDefined();
          expect(typeof attempt.status).toBe("number");
          expect(attempt.error).toBeDefined();
        }
      } else {
        // Single-provider error or raw SSE error — just verify it's not silently swallowed
        expect(body).toBeDefined();
      }
    },
    30_000
  );

  test.skipIf(!hasAnyCredentials())(
    "glm-5-turbo without prefix returns success or structured fallback error",
    async () => {
      const port = await ensureProxy();

      const { ok, body } = await sendMessage(port, "glm-5-turbo");

      if (ok) {
        expect(body.content).toBeDefined();
      } else if (body.error?.type === "all_providers_failed") {
        expect(body.error.attempts.length).toBeGreaterThan(0);
      } else {
        expect(body).toBeDefined();
      }
    },
    30_000
  );

  test.skipIf(!hasAnyCredentials())(
    "kimi-k2.5 without prefix returns success or structured fallback error",
    async () => {
      const port = await ensureProxy();

      const { ok, body } = await sendMessage(port, "kimi-k2.5");

      if (ok) {
        expect(body.content).toBeDefined();
      } else if (body.error?.type === "all_providers_failed") {
        expect(body.error.attempts.length).toBeGreaterThan(0);
      } else {
        expect(body).toBeDefined();
      }
    },
    30_000
  );
});

// ---------------------------------------------------------------------------
// Group 3: Real API — fallback actually tries multiple providers
// ---------------------------------------------------------------------------

describe("Group 3: Real API — multi-provider fallback in action", () => {
  test.skipIf(!hasAnyCredentials())(
    "bare model tries multiple providers and either succeeds or returns an error",
    async () => {
      const port = await ensureProxy();

      const { ok, body } = await sendMessage(port, "minimax-m2.5");

      if (ok) {
        // Fallback chain found a working provider
        expect(body.content).toBeDefined();
        expect(body.content.length).toBeGreaterThan(0);
      } else if (body.type === "message_stop" || body._raw_sse) {
        // SSE stream completed (Anthropic-compat provider responded) but no text was
        // extracted by the test helper. The fallback chain DID succeed at HTTP level —
        // the response was just too short or used a format the test parser doesn't cover.
        // This is still a valid outcome — the provider accepted the request.
        expect(body).toBeDefined();
      } else {
        // Real error — must have a structured error
        expect(body.error).toBeDefined();
        if (body.error.type === "all_providers_failed") {
          expect(body.error.attempts.length).toBeGreaterThanOrEqual(1);
          for (const attempt of body.error.attempts) {
            expect(attempt.provider).toBeDefined();
            expect(typeof attempt.status).toBe("number");
          }
        } else {
          // Single-provider error (non-retryable) — must have type and message
          expect(body.error.type).toBeDefined();
          expect(body.error.message).toBeDefined();
        }
      }
    },
    30_000
  );

  test.skipIf(!hasAnyCredentials())(
    "completely unknown model fails with a structured error",
    async () => {
      const port = await ensureProxy();

      const { ok, body } = await sendMessage(port, "nonexistent-model-xyz-999");

      // Unknown model should NOT succeed
      expect(ok).toBe(false);
      // Must return some structured error — either fallback chain or single provider
      expect(body.error).toBeDefined();
      expect(body.error.type).toBeDefined();
      expect(body.error.message).toBeDefined();
    },
    30_000
  );
});

// ---------------------------------------------------------------------------
// Group 4: Real API — explicit provider prefix bypasses fallback
// ---------------------------------------------------------------------------

describe("Group 4: Real API — explicit provider skips fallback", () => {
  test.skipIf(!process.env.MINIMAX_API_KEY)(
    "[email protected] (explicit) does NOT use fallback chain",
    async () => {
      const port = await ensureProxy();

      const result = await sendMessage(port, "[email protected]");

      // Explicit provider must NOT trigger fallback chain
      if (!result.ok && result.body.error?.type === "all_providers_failed") {
        throw new Error(
          `Explicit provider mm@ triggered fallback chain with ${result.body.error.attempts.length} attempts — should go direct to MiniMax only`
        );
      }
      // Either succeeds (direct MiniMax) or returns a single-provider error (not wrapped in fallback)
    },
    30_000
  );

  test.skipIf(!process.env.OPENROUTER_API_KEY)(
    "or@minimax/minimax-m2.5 (explicit OpenRouter) goes direct",
    async () => {
      const port = await ensureProxy();

      const { ok, body } = await sendMessage(port, "or@minimax/minimax-m2.5");

      if (ok) {
        expect(body.content).toBeDefined();
        expect(body.content.length).toBeGreaterThan(0);
      } else {
        // Explicit routing error must NOT be a fallback chain error
        expect(body.error?.type).not.toBe("all_providers_failed");
      }
    },
    30_000
  );
});

// ---------------------------------------------------------------------------
// Group 5: isRetryableError classification (unit tests)
// ---------------------------------------------------------------------------

describe("Group 5: isRetryableError — unit tests via FallbackHandler behavior", () => {
  // Keep the retryability assertions black-box: mock handlers make it observable
  // whether FallbackHandler tries the next candidate or stops.

  const { Hono } = require("hono");
  const {
    FallbackHandler,
    exhaustedChainStatus,
    isProvider,
    isRetryableError,
  } = require("./fallback-handler.js");
  const { extractUpstreamStatus } = require("./shared/anthropic-error.js");

  function remappedErrorBody(upstreamStatus?: unknown): string {
    const error: Record<string, unknown> = {
      type: "authentication_error",
      message: "invalid api key",
    };
    if (upstreamStatus !== undefined) error.upstream_status = upstreamStatus;
    return JSON.stringify({ type: "error", error });
  }

  // Captured verbatim from /Users/jack/.claudish/logs/claudish_2026-08-07_08-51-52.log:23
  const ZEN_GO_INVALID_TOOL_BODY =
    '{"error":{"type":"server_error","message":"Error from provider (Console Go): Upstream request failed: [bad_request_error] invalid params, invalid tool type:  (2013)"}}';

  // Captured verbatim from /Users/jack/.claudish/logs/claudish_2026-08-07_12-34-35.log:11
  const ZEN_GO_PROVIDER_ERROR_BODY =
    '{"error":{"type":"server_error","message":"Error from provider (Console Go): Upstream request failed: [400] Provider returned error"}}';

  function mockHandler(status: number, body: string) {
    return {
      handle: async () =>
        new Response(body, { status, headers: { "content-type": "application/json" } }),
      shutdown: async () => {},
    };
  }

  async function runFallback(firstStatus: number, firstBody: string): Promise<any> {
    const handler = new FallbackHandler([
      { name: "provider-a", handler: mockHandler(firstStatus, firstBody) },
      {
        name: "provider-b",
        handler: mockHandler(200, '{"content":[{"type":"text","text":"ok"}]}'),
      },
    ]);
    const app = new Hono();
    let result: any;
    app.post("/test", async (c: any) => {
      result = await handler.handle(c, { model: "test-model" });
      return result;
    });
    const res = await app.request("/test", { method: "POST", body: "{}" });
    const text = await res.text();
    return { status: res.status, text, usedFallback: text.includes('"ok"') };
  }

  async function runFallbackForProvider(
    candidateName: string,
    firstStatus: number,
    firstBody: string
  ): Promise<any> {
    const handler = new FallbackHandler([
      { name: candidateName, handler: mockHandler(firstStatus, firstBody) },
      {
        name: "provider-b",
        handler: mockHandler(200, '{"content":[{"type":"text","text":"ok"}]}'),
      },
    ]);
    const app = new Hono();
    app.post("/test", async (c: any) => handler.handle(c, { model: "test-model" }));
    const res = await app.request("/test", { method: "POST", body: "{}" });
    const text = await res.text();
    return { status: res.status, text, usedFallback: text.includes('"ok"') };
  }

  test("401 auth error is retryable — falls through to next provider", async () => {
    const result = await runFallback(401, '{"error":"unauthorized"}');
    expect(result.usedFallback).toBe(true);
  });

  test("403 forbidden is retryable — falls through to next provider", async () => {
    const result = await runFallback(403, '{"error":"forbidden"}');
    expect(result.usedFallback).toBe(true);
  });

  test("402 payment required is retryable — falls through to next provider", async () => {
    const result = await runFallback(402, '{"error":"payment required"}');
    expect(result.usedFallback).toBe(true);
  });

  test("404 not found is retryable — falls through to next provider", async () => {
    const result = await runFallback(404, '{"error":"model not found"}');
    expect(result.usedFallback).toBe(true);
  });

  test("429 rate limit is retryable — falls through to next provider", async () => {
    const result = await runFallback(429, '{"error":"rate limited"}');
    expect(result.usedFallback).toBe(true);
  });

  test("500 with insufficient balance is retryable", async () => {
    const result = await runFallback(500, '{"error":"insufficient balance (1008)"}');
    expect(result.usedFallback).toBe(true);
  });

  test("500 generic server error is NOT retryable — stops immediately", async () => {
    const result = await runFallback(500, '{"error":"internal server error"}');
    expect(result.usedFallback).toBe(false);
  });

  test("400 with unknown model is retryable", async () => {
    const result = await runFallback(400, '{"error":"unknown model xyz"}');
    expect(result.usedFallback).toBe(true);
  });

  test("400 generic bad request is NOT retryable — stops immediately", async () => {
    const result = await runFallback(400, '{"error":"invalid request format"}');
    expect(result.usedFallback).toBe(false);
  });

  test("422 with model not available is retryable", async () => {
    const result = await runFallback(422, '{"error":"model not available"}');
    expect(result.usedFallback).toBe(true);
  });

  test("422 generic is NOT retryable", async () => {
    const result = await runFallback(422, '{"error":"unprocessable entity"}');
    expect(result.usedFallback).toBe(false);
  });

  test("400 with no healthy deployments is retryable (LiteLLM)", async () => {
    const result = await runFallback(400, '{"error":"No healthy deployment available"}');
    expect(result.usedFallback).toBe(true);
  });

  test("503 service unavailable advances to the next candidate", async () => {
    const result = await runFallback(503, '{"error":"service unavailable"}');
    expect(result.usedFallback).toBe(true);
  });

  describe("an unavailable endpoint advances the chain", () => {
    for (const status of [502, 503, 504]) {
      test(`HTTP ${status} with a plain body is retryable`, () => {
        expect(isRetryableError(status, "{}")).toBe(true);
      });
    }

    test("plain HTTP 500 remains non-retryable", () => {
      expect(isRetryableError(500, '{"error":"internal server error"}')).toBe(false);
    });

    test("OpenCode Zen Go endpoint-unavailable 503 reaches the second candidate", async () => {
      const result = await runFallbackForProvider(
        "OpenCode Zen Go",
        503,
        '{"type":"error","error":{"type":"server_error","message":"Upstream request failed: Endpoint is unavailable."}}'
      );
      expect(result.usedFallback).toBe(true);
      expect(result.status).toBe(200);
    });

    test("502 then 504 reaches 200 and calls each candidate exactly once", async () => {
      const calls = [0, 0, 0];
      const handler = new FallbackHandler(
        [502, 504, 200].map((status, index) => {
          const candidate = mockHandler(status, "{}");
          return {
            name: `provider-${index}`,
            handler: {
              ...candidate,
              handle: async () => {
                calls[index]++;
                return candidate.handle();
              },
            },
          };
        })
      );
      const app = new Hono();
      app.post("/test", async (c: any) => handler.handle(c, { model: "test-model" }));
      const response = await app.request("/test", { method: "POST", body: "{}" });
      expect(response.status).toBe(200);
      expect(calls).toEqual([1, 1, 1]);
    });

    test("all candidates unavailable: 502 then 504 ends with HTTP 503", async () => {
      const handler = new FallbackHandler([
        { name: "a", handler: mockHandler(502, "{}") },
        { name: "b", handler: mockHandler(504, "{}") },
      ]);
      const app = new Hono();
      app.post("/test", async (c: any) => handler.handle(c, { model: "test-model" }));
      const response = await app.request("/test", { method: "POST", body: "{}" });
      expect(response.status).toBe(503);
    });
  });

  describe("exhaustedChainStatus counts 502 and 504 as transient", () => {
    test("502 and 504 exhaust to 503", () => {
      expect(
        exhaustedChainStatus([
          { provider: "a", status: 502, message: "{}" },
          { provider: "b", status: 504, message: "{}" },
        ])
      ).toBe(503);
    });

    for (const upstreamStatus of [502, 504]) {
      test(`remapped 400 with upstream_status ${upstreamStatus} and 503 exhaust to 503`, () => {
        expect(
          exhaustedChainStatus([
            {
              provider: "a",
              status: 400,
              message: JSON.stringify({
                type: "error",
                error: {
                  type: "invalid_request_error",
                  message: "x",
                  upstream_status: upstreamStatus,
                },
              }),
            },
            { provider: "b", status: 503, message: "{}" },
          ])
        ).toBe(503);
      });
    }

    test("terminal 401 mixed with 502 exhausts to 400", () => {
      expect(
        exhaustedChainStatus([
          { provider: "a", status: 401, message: "{}" },
          { provider: "b", status: 502, message: "{}" },
        ])
      ).toBe(400);
    });
  });

  test("401 authentication_error (refreshAuth failure) is retryable — falls through to next provider", async () => {
    // This covers the Gemini Code Assist onboarding failure path:
    // refreshAuth() throws → ComposedHandler returns 401 → FallbackHandler tries next provider.
    const result = await runFallback(
      401,
      '{"error":{"type":"authentication_error","message":"Gemini onboarding completed but no project ID returned."}}'
    );
    expect(result.usedFallback).toBe(true);
  });

  for (const upstreamStatus of [401, 403, 402, 429] as const) {
    test(`remapped 400 with upstream_status ${upstreamStatus} is retryable`, () => {
      expect(isRetryableError(400, remappedErrorBody(upstreamStatus))).toBe(true);
    });
  }

  test("remapped 400 with non-retryable upstream_status 400 remains non-retryable", () => {
    expect(isRetryableError(400, remappedErrorBody(400))).toBe(false);
  });

  test("400 without upstream_status preserves existing bad-request classification", () => {
    expect(isRetryableError(400, remappedErrorBody())).toBe(false);
    expect(isRetryableError(400, '{"error":{"message":"unknown model example"}}')).toBe(true);
  });

  test("malformed or non-numeric upstream_status is absent and never throws", () => {
    expect(() => isRetryableError(400, "not json")).not.toThrow();
    expect(isRetryableError(400, "not json")).toBe(false);
    expect(() => isRetryableError(400, remappedErrorBody("401"))).not.toThrow();
    expect(isRetryableError(400, remappedErrorBody("401"))).toBe(false);
  });

  test("extractUpstreamStatus ignores empty, malformed, and non-numeric bodies", () => {
    expect(extractUpstreamStatus("")).toBeUndefined();
    expect(extractUpstreamStatus("not json")).toBeUndefined();
    expect(extractUpstreamStatus(remappedErrorBody("401"))).toBeUndefined();
  });

  test("upstream-status recovery is restricted to remapped HTTP 400 responses", () => {
    // Mutation guard: the 500 assertion fails if the `status === 400 ?` guard
    // is widened. A real 401 must still return true through the existing status
    // branch, even when its body carries a non-retryable upstream_status.
    expect(isRetryableError(500, remappedErrorBody(401))).toBe(false);
    expect(isRetryableError(401, remappedErrorBody(400))).toBe(true);
  });

  for (const body of [ZEN_GO_INVALID_TOOL_BODY, ZEN_GO_PROVIDER_ERROR_BODY]) {
    test(`400 Zen Go upstream rejection advances: ${body}`, async () => {
      const result = await runFallbackForProvider("OpenCode Zen Go", 400, body);
      expect(result.usedFallback).toBe(true);
    });
  }

  // Production passes the display name ("OpenCode Zen Go"), while direct unit
  // calls may use the hyphenated canonical id. Pin both so punctuation-sensitive
  // matching cannot make this rule look tested while silently disabling it live.
  test("400 Zen Go upstream rejection advances for the canonical provider id", async () => {
    const result = await runFallbackForProvider("opencode-zen-go", 400, ZEN_GO_INVALID_TOOL_BODY);
    expect(result.usedFallback).toBe(true);
  });

  test("400 upstream rejection advances for the free Zen tier", async () => {
    const result = await runFallbackForProvider("OpenCode Zen", 400, ZEN_GO_INVALID_TOOL_BODY);
    expect(result.usedFallback).toBe(true);
  });

  for (const candidateName of ["MiniMax", "OpenRouter"] as const) {
    test(`400 upstream-framed rejection does not advance for non-Zen ${candidateName}`, async () => {
      const result = await runFallbackForProvider(candidateName, 400, ZEN_GO_INVALID_TOOL_BODY);
      expect(result.usedFallback).toBe(false);
    });
  }

  test("400 ordinary payload bug does not advance for Zen", async () => {
    const result = await runFallbackForProvider(
      "OpenCode Zen Go",
      400,
      '{"error":{"message":"messages.0: invalid role"}}'
    );
    expect(result.usedFallback).toBe(false);
  });

  test("isProvider is case- and punctuation-insensitive and rejects undefined", () => {
    expect(isProvider("OpenCode Zen Go", "opencodezen")).toBe(true);
    expect(isProvider("opencode-zen-go", "opencodezen")).toBe(true);
    expect(isProvider("opencode_zen_go", "opencodezen")).toBe(true);
    expect(isProvider(undefined, "opencodezen")).toBe(false);
  });
});