Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

17.2 KB
Raw Blame
import { afterEach, beforeAll, describe, expect, test } from "bun:test";
import type { Context } from "hono";
import { GrokModelDialect } from "../adapters/grok-model-dialect.js";
import type { ProviderTransport } from "../providers/transport/types.js";
import { ComposedHandler } from "./composed-handler.js";
import { FallbackHandler } from "./fallback-handler.js";
import type { ModelHandler } from "./types.js";

/**
 * The five sites in ComposedHandler that make an outbound call which can throw
 * WITHOUT a Response, and what each one is allowed to answer.
 *
 * Four of them used to answer something untrue when the failure was the
 * network rather than the credential:
 *
 *   1. `refreshAuth` catch          → unconditional 401
 *   2. `forceRefreshAuth` catch     → unconditional 401 (wraps the retry fetch too)
 *   3. parameter-recovery re-fetch  → no `try` at all; escaped `handle()`
 *   4. `getHeaders()`               → outside every `try`; escaped `handle()`
 *   5. the primary fetch            → already classified (the control)
 *
 * 401 is not a cosmetic mislabel. `fallback-handler.ts`'s `isRetryableError`
 * reads 401 as retryable, so a network outage during a token refresh ADVANCES
 * THE CHAIN — moving a subscription user onto a per-token provider, mid-outage,
 * for a fault that had nothing to do with their credentials. An escaped throw
 * does the same thing one layer out: `fallback-handler.ts`'s catch records
 * `status: 0` and advances unconditionally, without even the cost warning.
 *
 * So the assertions that matter here are ABSENCES: not 401, and no chain
 * advance. A test that only checked the status of the first response would pass
 * against the bug.
 *
 * These tests add NO retry machinery and assert none — the retry ladder is a
 * later phase. This phase only stops the misclassification.
 */

const realFetch = globalThis.fetch;
afterEach(() => {
  globalThis.fetch = realFetch;
});

// ---------------------------------------------------------------------------
// Fixtures — PROVOKED, not hand-written
// ---------------------------------------------------------------------------

/**
 * A real Bun connect failure, taken by actually failing to connect.
 *
 * Hand-writing `{ code: "ECONNREFUSED" }` would test the classifier's table
 * rather than the runtime's behaviour, and claudish runs on Bun, whose fetch
 * uses NEITHER Node's errno names NOR `.cause` — it throws a flat TypeError
 * carrying its own `code`. Measured on this machine:
 *
 *   { name: "TypeError",
 *     message: "Unable to connect. Is the computer able to access the url?",
 *     code: "ConnectionRefused", errno: 0,
 *     path: "http://127.0.0.1:1/x", cause: undefined }
 *
 * Port 1 is privileged and unbound, so this is refused in ~1 ms with no network
 * involved — hermetic, and it cannot drift away from what Bun really throws.
 */
let CONNECT_FAILURE: unknown;
beforeAll(async () => {
  try {
    await realFetch("http://127.0.0.1:1/refused-on-purpose");
    throw new Error("127.0.0.1:1 accepted a connection — this fixture needs a closed port");
  } catch (e) {
    CONNECT_FAILURE = e;
  }
  // Guard the guard: if Bun ever stops setting a recognisable code, every test
  // below would go green for the wrong reason.
  expect((CONNECT_FAILURE as { code?: string }).code).toBe("ConnectionRefused");
});

/** The shape a transport produces when it wraps a connect failure it caught. */
function wrappedConnectFailure(message: string): Error {
  return new Error(message, { cause: CONNECT_FAILURE });
}

// ---------------------------------------------------------------------------
// Test doubles
// ---------------------------------------------------------------------------

interface TransportHooks {
  refreshAuth?: () => Promise<void>;
  forceRefreshAuth?: () => Promise<void>;
  getHeaders?: () => Promise<Record<string, string>>;
}

function makeTransport(hooks: TransportHooks = {}): ProviderTransport {
  const t: Record<string, unknown> = {
    name: "sakana",
    displayName: "Sakana Fugu",
    streamFormat: "openai-sse",
    getEndpoint: () => "https://api.sakana.example/v1/chat/completions",
    getHeaders: hooks.getHeaders ?? (async () => ({})),
  };
  if (hooks.refreshAuth) t.refreshAuth = hooks.refreshAuth;
  if (hooks.forceRefreshAuth) t.forceRefreshAuth = hooks.forceRefreshAuth;
  return t as unknown as ProviderTransport;
}

interface CapturedResponse {
  body?: { type?: string; error?: { type?: string; message?: string } };
  status?: number;
}

function makeContext(): { c: Context; captured: CapturedResponse } {
  const captured: CapturedResponse = {};
  const c = {
    // These tests exercise CLASSIFICATION and STATUS, not the retry ladder —
    // see the file header: "These tests add NO retry machinery and assert
    // none." They opt out of the ladder exactly the way `probe-live.ts:157`
    // does, with the header `composed-handler.ts:441` reads.
    //
    // Without this, a provoked connect failure enters the ladder and every
    // assertion below times out at the test runner's limit instead of failing
    // honestly — 12 tests reporting ~5000ms, which reads as a hang rather than
    // as the deliberate behaviour change it is.
    req: {
      header: (name?: string) =>
        name === undefined ? {} : name === "x-claudish-no-recovery" ? "1" : undefined,
    },
    header: () => {},
    body: (body: BodyInit | null, init?: ResponseInit) => new Response(body, init),
    json: (body: unknown, status?: number) => {
      captured.body = body as CapturedResponse["body"];
      captured.status = status;
      return new Response(JSON.stringify(body), { status: status ?? 200 });
    },
  } as unknown as Context;
  return { c, captured };
}

const PAYLOAD = {
  model: "fugu-ultra",
  max_tokens: 16,
  messages: [{ role: "user", content: "hi" }],
};

function makeHandler(transport: ProviderTransport, model = "fugu-ultra"): ComposedHandler {
  return new ComposedHandler(transport, model, model, 8080, {});
}

/**
 * Run `handler` as the FIRST candidate of a two-candidate chain and report
 * whether the SECOND was ever reached.
 *
 * This is the assertion that costs money when it is wrong: candidate 2 is the
 * metered provider a subscription user gets silently moved onto.
 */
async function runInChain(first: ModelHandler): Promise<{
  status: number;
  advanced: boolean;
  body: string;
}> {
  let advanced = false;
  const second: ModelHandler = {
    async handle() {
      advanced = true;
      return new Response(JSON.stringify({ ok: true }), { status: 200 });
    },
  } as unknown as ModelHandler;

  const chain = new FallbackHandler([
    { name: "Sakana Fugu", handler: first },
    { name: "Metered Fallback", handler: second },
  ]);
  const { c } = makeContext();
  const response = await chain.handle(c, PAYLOAD);
  return { status: response.status, advanced, body: await response.text() };
}

// ---------------------------------------------------------------------------
// Site 1 — the `refreshAuth` catch
// ---------------------------------------------------------------------------

describe("site 1: refreshAuth", () => {
  test("a connection failure answers connection_error 400, never 401", async () => {
    const handler = makeHandler(
      makeTransport({
        refreshAuth: async () => {
          // What LocalTransport / VertexOAuth now throw: their own sentence with
          // the real syscall error preserved as `cause`.
          throw wrappedConnectFailure("Cannot connect to Ollama at http://localhost:11434.");
        },
      })
    );
    const { c, captured } = makeContext();

    await handler.handle(c, PAYLOAD);

    expect(captured.status).not.toBe(401);
    expect(captured.status).toBe(400);
    expect(captured.body?.error?.type).toBe("connection_error");
  });

  test("classification happens BEFORE the terminal check, not after", async () => {
    // A transport can mark an error terminal AND have it be a connect failure —
    // ordering decides which reading wins, and only the network one is true.
    const err = wrappedConnectFailure("Sakana auth failed") as Error & { terminal?: boolean };
    err.terminal = true;
    const handler = makeHandler(
      makeTransport({
        refreshAuth: async () => {
          throw err;
        },
      })
    );
    const { c, captured } = makeContext();

    await handler.handle(c, PAYLOAD);

    expect(captured.body?.error?.type).toBe("connection_error");
    expect(captured.body?.error?.type).not.toBe("invalid_request_error");
  });

  test("THE MONEY TEST: a connection failure does not advance the fallback chain", async () => {
    const handler = makeHandler(
      makeTransport({
        refreshAuth: async () => {
          throw wrappedConnectFailure("Cannot connect to Ollama at http://localhost:11434.");
        },
      })
    );

    const { status, advanced, body } = await runInChain(handler);

    // The absence is the point: candidate 2 bills per token.
    expect(advanced).toBe(false);
    expect(status).toBe(400);
    expect(body).toContain("connection_error");
  });

  // ── Unchanged behaviour, pinned so the fix cannot over-reach ──────────────

  test("a terminal NON-connection failure still answers 400 invalid_request_error", async () => {
    const err = new Error("Entitlement revoked") as Error & { terminal?: boolean };
    err.terminal = true;
    const handler = makeHandler(
      makeTransport({
        refreshAuth: async () => {
          throw err;
        },
      })
    );
    const { c, captured } = makeContext();

    await handler.handle(c, PAYLOAD);

    expect(captured.status).toBe(400);
    expect(captured.body?.error?.type).toBe("invalid_request_error");
  });

  test("an ordinary auth failure still answers 401 and still advances the chain", async () => {
    const handler = makeHandler(
      makeTransport({
        refreshAuth: async () => {
          throw new Error("Invalid API key");
        },
      })
    );
    const { c, captured } = makeContext();
    await handler.handle(c, PAYLOAD);
    expect(captured.status).toBe(401);
    expect(captured.body?.error?.type).toBe("authentication_error");

    // And the chain still advances for it — this fix must not have turned every
    // auth failure into a dead end.
    const chainHandler = makeHandler(
      makeTransport({
        refreshAuth: async () => {
          throw new Error("Invalid API key");
        },
      })
    );
    const { advanced } = await runInChain(chainHandler);
    expect(advanced).toBe(true);
  });
});

// ---------------------------------------------------------------------------
// Site 4 — `getHeaders()`, which used to sit outside every `try`
// ---------------------------------------------------------------------------

describe("site 4: getHeaders", () => {
  test("a connection failure answers connection_error 400 instead of escaping handle()", async () => {
    const handler = makeHandler(
      makeTransport({
        // For `gk@` this is the request's FIRST network touch:
        // resolveGrokAccessToken() → fetch(auth.x.ai/oauth2/token).
        getHeaders: async () => {
          throw wrappedConnectFailure(
            "Could not reach https://auth.x.ai/oauth2/token to refresh the Grok token"
          );
        },
      })
    );
    const { c, captured } = makeContext();

    const response = await handler.handle(c, PAYLOAD);

    expect(response.status).toBe(400);
    expect(captured.body?.error?.type).toBe("connection_error");
  });

  test("a connection failure in getHeaders does not advance the fallback chain", async () => {
    const handler = makeHandler(
      makeTransport({
        getHeaders: async () => {
          throw wrappedConnectFailure("Could not reach https://auth.x.ai/oauth2/token");
        },
      })
    );

    const { advanced, status } = await runInChain(handler);

    expect(advanced).toBe(false);
    expect(status).toBe(400);
  });

  test("a NON-connection failure in getHeaders still escapes unchanged", async () => {
    // Deliberately unchanged: only the classified case is claimed here. An
    // unclassified throw keeps its existing route out of handle().
    const handler = makeHandler(
      makeTransport({
        getHeaders: async () => {
          throw new Error("no credential configured");
        },
      })
    );
    const { c } = makeContext();

    expect(handler.handle(c, PAYLOAD)).rejects.toThrow("no credential configured");
  });
});

// ---------------------------------------------------------------------------
// Site 2 — the `forceRefreshAuth` catch, which wraps the 401-retry fetch too
// ---------------------------------------------------------------------------

describe("site 2: forceRefreshAuth", () => {
  /** Upstream answers 401 once, which is what opens the forced-refresh branch. */
  function stub401() {
    globalThis.fetch = (async () =>
      new Response(JSON.stringify({ error: { message: "token expired" } }), {
        status: 401,
        headers: { "Content-Type": "application/json" },
      })) as unknown as typeof fetch;
  }

  test("a connection failure during the forced refresh is not an authentication_error", async () => {
    stub401();
    const handler = makeHandler(
      makeTransport({
        forceRefreshAuth: async () => {
          throw wrappedConnectFailure("Could not reach https://auth.x.ai/oauth2/token");
        },
      })
    );
    const { c, captured } = makeContext();

    await handler.handle(c, PAYLOAD);

    expect(captured.status).not.toBe(401);
    expect(captured.status).toBe(400);
    expect(captured.body?.error?.type).toBe("connection_error");
  });

  test("the retry fetch inside the same try is covered too", async () => {
    // The forced refresh SUCCEEDS; the network dies on the retry request that
    // follows it — inside the same `try`, and previously reported as 401.
    let call = 0;
    globalThis.fetch = (async () => {
      call++;
      if (call === 1) {
        return new Response(JSON.stringify({ error: { message: "token expired" } }), {
          status: 401,
          headers: { "Content-Type": "application/json" },
        });
      }
      throw CONNECT_FAILURE;
    }) as unknown as typeof fetch;

    const handler = makeHandler(makeTransport({ forceRefreshAuth: async () => {} }));
    const { c, captured } = makeContext();

    await handler.handle(c, PAYLOAD);

    expect(call).toBe(2);
    expect(captured.status).not.toBe(401);
    expect(captured.body?.error?.type).toBe("connection_error");
  });

  test("a connection failure during the forced refresh does not advance the chain", async () => {
    stub401();
    const handler = makeHandler(
      makeTransport({
        forceRefreshAuth: async () => {
          throw wrappedConnectFailure("Could not reach https://auth.x.ai/oauth2/token");
        },
      })
    );

    const { advanced, status } = await runInChain(handler);

    expect(advanced).toBe(false);
    expect(status).toBe(400);
  });

  test("an ordinary forced-refresh failure still answers 401", async () => {
    stub401();
    const handler = makeHandler(
      makeTransport({
        forceRefreshAuth: async () => {
          throw new Error("refresh token rejected");
        },
      })
    );
    const { c, captured } = makeContext();

    await handler.handle(c, PAYLOAD);

    expect(captured.status).toBe(401);
    expect(captured.body?.error?.type).toBe("authentication_error");
  });
});

// ---------------------------------------------------------------------------
// Site 3 — the parameter-recovery re-fetch, which had no `try` at all
// ---------------------------------------------------------------------------

describe("site 3: parameter-recovery re-fetch", () => {
  // Explicitly supply GrokModelDialect to exercise `recoverFromRejection`.
  // The id is unique to this file on purpose: the
  // dialect memoises "this model rejected reasoning_effort" at module scope,
  // and Bun shares that module across test files in a run.
  const MODEL = "grok-9-connfail-probe";
  const PARAM_REJECTION = `Model ${MODEL} does not support parameter reasoningEffort.`;
  const GROK_PAYLOAD = {
    ...PAYLOAD,
    model: MODEL,
    output_config: { effort: "high" },
  };

  test("a connection failure on the re-fetch answers connection_error, not an escape", async () => {
    let call = 0;
    globalThis.fetch = (async () => {
      call++;
      if (call === 1) {
        return new Response(JSON.stringify({ error: { message: PARAM_REJECTION } }), {
          status: 400,
          headers: { "Content-Type": "application/json" },
        });
      }
      throw CONNECT_FAILURE;
    }) as unknown as typeof fetch;

    const handler = new ComposedHandler(makeTransport(), MODEL, MODEL, 8080, {
      dialect: new GrokModelDialect(MODEL),
    });
    const { c, captured } = makeContext();

    const response = await handler.handle(c, GROK_PAYLOAD);

    // Two calls means the recovery branch really did re-fetch — without this the
    // test would pass on a payload that never carried reasoning_effort at all.
    expect(call).toBe(2);
    expect(response.status).toBe(400);
    expect(captured.body?.error?.type).toBe("connection_error");
    // The network is what is wrong NOW; re-reporting the parameter complaint
    // would send the chain hunting for a provider that accepts it while the
    // machine is offline.
    expect(captured.body?.error?.message).not.toContain("reasoningEffort");
  });
});