Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

9.9 KB
Raw Blame
// @vitest-environment node
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { fetchClient } from "./api";
import { ApiError } from "./client";
import { shouldRetry } from "@/contexts/ReactQueryProvider";
import {
  registerAuthHeaderNameGetter,
  registerAuthTokenGetter,
  registerBaseUrlGetter,
  registerErrorHandler,
} from "./runtime";

const jsonResponse = (status: number, body: unknown): Response =>
  new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } });

const capturingFetch = (response: Response) => {
  const requests: Request[] = [];
  const fetch = vi.fn(async (request: Request) => {
    requests.push(request);
    return response;
  });
  return { fetch, requests };
};

const spyOnRequestConstruction = () => {
  const NativeRequest = globalThis.Request;
  const inits: Array<RequestInit | Request | undefined> = [];
  class SpyingRequest extends NativeRequest {
    constructor(input: RequestInfo | URL, init?: RequestInit) {
      inits.push(init);
      super(input, init);
    }
  }
  vi.stubGlobal("Request", SpyingRequest);
  const streamBodiedInits = () =>
    inits.filter((init) => (init instanceof NativeRequest ? init.body !== null : init?.body instanceof ReadableStream));
  return { streamBodiedInits };
};

describe("typed api client middleware", () => {
  beforeEach(() => {
    registerBaseUrlGetter(() => "http://localhost:4000");
    registerAuthHeaderNameGetter(() => "Authorization");
    registerErrorHandler(() => {});
    registerAuthTokenGetter(() => null);
  });

  afterEach(() => {
    vi.restoreAllMocks();
    vi.unstubAllGlobals();
  });

  it("injects the bearer token under the registered auth header name", async () => {
    registerAuthTokenGetter(() => "sk-test");
    registerAuthHeaderNameGetter(() => "x-litellm-key");
    const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] }));

    await fetchClient.GET("/model_group/info", { fetch });

    expect(requests[0].headers.get("x-litellm-key")).toBe("Bearer sk-test");
    expect(requests[0].headers.get("Authorization")).toBeNull();
  });

  it("preserves an explicit token when the session has a different token", async () => {
    registerAuthTokenGetter(() => "session-token");
    registerAuthHeaderNameGetter(() => "x-litellm-key");
    const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] }));
    await fetchClient.GET("/model_group/info", {
      fetch,
      headers: { "x-litellm-key": "Bearer explicit-token" },
    });
    expect(requests[0].headers.get("x-litellm-key")).toBe("Bearer explicit-token");
  });

  it("asks for JSON on a bodyless GET so the next dev rewrite routes it to the proxy, not the page", async () => {
    const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] }));

    await fetchClient.GET("/lens", { fetch });

    expect(requests[0].headers.get("Accept")).toBe("application/json");
    expect(requests[0].headers.get("Content-Type")).toBeNull();
  });

  it("keeps an Accept header the caller set", async () => {
    const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] }));

    await fetchClient.GET("/lens", { fetch, headers: { Accept: "text/event-stream" } });

    expect(requests[0].headers.get("Accept")).toBe("text/event-stream");
  });

  it("reports a non-JSON success body as an ApiError naming the path that react-query will not retry", async () => {
    const onError = vi.fn();
    registerErrorHandler(onError);
    const html = new Response("<!DOCTYPE html><html></html>", {
      status: 200,
      headers: { "Content-Type": "text/html" },
    });
    const { fetch } = capturingFetch(html);

    const error = await fetchClient.GET("/lens", { fetch }).catch((e: unknown) => e);

    expect(error).toBeInstanceOf(ApiError);
    expect((error as ApiError).message).toBe("Expected JSON from /lens but the server returned text/html");
    expect((error as ApiError).status).toBe(200);
    expect(onError).toHaveBeenCalledWith("Expected JSON from /lens but the server returned text/html");
    expect(shouldRetry(0, error)).toBe(false);
  });

  it("accepts a JSON success body whose content type carries a charset", async () => {
    const response = new Response(JSON.stringify({ data: [] }), {
      status: 200,
      headers: { "Content-Type": "application/json; charset=utf-8" },
    });
    const { fetch } = capturingFetch(response);

    const { data } = await fetchClient.GET("/model_group/info", { fetch });

    expect(data).toEqual({ data: [] });
  });

  it("accepts a JSON body served without a JSON content type", async () => {
    const { fetch } = capturingFetch(new Response(JSON.stringify({ data: [] }), { status: 200 }));

    const { data } = await fetchClient.GET("/model_group/info", { fetch });

    expect(data).toEqual({ data: [] });
  });

  it("omits the auth header when no token is set", async () => {
    const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] }));

    await fetchClient.GET("/model_group/info", { fetch });

    expect(requests[0].headers.get("Authorization")).toBeNull();
  });

  it("builds the request url from the registered base url, preserving path and query", async () => {
    registerBaseUrlGetter(() => "https://proxy.example.com/");
    const { fetch, requests } = capturingFetch(jsonResponse(200, { data: [] }));

    await fetchClient.GET("/model_group/info", { fetch, params: { query: { model_group: "gpt-4o" } } });

    const url = new URL(requests[0].url);
    expect(url.origin).toBe("https://proxy.example.com");
    expect(url.pathname).toBe("/model_group/info");
    expect(url.searchParams.get("model_group")).toBe("gpt-4o");
  });

  it("sends a POST body as bytes, never as a ReadableStream (Chromium rejects stream uploads over HTTP/1.1)", async () => {
    registerAuthTokenGetter(() => "sk-test");
    const { streamBodiedInits } = spyOnRequestConstruction();
    const { fetch, requests } = capturingFetch(jsonResponse(200, { key: "sk-new" }));

    await fetchClient.POST("/key/generate", { fetch, body: { key_alias: "my-key" } });

    expect(streamBodiedInits()).toEqual([]);
    expect(requests[0].headers.get("Authorization")).toBe("Bearer sk-test");
    expect(await requests[0].text()).toBe(JSON.stringify({ key_alias: "my-key" }));
  });

  it("keeps the POST body as bytes when a different runtime base url is registered", async () => {
    registerBaseUrlGetter(() => "https://proxy.example.com");
    registerAuthTokenGetter(() => "sk-test");
    const { streamBodiedInits } = spyOnRequestConstruction();
    const { fetch, requests } = capturingFetch(jsonResponse(200, { key: "sk-new" }));

    await fetchClient.POST("/key/generate", { fetch, body: { key_alias: "my-key" } });

    expect(streamBodiedInits()).toEqual([]);
    const sent = requests[0];
    expect(new URL(sent.url).origin).toBe("https://proxy.example.com");
    expect(sent.method).toBe("POST");
    expect(sent.headers.get("Authorization")).toBe("Bearer sk-test");
    expect(sent.headers.get("Content-Type")).toBe("application/json");
    expect(await sent.text()).toBe(JSON.stringify({ key_alias: "my-key" }));
  });

  it("reads the base url on every call, so a base registered after import still takes effect", async () => {
    const requests: Request[] = [];
    const fetch = vi.fn(async (request: Request) => {
      requests.push(request);
      return jsonResponse(200, { data: [] });
    });

    registerBaseUrlGetter(() => "https://first.example.com");
    await fetchClient.GET("/model_group/info", { fetch });
    registerBaseUrlGetter(() => "https://second.example.com");
    await fetchClient.GET("/model_group/info", { fetch });

    expect(requests.map((request) => new URL(request.url).origin)).toEqual([
      "https://first.example.com",
      "https://second.example.com",
    ]);
  });

  it("forwards the caller's abort signal so an in-flight request can be cancelled", async () => {
    const controller = new AbortController();
    const seen: Request[] = [];
    const fetch = vi.fn(
      (request: Request) =>
        new Promise<Response>((_resolve, reject) => {
          seen.push(request);
          request.signal.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError")));
        }),
    );

    const pending = fetchClient.GET("/model_group/info", { fetch, signal: controller.signal });
    await vi.waitFor(() => expect(seen).toHaveLength(1));
    controller.abort();

    await expect(pending).rejects.toMatchObject({ name: "AbortError" });
    expect(seen[0].signal.aborted).toBe(true);
  }, 5000);

  it("maps a non-2xx response to an ApiError carrying status and the derived message", async () => {
    const { fetch } = capturingFetch(jsonResponse(403, { error: { message: "no access" } }));

    await expect(fetchClient.GET("/model_group/info", { fetch })).rejects.toMatchObject({
      name: "ApiError",
      status: 403,
      message: "no access",
    });
  });

  it("returns the parsed body on a successful response", async () => {
    const body = { data: [{ model_group: "gpt-4o" }] };
    const { fetch } = capturingFetch(jsonResponse(200, body));

    const { data, error } = await fetchClient.GET("/model_group/info", { fetch });

    expect(error).toBeUndefined();
    expect(data).toEqual(body);
  });

  it("reports the derived message to the registered error handler on a non-2xx response", async () => {
    const onError = vi.fn();
    registerErrorHandler(onError);
    const { fetch } = capturingFetch(jsonResponse(401, { error: { message: "Authentication Error - Expired Key" } }));

    await expect(fetchClient.GET("/model_group/info", { fetch })).rejects.toBeInstanceOf(Error);

    expect(onError).toHaveBeenCalledWith("Authentication Error - Expired Key");
  });

  it("does not call the error handler on a successful response", async () => {
    const onError = vi.fn();
    registerErrorHandler(onError);
    const { fetch } = capturingFetch(jsonResponse(200, { data: [] }));

    await fetchClient.GET("/model_group/info", { fetch });

    expect(onError).not.toHaveBeenCalled();
  });
});