Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

feat: add LiteLLM provider with /v1/models discovery (#220)

Prodman Devokadevb48812ccdbdf
diff --git a/src/ai/llm/factory.ts b/src/ai/llm/factory.ts
index 632cef6d..1d1a53ff 100644
--- a/src/ai/llm/factory.ts
+++ b/src/ai/llm/factory.ts
@@ -4,12 +4,17 @@
* `baseUrl` at OpenRouter, Fireworks, a LiteLLM proxy, Groq, a local server, …),
* `anthropic` speaks the native Messages API. Adding a vendor is a base URL, not
* a code change; adding a wire format is one new adapter here.
+ *
+ * `litellm` is a convenience over `openai`: same wire format, but pointed at a
+ * LiteLLM proxy by default and paired with `/v1/models` auto-discovery
+ * (see litellm.ts), so one endpoint reaches 100+ providers.
*/
import type { ChatModel } from "./types.js";
import { OpenAIChatModel } from "./openai.js";
import { AnthropicChatModel } from "./anthropic.js";
+import { LiteLLMChatModel } from "./litellm.js";
-export type ProviderKind = "openai" | "anthropic";
+export type ProviderKind = "openai" | "anthropic" | "litellm";
export interface ChatModelConfig {
provider: ProviderKind;
@@ -31,6 +36,13 @@ export function createChatModel(cfg: ChatModelConfig): ChatModel {
baseUrl: cfg.baseUrl,
headers: cfg.headers,
});
+ case "litellm":
+ return new LiteLLMChatModel({
+ apiKey: cfg.apiKey,
+ model: cfg.model,
+ baseUrl: cfg.baseUrl,
+ headers: cfg.headers,
+ });
default: {
const _exhaustive: never = cfg.provider;
throw new Error(`unknown provider: ${String(_exhaustive)}`);
diff --git a/src/ai/llm/litellm.ts b/src/ai/llm/litellm.ts
new file mode 100644
index 00000000..3ebb24b8
--- /dev/null
+++ b/src/ai/llm/litellm.ts
@@ -0,0 +1,54 @@
+/**
+ * LiteLLM transport. A LiteLLM proxy speaks the OpenAI-compatible wire format,
+ * so this reuses the OpenAI adapter's request/response translation and simply
+ * points it at the proxy. The value over a bare `openai` provider + `baseUrl`
+ * is a first-class, self-describing provider: a sensible default proxy address
+ * and `/v1/models` auto-discovery (see {@link listLiteLLMModels}), so callers
+ * can enumerate whatever models the proxy is configured to serve — reaching
+ * 100+ providers (OpenAI, Anthropic, Gemini, Bedrock, Vertex, …) through one
+ * endpoint without a per-vendor code path.
+ */
+import OpenAI from "openai";
+import { OpenAIChatModel, type OpenAIChatModelOptions } from "./openai.js";
+import { transportRetries } from "./types.js";
+
+/** Conventional local address of a self-hosted LiteLLM proxy. */
+export const DEFAULT_LITELLM_BASE_URL = "http://localhost:4000";
+
+export type LiteLLMChatModelOptions = OpenAIChatModelOptions;
+
+/**
+ * ChatModel backed by a LiteLLM proxy. Inherits every translation detail from
+ * {@link OpenAIChatModel} (the proxy is OpenAI-compatible) and only changes the
+ * defaults: the proxy base URL and a `litellm:<model>` label.
+ */
+export class LiteLLMChatModel extends OpenAIChatModel {
+ constructor(opts: LiteLLMChatModelOptions) {
+ super({
+ ...opts,
+ baseUrl: opts.baseUrl ?? DEFAULT_LITELLM_BASE_URL,
+ label: opts.label ?? `litellm:${opts.model}`,
+ });
+ }
+}
+
+/**
+ * Auto-discover the models a LiteLLM proxy is serving via `GET /v1/models`.
+ * Returns the model ids so a caller can present/validate them instead of
+ * hardcoding a model string. A stub `client` may be injected for tests.
+ */
+export async function listLiteLLMModels(opts: {
+ apiKey: string;
+ baseUrl?: string;
+ client?: OpenAI;
+}): Promise<string[]> {
+ const client =
+ opts.client ??
+ new OpenAI({
+ apiKey: opts.apiKey,
+ baseURL: opts.baseUrl ?? DEFAULT_LITELLM_BASE_URL,
+ maxRetries: transportRetries(),
+ });
+ const res = await client.models.list();
+ return res.data.map((m) => m.id);
+}
diff --git a/src/ai/providers.ts b/src/ai/providers.ts
index b03d9a57..2f743713 100644
--- a/src/ai/providers.ts
+++ b/src/ai/providers.ts
@@ -60,6 +60,8 @@ const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
export const DEFAULT_MODELS: Record<ProviderKind, string> = {
openai: "openai/gpt-4o-mini",
anthropic: "claude-sonnet-5",
+ // Provider-prefixed so the LiteLLM proxy routes it; override with GRAFT_MODEL.
+ litellm: "openai/gpt-4o-mini",
};
export const DEFAULTS = {
diff --git a/test/litellm-adapter.test.ts b/test/litellm-adapter.test.ts
new file mode 100644
index 00000000..fa7baa98
--- /dev/null
+++ b/test/litellm-adapter.test.ts
@@ -0,0 +1,60 @@
+/**
+ * Network-free tests for the LiteLLM provider. A LiteLLM proxy is
+ * OpenAI-compatible, so the adapter reuses the OpenAI translation (verified in
+ * llm-adapters.test.ts); here we assert the litellm-specific behavior: reuse of
+ * that translation, the `litellm:` label, factory wiring, and `/v1/models`
+ * auto-discovery. Stub clients mean no key and no network.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import type OpenAI from "openai";
+import { LiteLLMChatModel, listLiteLLMModels, DEFAULT_LITELLM_BASE_URL } from "../src/ai/llm/litellm.js";
+import { createChatModel } from "../src/ai/llm/factory.js";
+
+function fakeOpenAI(resp: unknown) {
+ const box: { params?: any } = {};
+ const client = {
+ chat: { completions: { create: async (params: any) => ((box.params = params), resp) } },
+ } as unknown as OpenAI;
+ return { client, box };
+}
+
+function openAiResp(over: Partial<any> = {}): any {
+ return {
+ choices: [{ message: { content: "ok", tool_calls: [] }, finish_reason: "stop" }],
+ usage: { prompt_tokens: 10, completion_tokens: 5 },
+ ...over,
+ };
+}
+
+test("litellm: reuses OpenAI-compatible translation and labels as litellm", async () => {
+ const { client, box } = fakeOpenAI(openAiResp());
+ const m = new LiteLLMChatModel({ apiKey: "x", model: "anthropic/claude-3-5-sonnet", client });
+ const res = await m.create({ messages: [{ role: "user", content: "hi" }], temperature: 0 });
+ assert.equal(box.params.model, "anthropic/claude-3-5-sonnet");
+ assert.equal(box.params.temperature, 0); // OpenAI-compatible: temperature forwarded
+ assert.equal(res.text, "ok");
+ assert.equal(m.label, "litellm:anthropic/claude-3-5-sonnet");
+});
+
+test("litellm: /v1/models auto-discovery returns the proxy's model ids", async () => {
+ const client = {
+ models: {
+ list: async () => ({
+ data: [{ id: "openai/gpt-4o-mini" }, { id: "anthropic/claude-3-5-sonnet" }],
+ }),
+ },
+ } as unknown as OpenAI;
+ const ids = await listLiteLLMModels({ apiKey: "x", client });
+ assert.deepEqual(ids, ["openai/gpt-4o-mini", "anthropic/claude-3-5-sonnet"]);
+});
+
+test("litellm: factory builds a LiteLLMChatModel for provider 'litellm'", () => {
+ const m = createChatModel({ provider: "litellm", apiKey: "x", model: "gpt-4o-mini" });
+ assert.ok(m instanceof LiteLLMChatModel);
+ assert.equal(m.label, "litellm:gpt-4o-mini");
+});
+
+test("litellm: exposes a default proxy base URL", () => {
+ assert.equal(DEFAULT_LITELLM_BASE_URL, "http://localhost:4000");
+});