Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
feat: add LiteLLM provider with /v1/models discovery (#220)
Prodman Devokadevb48812ccdbdf
diff --git a/src/ai/llm/factory.ts b/src/ai/llm/factory.tsindex 632cef6d..1d1a53ff 100644--- a/src/ai/llm/factory.ts+++ b/src/ai/llm/factory.ts@@ -4,12 +4,17 @@* `baseUrl` at OpenRouter, Fireworks, a LiteLLM proxy, Groq, a local server, …),* `anthropic` speaks the native Messages API. Adding a vendor is a base URL, not* a code change; adding a wire format is one new adapter here.+ *+ * `litellm` is a convenience over `openai`: same wire format, but pointed at a+ * LiteLLM proxy by default and paired with `/v1/models` auto-discovery+ * (see litellm.ts), so one endpoint reaches 100+ providers.*/import type { ChatModel } from "./types.js";import { OpenAIChatModel } from "./openai.js";import { AnthropicChatModel } from "./anthropic.js";+import { LiteLLMChatModel } from "./litellm.js";-export type ProviderKind = "openai" | "anthropic";+export type ProviderKind = "openai" | "anthropic" | "litellm";export interface ChatModelConfig {provider: ProviderKind;@@ -31,6 +36,13 @@ export function createChatModel(cfg: ChatModelConfig): ChatModel {baseUrl: cfg.baseUrl,headers: cfg.headers,});+ case "litellm":+ return new LiteLLMChatModel({+ apiKey: cfg.apiKey,+ model: cfg.model,+ baseUrl: cfg.baseUrl,+ headers: cfg.headers,+ });default: {const _exhaustive: never = cfg.provider;throw new Error(`unknown provider: ${String(_exhaustive)}`);diff --git a/src/ai/llm/litellm.ts b/src/ai/llm/litellm.tsnew file mode 100644index 00000000..3ebb24b8--- /dev/null+++ b/src/ai/llm/litellm.ts@@ -0,0 +1,54 @@+/**+ * LiteLLM transport. A LiteLLM proxy speaks the OpenAI-compatible wire format,+ * so this reuses the OpenAI adapter's request/response translation and simply+ * points it at the proxy. The value over a bare `openai` provider + `baseUrl`+ * is a first-class, self-describing provider: a sensible default proxy address+ * and `/v1/models` auto-discovery (see {@link listLiteLLMModels}), so callers+ * can enumerate whatever models the proxy is configured to serve — reaching+ * 100+ providers (OpenAI, Anthropic, Gemini, Bedrock, Vertex, …) through one+ * endpoint without a per-vendor code path.+ */+import OpenAI from "openai";+import { OpenAIChatModel, type OpenAIChatModelOptions } from "./openai.js";+import { transportRetries } from "./types.js";++/** Conventional local address of a self-hosted LiteLLM proxy. */+export const DEFAULT_LITELLM_BASE_URL = "http://localhost:4000";++export type LiteLLMChatModelOptions = OpenAIChatModelOptions;++/**+ * ChatModel backed by a LiteLLM proxy. Inherits every translation detail from+ * {@link OpenAIChatModel} (the proxy is OpenAI-compatible) and only changes the+ * defaults: the proxy base URL and a `litellm:<model>` label.+ */+export class LiteLLMChatModel extends OpenAIChatModel {+ constructor(opts: LiteLLMChatModelOptions) {+ super({+ ...opts,+ baseUrl: opts.baseUrl ?? DEFAULT_LITELLM_BASE_URL,+ label: opts.label ?? `litellm:${opts.model}`,+ });+ }+}++/**+ * Auto-discover the models a LiteLLM proxy is serving via `GET /v1/models`.+ * Returns the model ids so a caller can present/validate them instead of+ * hardcoding a model string. A stub `client` may be injected for tests.+ */+export async function listLiteLLMModels(opts: {+ apiKey: string;+ baseUrl?: string;+ client?: OpenAI;+}): Promise<string[]> {+ const client =+ opts.client ??+ new OpenAI({+ apiKey: opts.apiKey,+ baseURL: opts.baseUrl ?? DEFAULT_LITELLM_BASE_URL,+ maxRetries: transportRetries(),+ });+ const res = await client.models.list();+ return res.data.map((m) => m.id);+}diff --git a/src/ai/providers.ts b/src/ai/providers.tsindex b03d9a57..2f743713 100644--- a/src/ai/providers.ts+++ b/src/ai/providers.ts@@ -60,6 +60,8 @@ const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";export const DEFAULT_MODELS: Record<ProviderKind, string> = {openai: "openai/gpt-4o-mini",anthropic: "claude-sonnet-5",+ // Provider-prefixed so the LiteLLM proxy routes it; override with GRAFT_MODEL.+ litellm: "openai/gpt-4o-mini",};export const DEFAULTS = {diff --git a/test/litellm-adapter.test.ts b/test/litellm-adapter.test.tsnew file mode 100644index 00000000..fa7baa98--- /dev/null+++ b/test/litellm-adapter.test.ts@@ -0,0 +1,60 @@+/**+ * Network-free tests for the LiteLLM provider. A LiteLLM proxy is+ * OpenAI-compatible, so the adapter reuses the OpenAI translation (verified in+ * llm-adapters.test.ts); here we assert the litellm-specific behavior: reuse of+ * that translation, the `litellm:` label, factory wiring, and `/v1/models`+ * auto-discovery. Stub clients mean no key and no network.+ */+import { test } from "node:test";+import assert from "node:assert/strict";+import type OpenAI from "openai";+import { LiteLLMChatModel, listLiteLLMModels, DEFAULT_LITELLM_BASE_URL } from "../src/ai/llm/litellm.js";+import { createChatModel } from "../src/ai/llm/factory.js";++function fakeOpenAI(resp: unknown) {+ const box: { params?: any } = {};+ const client = {+ chat: { completions: { create: async (params: any) => ((box.params = params), resp) } },+ } as unknown as OpenAI;+ return { client, box };+}++function openAiResp(over: Partial<any> = {}): any {+ return {+ choices: [{ message: { content: "ok", tool_calls: [] }, finish_reason: "stop" }],+ usage: { prompt_tokens: 10, completion_tokens: 5 },+ ...over,+ };+}++test("litellm: reuses OpenAI-compatible translation and labels as litellm", async () => {+ const { client, box } = fakeOpenAI(openAiResp());+ const m = new LiteLLMChatModel({ apiKey: "x", model: "anthropic/claude-3-5-sonnet", client });+ const res = await m.create({ messages: [{ role: "user", content: "hi" }], temperature: 0 });+ assert.equal(box.params.model, "anthropic/claude-3-5-sonnet");+ assert.equal(box.params.temperature, 0); // OpenAI-compatible: temperature forwarded+ assert.equal(res.text, "ok");+ assert.equal(m.label, "litellm:anthropic/claude-3-5-sonnet");+});++test("litellm: /v1/models auto-discovery returns the proxy's model ids", async () => {+ const client = {+ models: {+ list: async () => ({+ data: [{ id: "openai/gpt-4o-mini" }, { id: "anthropic/claude-3-5-sonnet" }],+ }),+ },+ } as unknown as OpenAI;+ const ids = await listLiteLLMModels({ apiKey: "x", client });+ assert.deepEqual(ids, ["openai/gpt-4o-mini", "anthropic/claude-3-5-sonnet"]);+});++test("litellm: factory builds a LiteLLMChatModel for provider 'litellm'", () => {+ const m = createChatModel({ provider: "litellm", apiKey: "x", model: "gpt-4o-mini" });+ assert.ok(m instanceof LiteLLMChatModel);+ assert.equal(m.label, "litellm:gpt-4o-mini");+});++test("litellm: exposes a default proxy base URL", () => {+ assert.equal(DEFAULT_LITELLM_BASE_URL, "http://localhost:4000");+});