A desktop coding agent with free access to frontier models.
TypeScript
import { execFile } from "node:child_process";
import { createHash, randomBytes } from "node:crypto";
import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
import { createServer } from "node:http";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import {
checkedExec,
defaultDaytonaExec,
deleteSandboxes,
execInSandbox,
} from "@omnirush/hosts";
import { progress, trackResource } from "@omnirush/world";
import type { Server, ServerResponse } from "node:http";
import { SkipError } from "./needs.ts";
import type { Place } from "./place.ts";
import type { DaytonaExec, DaytonaExecResult } from "@omnirush/hosts";
const IMAGE = "ghcr.io/berriai/litellm:v1.97.0@sha256:468c25f35f3e5ec4e414974f00deab93337b1b4d9953cabcfd3722e59415f834";
const POSTGRES_IMAGE = "postgres:16-alpine@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685";
const COMMAND_TIMEOUT_MS = 180_000;
const STARTUP_TIMEOUT_MS = 90_000;
const DATABASE_STARTUP_TIMEOUT_MS = 300_000;
const POSTGRES_STARTUP_TIMEOUT_MS = 90_000;
const EXEC_READY_TIMEOUT_MS = 180_000;
const PREVIEW_EXPIRY_SECONDS = 7_200;
const DAYTONA_PROXY_PORT = 4_000;
const DAYTONA_WITNESS_PORT = 4_001;
const DAYTONA_CONFIG = "/tmp/omnirush-litellm-config.json";
const DAYTONA_WITNESS = "/tmp/omnirush-litellm-witness.py";
const DAYTONA_LOG = "/tmp/omnirush-litellm.log";
const DAYTONA_WITNESS_LOG = "/tmp/omnirush-litellm-witness.log";
const BASE64_CHUNK_LENGTH = 8 * 1_024;
const MAX_DAYTONA_COMMAND_LENGTH = 12 * 1_024;
const steps = progress();
const DEFAULT_MAX_INPUT_TOKENS = 128_000;
const DEFAULT_MAX_OUTPUT_TOKENS = 16_384;
const DAYTONA_DOCKERFILE = `FROM ${IMAGE}
USER root
ENTRYPOINT ["/usr/bin/bash", "-lc"]
CMD ["sleep infinity"]
`;
const DAYTONA_WITNESS_SOURCE = `import argparse
import base64
import hashlib
import hmac
import json
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlsplit
MAX_BODY_BYTES = 1048576
REQUESTS = []
SEQUENCE = 0
LOCK = threading.Lock()
def fingerprint(value):
return hashlib.sha256(value.encode("utf-8")).hexdigest()
class Handler(BaseHTTPRequestHandler):
def log_message(self, format, *args):
return
def send_json(self, status, value):
payload = json.dumps(value, separators=(",", ":")).encode("utf-8")
self.send_response(status)
self.send_header("content-type", "application/json")
self.send_header("content-length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def token_fingerprint(self):
authorization = self.headers.get("authorization", "")
token = authorization[7:].strip() if authorization.startswith("Bearer ") else ""
return fingerprint(token)
def control_authorized(self):
return hmac.compare_digest(self.token_fingerprint(), OPTIONS.control_token_id)
def do_GET(self):
parsed = urlsplit(self.path)
if parsed.path == "/v1/models":
self.send_json(200, {
"object": "list",
"data": [{"id": MODEL, "object": "model", "owned_by": "omnirush-testkit"}],
})
return
if parsed.path not in ("/__omnirush_litellm/health", "/__omnirush_litellm/requests"):
self.send_json(404, {"error": {"message": "not found"}})
return
if not self.control_authorized():
self.send_json(401, {"error": "unauthorized"})
return
with LOCK:
sequence = SEQUENCE
if parsed.path == "/__omnirush_litellm/health":
self.send_json(200, {"ok": True, "sequence": sequence})
return
values = parse_qs(parsed.query).get("after", ["0"])
try:
after = int(values[0])
except ValueError:
self.send_json(400, {"error": "invalid cursor"})
return
if after < 0:
self.send_json(400, {"error": "invalid cursor"})
return
requests = [entry.copy() for entry in REQUESTS if entry["sequence"] > after]
self.send_json(200, {"sequence": sequence, "requests": requests})
def do_POST(self):
global SEQUENCE
path = urlsplit(self.path).path
if path not in ("/v1/chat/completions", "/chat/completions"):
self.send_json(404, {"error": {"message": "not found"}})
return
try:
length = int(self.headers.get("content-length", "0"))
except ValueError:
self.send_json(400, {"error": {"message": "invalid content length"}})
return
if length < 0 or length > MAX_BODY_BYTES:
self.send_json(413, {"error": {"message": "request body too large"}})
return
body_text = self.rfile.read(length).decode("utf-8", errors="replace")
try:
body = json.loads(body_text)
except json.JSONDecodeError:
body = None
model = body.get("model", "") if isinstance(body, dict) and isinstance(body.get("model"), str) else ""
token_id = self.token_fingerprint()
with LOCK:
SEQUENCE += 1
sequence = SEQUENCE
REQUESTS.append({
"sequence": sequence,
"model": model,
"tokenId": token_id,
"bodyText": body_text,
})
if not hmac.compare_digest(token_id, OPTIONS.upstream_token_id):
self.send_json(401, {"error": {"message": "unauthorized"}})
return
completion_id = "chatcmpl-omnirush-" + str(sequence)
if isinstance(body, dict) and body.get("stream") is True:
self.send_response(200)
self.send_header("content-type", "text/event-stream")
self.send_header("cache-control", "no-cache")
self.send_header("connection", "close")
self.end_headers()
chunks = [
{"id": completion_id, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"role": "assistant", "content": REPLY}, "finish_reason": None}]},
{"id": completion_id, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
]
for chunk in chunks:
self.wfile.write(("data: " + json.dumps(chunk, separators=(",", ":")) + "\\n\\n").encode("utf-8"))
self.wfile.write(b"data: [DONE]\\n\\n")
self.wfile.flush()
return
self.send_json(200, {
"id": completion_id,
"object": "chat.completion",
"model": model,
"choices": [{"index": 0, "message": {"role": "assistant", "content": REPLY}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 8, "completion_tokens": 8, "total_tokens": 16},
})
parser = argparse.ArgumentParser()
parser.add_argument("--port", type=int, required=True)
parser.add_argument("--model-b64", required=True)
parser.add_argument("--reply-b64", required=True)
parser.add_argument("--upstream-token-id", required=True)
parser.add_argument("--control-token-id", required=True)
OPTIONS = parser.parse_args()
MODEL = base64.b64decode(OPTIONS.model_b64).decode("utf-8")
REPLY = base64.b64decode(OPTIONS.reply_b64).decode("utf-8")
ThreadingHTTPServer(("0.0.0.0", OPTIONS.port), Handler).serve_forever()
`;
export interface LiteLlmUpstreamRequest {
sequence: number;
model: string;
tokenId: string;
bodyText: string;
}
export interface LiteLlmHandle extends AsyncDisposable {
baseUrl: string;
apiKey: string;
upstreamKey: string;
tokenId(key: string): string;
checkpoint(): Promise<number>;
waitForUpstreamRequest(input: { after: number; model: string; key: string; timeoutMs: number }): Promise<LiteLlmUpstreamRequest>;
upstreamRequests(input: { after: number }): Promise<LiteLlmUpstreamRequest[]>;
}
interface CommandResult {
stdout: string;
stderr: string;
}
interface WitnessState {
requests: LiteLlmUpstreamRequest[];
sequence: number;
}
interface LiteLlmSecrets {
masterKey: string;
upstreamKey: string;
controlKey: string;
}
interface HandleInput extends LiteLlmSecrets {
baseUrl: string;
controlUrl: string;
fetchImpl: typeof fetch;
redactedSecrets?: string[];
dispose(): Promise<void>;
}
function run(command: string, args: string[], timeout = COMMAND_TIMEOUT_MS): Promise<CommandResult> {
return new Promise((resolve, reject) => {
execFile(command, args, { timeout, maxBuffer: 4 * 1_024 * 1_024 }, (error, stdout, stderr) => {
if (error) {
reject(new Error(`${command} failed: ${error.message}\n${stderr}`));
return;
}
resolve({ stdout, stderr });
});
});
}
function tokenId(key: string): string {
return createHash("sha256").update(key).digest("hex");
}
function bearerToken(value: string | undefined): string {
return value?.startsWith("Bearer ") ? value.slice("Bearer ".length).trim() : "";
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function messageText(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
async function closeServer(server: Server): Promise<void> {
if (!server.listening) return;
await new Promise<void>((resolve, reject) => {
server.close((error) => error ? reject(error) : resolve());
server.closeAllConnections();
});
}
function redact(text: string, secrets: string[]): string {
return secrets.reduce((result, secret) => result.split(secret).join("[REDACTED]"), text);
}
function redactedError(error: unknown, secrets: string[]): Error {
return new Error(redact(messageText(error), secrets));
}
function writeJson(response: ServerResponse, status: number, body: unknown): void {
const text = JSON.stringify(body);
response.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(text) });
response.end(text);
}
function validCursor(value: number): number {
if (!Number.isInteger(value) || value < 0) throw new Error("LiteLLM sequence cursor must be a non-negative integer.");
return value;
}
function parseUpstreamRequest(value: unknown): LiteLlmUpstreamRequest {
if (!isRecord(value)
|| typeof value.sequence !== "number"
|| !Number.isInteger(value.sequence)
|| value.sequence < 1
|| typeof value.model !== "string"
|| typeof value.tokenId !== "string"
|| typeof value.bodyText !== "string") {
throw new Error("LiteLLM witness request response has an invalid shape.");
}
return {
sequence: value.sequence,
model: value.model,
tokenId: value.tokenId,
bodyText: value.bodyText,
};
}
function parseHealth(value: unknown): number {
if (!isRecord(value)
|| value.ok !== true
|| typeof value.sequence !== "number"
|| !Number.isInteger(value.sequence)
|| value.sequence < 0) {
throw new Error("LiteLLM witness health response has an invalid shape.");
}
return value.sequence;
}
function parseRequests(value: unknown, after: number): LiteLlmUpstreamRequest[] {
if (!isRecord(value) || !Array.isArray(value.requests)) {
throw new Error("LiteLLM witness requests response has an invalid shape.");
}
return value.requests.map(parseUpstreamRequest).filter((request) => request.sequence > after);
}
async function controlJson(fetchImpl: typeof fetch, controlUrl: string, controlKey: string, path: string): Promise<unknown> {
const response = await fetchImpl(`${controlUrl}${path}`, {
headers: { authorization: `Bearer ${controlKey}` },
signal: AbortSignal.timeout(5_000),
});
if (!response.ok) throw new Error(`LiteLLM witness control ${path} failed with HTTP ${response.status}.`);
try {
const value: unknown = await response.json();
return value;
} catch {
throw new Error(`LiteLLM witness control ${path} returned invalid JSON.`);
}
}
function makeHandle(input: HandleInput): LiteLlmHandle {
const secrets = [input.masterKey, input.upstreamKey, input.controlKey, ...(input.redactedSecrets ?? [])];
let disposed = false;
const upstreamRequests = async ({ after }: { after: number }): Promise<LiteLlmUpstreamRequest[]> => {
const cursor = validCursor(after);
try {
return parseRequests(
await controlJson(input.fetchImpl, input.controlUrl, input.controlKey, `/__omnirush_litellm/requests?after=${cursor}`),
cursor,
);
} catch (error) {
throw redactedError(error, secrets);
}
};
return {
baseUrl: input.baseUrl,
apiKey: input.masterKey,
upstreamKey: input.upstreamKey,
tokenId,
async checkpoint(): Promise<number> {
try {
return parseHealth(await controlJson(
input.fetchImpl,
input.controlUrl,
input.controlKey,
"/__omnirush_litellm/health",
));
} catch (error) {
throw redactedError(error, secrets);
}
},
upstreamRequests,
async waitForUpstreamRequest({ after, model, key, timeoutMs }) {
const cursor = validCursor(after);
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) throw new Error("LiteLLM request timeout must be positive.");
const deadline = Date.now() + timeoutMs;
const expectedTokenId = tokenId(key);
let observed: LiteLlmUpstreamRequest[] = [];
while (Date.now() < deadline) {
observed = await upstreamRequests({ after: cursor });
const found = observed.find((request) => request.model === model && request.tokenId === expectedTokenId);
if (found) return found;
await delay(100);
}
const summary = observed.map((request) => ({
sequence: request.sequence,
model: request.model,
tokenId: request.tokenId,
}));
throw new Error(`Upstream did not receive model ${model} with token fingerprint ${expectedTokenId}. Observed: ${JSON.stringify(summary)}`);
},
async [Symbol.asyncDispose]() {
if (disposed) return;
try {
await input.dispose();
disposed = true;
} catch (error) {
throw redactedError(error, secrets);
}
},
};
}
function startWitness(
modelId: string,
reply: string,
upstreamTokenId: string,
controlTokenId: string,
state: WitnessState,
): Promise<{ server: Server; port: number }> {
const server = createServer((request, response) => {
const url = new URL(request.url ?? "/", "http://127.0.0.1");
if (request.method === "GET" && url.pathname === "/v1/models") {
writeJson(response, 200, { object: "list", data: [{ id: modelId, object: "model", owned_by: "omnirush-testkit" }] });
return;
}
if (request.method === "GET"
&& (url.pathname === "/__omnirush_litellm/health" || url.pathname === "/__omnirush_litellm/requests")) {
if (tokenId(bearerToken(request.headers.authorization)) !== controlTokenId) {
writeJson(response, 401, { error: "unauthorized" });
return;
}
if (url.pathname === "/__omnirush_litellm/health") {
writeJson(response, 200, { ok: true, sequence: state.sequence });
return;
}
const afterText = url.searchParams.get("after") ?? "0";
const after = Number(afterText);
if (!Number.isInteger(after) || after < 0) {
writeJson(response, 400, { error: "invalid cursor" });
return;
}
writeJson(response, 200, {
sequence: state.sequence,
requests: state.requests.filter((entry) => entry.sequence > after),
});
return;
}
if (request.method !== "POST" || (url.pathname !== "/v1/chat/completions" && url.pathname !== "/chat/completions")) {
writeJson(response, 404, { error: { message: "not found" } });
return;
}
let bodyText = "";
request.setEncoding("utf8");
request.on("data", (chunk: string) => { bodyText += chunk; });
request.on("end", () => {
let body: unknown = null;
try { body = JSON.parse(bodyText); } catch { body = null; }
const model = isRecord(body) && typeof body.model === "string" ? body.model : "";
const requestTokenId = tokenId(bearerToken(request.headers.authorization));
state.sequence += 1;
const sequence = state.sequence;
state.requests.push({ sequence, model, tokenId: requestTokenId, bodyText });
if (requestTokenId !== upstreamTokenId) {
writeJson(response, 401, { error: { message: "unauthorized" } });
return;
}
const id = `chatcmpl-omnirush-${sequence}`;
if (isRecord(body) && body.stream === true) {
response.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-cache",
connection: "close",
});
response.write(`data: ${JSON.stringify({ id, object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: reply }, finish_reason: null }] })}\n\n`);
response.write(`data: ${JSON.stringify({ id, object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`);
response.end("data: [DONE]\n\n");
return;
}
writeJson(response, 200, {
id,
object: "chat.completion",
model,
choices: [{ index: 0, message: { role: "assistant", content: reply }, finish_reason: "stop" }],
usage: { prompt_tokens: 8, completion_tokens: 8, total_tokens: 16 },
});
});
});
return new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", () => {
const address = server.address();
if (!address || typeof address === "string") {
reject(new Error("LiteLLM upstream witness did not bind a TCP port."));
return;
}
resolve({ server, port: address.port });
});
});
}
function liteLlmConfig(
modelId: string,
apiBase: string,
masterKey: string,
upstreamKey: string,
maxInputTokens: number,
maxOutputTokens: number,
): string {
return JSON.stringify({
model_list: [{
model_name: modelId,
litellm_params: {
model: `openai/${modelId}`,
api_base: apiBase,
api_key: upstreamKey,
},
model_info: {
max_input_tokens: maxInputTokens,
max_output_tokens: maxOutputTokens,
supports_function_calling: true,
supports_vision: true,
supports_reasoning: false,
supports_response_schema: true,
supported_openai_params: ["temperature", "tools", "response_format"],
},
}],
general_settings: { master_key: masterKey },
}, null, 2);
}
function positiveTokenLimit(value: number, label: string): number {
if (!Number.isFinite(value) || value <= 0) throw new Error(`${label} must be a finite positive number.`);
return value;
}
async function mappedPort(container: string): Promise<number> {
const result = await run("docker", ["port", container, "4000/tcp"], 10_000);
const match = result.stdout.match(/:(\d+)\s*$/m);
const port = match ? Number(match[1]) : 0;
if (!Number.isInteger(port) || port <= 0) throw new Error(`docker port returned an invalid mapping: ${result.stdout.trim()}`);
return port;
}
async function mappedPostgresPort(container: string): Promise<number> {
const result = await run("docker", ["port", container, "5432/tcp"], 10_000);
const match = result.stdout.match(/:(\d+)\s*$/m);
const port = match ? Number(match[1]) : 0;
if (!Number.isInteger(port) || port <= 0) throw new Error(`docker port returned an invalid Postgres mapping: ${result.stdout.trim()}`);
return port;
}
function modelIds(value: unknown): string[] {
if (!isRecord(value) || !Array.isArray(value.data)) return [];
return value.data.flatMap((entry) => isRecord(entry) && typeof entry.id === "string" ? [entry.id] : []);
}
async function waitForLocalProxy(container: string, apiKey: string, modelId: string): Promise<number> {
const deadline = Date.now() + STARTUP_TIMEOUT_MS;
let port = 0;
let last = "proxy not queried";
while (Date.now() < deadline) {
try {
port ||= await mappedPort(container);
const response = await fetch(`http://127.0.0.1:${port}/v1/models`, {
headers: { authorization: `Bearer ${apiKey}` },
signal: AbortSignal.timeout(3_000),
});
const body: unknown = await response.json();
if (response.ok && modelIds(body).includes(modelId)) return port;
last = `HTTP ${response.status}`;
} catch (error) {
last = messageText(error);
}
await delay(500);
}
throw new Error(`LiteLLM did not expose model ${modelId} within ${STARTUP_TIMEOUT_MS}ms (last observation: ${last}).`);
}
async function waitForLocalDatabaseProxy(container: string, apiKey: string, modelId: string): Promise<number> {
const deadline = Date.now() + DATABASE_STARTUP_TIMEOUT_MS;
let port = 0;
let modelsReady = false;
let last = "proxy not queried";
while (Date.now() < deadline) {
try {
port ||= await mappedPort(container);
const response = await fetch(`http://127.0.0.1:${port}/v1/models`, {
headers: { authorization: `Bearer ${apiKey}` },
signal: AbortSignal.timeout(3_000),
});
const body: unknown = await response.json();
if (response.ok && modelIds(body).includes(modelId)) {
modelsReady = true;
break;
}
last = `models HTTP ${response.status}`;
} catch (error) {
last = messageText(error);
}
await delay(1_000);
}
if (!port || !modelsReady) {
throw new Error(`LiteLLM did not expose model ${modelId} within ${DATABASE_STARTUP_TIMEOUT_MS}ms (last observation: ${last}).`);
}
const keyManagementDeadline = Date.now() + DATABASE_STARTUP_TIMEOUT_MS;
while (Date.now() < keyManagementDeadline) {
try {
const response = await fetch(`http://127.0.0.1:${port}/key/generate`, {
method: "POST",
headers: {
authorization: `Bearer ${apiKey}`,
"content-type": "application/json",
},
body: JSON.stringify({ models: [modelId], duration: "5m" }),
signal: AbortSignal.timeout(5_000),
});
if (response.ok) return port;
last = `key management HTTP ${response.status}`;
} catch (error) {
last = messageText(error);
}
await delay(1_000);
}
throw new Error(`LiteLLM key management did not become ready within ${DATABASE_STARTUP_TIMEOUT_MS}ms (last observation: ${last}).`);
}
async function waitForPostgres(container: string): Promise<void> {
const deadline = Date.now() + POSTGRES_STARTUP_TIMEOUT_MS;
let last = "pg_isready not attempted";
while (Date.now() < deadline) {
try {
await run("docker", ["exec", container, "pg_isready", "--username", "postgres", "--dbname", "litellm"], 5_000);
return;
} catch (error) {
last = messageText(error);
}
await delay(500);
}
throw new Error(`LiteLLM Postgres did not become ready within ${POSTGRES_STARTUP_TIMEOUT_MS}ms (last observation: ${last}).`);
}
async function startLocalLiteLlm(
input: {
modelId: string;
reply: string;
database?: boolean;
maxInputTokens: number;
maxOutputTokens: number;
},
secrets: LiteLlmSecrets,
): Promise<LiteLlmHandle> {
try {
await run("docker", ["info"], 15_000);
} catch {
throw new SkipError("Docker daemon is unavailable");
}
const state: WitnessState = { requests: [], sequence: 0 };
const container = `omnirush-litellm-${randomBytes(8).toString("hex")}`;
const postgresContainer = input.database ? `omnirush-litellm-postgres-${randomBytes(8).toString("hex")}` : "";
const postgresPassword = input.database ? randomBytes(32).toString("hex") : "";
let root = "";
let witness: Server | null = null;
let gatewayStep: ReturnType<typeof steps.step> | undefined;
try {
const witnessStep = steps.step("litellm-witness", "LiteLLM witness");
const startedWitness = await startWitness(
input.modelId,
input.reply,
tokenId(secrets.upstreamKey),
tokenId(secrets.controlKey),
state,
);
await witnessStep.ok(String(startedWitness.port));
witness = startedWitness.server;
root = await realpath(await mkdtemp(join(tmpdir(), "omnirush-litellm-")));
await trackResource({ kind: "tmpdir", id: root, label: "litellm-config" });
const configPath = join(root, "config.json");
await writeFile(
configPath,
liteLlmConfig(
input.modelId,
`http://host.docker.internal:${startedWitness.port}/v1`,
secrets.masterKey,
secrets.upstreamKey,
input.maxInputTokens,
input.maxOutputTokens,
),
{ mode: 0o600 },
);
let postgresPort = 0;
if (input.database) {
const postgresStep = steps.step("litellm-postgres", "LiteLLM Postgres");
await run("docker", [
"create", "--name", postgresContainer,
"--env", `POSTGRES_PASSWORD=${postgresPassword}`,
"--env", "POSTGRES_DB=litellm",
"--publish", "127.0.0.1::5432",
POSTGRES_IMAGE,
]);
await trackResource({ kind: "docker", id: postgresContainer, label: "litellm-postgres" });
await run("docker", ["start", postgresContainer], 30_000);
postgresPort = await mappedPostgresPort(postgresContainer);
await waitForPostgres(postgresContainer);
await postgresStep.ok();
}
const createArgs = input.database
? [
"create", "--name", container,
"--add-host", "host.docker.internal:host-gateway",
"--publish", "127.0.0.1::4000",
"--env", `DATABASE_URL=postgresql://postgres:${postgresPassword}@host.docker.internal:${postgresPort}/litellm`,
IMAGE, "--config", "/app/config.json", "--port", "4000",
]
: [
"create", "--name", container,
"--add-host", "host.docker.internal:host-gateway",
"--publish", "127.0.0.1::4000",
IMAGE, "--config", "/app/config.json", "--port", "4000",
];
gatewayStep = steps.step("litellm", "LiteLLM gateway");
await run("docker", createArgs);
await trackResource({ kind: "docker", id: container, label: "litellm" });
await run("docker", ["cp", configPath, `${container}:/app/config.json`], 30_000);
await run("docker", ["start", container], 30_000);
const port = input.database
? await waitForLocalDatabaseProxy(container, secrets.masterKey, input.modelId)
: await waitForLocalProxy(container, secrets.masterKey, input.modelId);
const baseUrl = `http://127.0.0.1:${port}/v1`;
await gatewayStep.ok(baseUrl);
let placementDisposed = false;
return makeHandle({
...secrets,
baseUrl,
controlUrl: `http://127.0.0.1:${startedWitness.port}`,
fetchImpl: fetch,
redactedSecrets: input.database ? [postgresPassword] : undefined,
async dispose(): Promise<void> {
if (placementDisposed) return;
await run("docker", ["rm", "--force", "--volumes", container], 20_000).catch(() => undefined);
if (input.database) {
await run("docker", ["rm", "--force", "--volumes", postgresContainer], 20_000).catch(() => undefined);
}
await Promise.all([
rm(root, { recursive: true, force: true }),
closeServer(startedWitness.server),
]);
placementDisposed = true;
},
});
} catch (error) {
await gatewayStep?.fail(messageText(error));
const logs = await run("docker", ["logs", container], 10_000).then((result) => result.stdout + result.stderr, () => "");
const postgresLogs = input.database
? await run("docker", ["logs", postgresContainer], 10_000).then((result) => result.stdout + result.stderr, () => "")
: "";
await run("docker", ["rm", "--force", "--volumes", container], 20_000).catch(() => undefined);
if (input.database) {
await run("docker", ["rm", "--force", "--volumes", postgresContainer], 20_000).catch(() => undefined);
}
if (root) await rm(root, { recursive: true, force: true }).catch(() => undefined);
if (witness) await closeServer(witness).catch(() => undefined);
throw redactedError(
`${messageText(error)}${logs ? `\nDocker logs:\n${logs.slice(-4_000)}` : ""}${postgresLogs ? `\nPostgres logs:\n${postgresLogs.slice(-4_000)}` : ""}`,
[secrets.masterKey, secrets.upstreamKey, secrets.controlKey, ...(input.database ? [postgresPassword] : [])],
);
}
}
export function liteLlmSandboxName(): string {
return `omnirush-litellm-eval-${process.pid}-${Date.now().toString(36)}-${randomBytes(4).toString("hex")}`;
}
function uploadCommands(content: string, remotePath: string): string[] {
const source = Buffer.from(content);
const encoded = source.toString("base64");
const encodedPath = `${remotePath}.b64`;
const commands = [`: > ${encodedPath}`];
for (let offset = 0; offset < encoded.length; offset += BASE64_CHUNK_LENGTH) {
commands.push(`printf %s ${encoded.slice(offset, offset + BASE64_CHUNK_LENGTH)} >> ${encodedPath}`);
}
commands.push([
"decode_status=0",
`base64 -d ${encodedPath} > ${remotePath} || decode_status=$?`,
`actual_bytes=$(wc -c < ${remotePath})`,
`rm -f ${encodedPath}`,
`test "$decode_status" -eq 0 && test "$actual_bytes" -eq ${source.byteLength}`,
].join("; "));
return commands;
}
async function remoteExec(
exec: DaytonaExec,
sandbox: string,
script: string,
context: string,
timeoutMs = 30_000,
): Promise<DaytonaExecResult> {
const commandLength = ["exec", sandbox, "--", `bash -lc '${script}'`].join(" ").length;
if (commandLength > MAX_DAYTONA_COMMAND_LENGTH) {
throw new Error(`LiteLLM Daytona command is ${commandLength} characters; maximum is ${MAX_DAYTONA_COMMAND_LENGTH}.`);
}
return execInSandbox(exec, sandbox, script, { context, timeoutMs });
}
async function waitForExecReady(exec: DaytonaExec, sandbox: string): Promise<void> {
const deadline = Date.now() + EXEC_READY_TIMEOUT_MS;
let last = "not attempted";
while (Date.now() < deadline) {
try {
await remoteExec(exec, sandbox, "true", `LiteLLM sandbox exec-ready gate for ${sandbox}`, 15_000);
return;
} catch (error) {
last = messageText(error);
}
await delay(2_000);
}
throw new Error(`LiteLLM sandbox ${sandbox} did not become exec-ready within ${EXEC_READY_TIMEOUT_MS}ms. Last: ${last}`);
}
function firstHttpsUrl(text: string): string | null {
const match = /https:\/\/[^\s"'<>)]+/.exec(text);
return match ? match[0].replace(/[.,;:]+$/, "") : null;
}
async function previewUrl(exec: DaytonaExec, sandbox: string, port: number): Promise<string> {
const result = await checkedExec(
exec,
["preview-url", sandbox, "-p", String(port), "--expires", String(PREVIEW_EXPIRY_SECONDS)],
`LiteLLM preview URL gate for ${sandbox}:${port}`,
{ timeoutMs: 60_000 },
);
const url = firstHttpsUrl(result.stdout);
if (!url) throw new Error(`LiteLLM preview URL gate for ${sandbox}:${port} did not return an HTTPS URL.`);
return url;
}
async function waitForDaytonaReady(
fetchImpl: typeof fetch,
gatewayUrl: string,
controlUrl: string,
modelId: string,
secrets: LiteLlmSecrets,
): Promise<void> {
const deadline = Date.now() + STARTUP_TIMEOUT_MS;
let last = "not attempted";
while (Date.now() < deadline) {
let healthResponse: Response;
try {
healthResponse = await fetchImpl(`${controlUrl}/__omnirush_litellm/health`, {
headers: { authorization: `Bearer ${secrets.controlKey}` },
signal: AbortSignal.timeout(5_000),
});
} catch (error) {
last = `control: ${messageText(error)}`;
await delay(1_000);
continue;
}
if (!healthResponse.ok) {
last = `control HTTP ${healthResponse.status}`;
await delay(1_000);
continue;
}
let health: unknown;
try {
health = await healthResponse.json();
} catch {
throw new Error("LiteLLM authenticated control health returned invalid JSON.");
}
parseHealth(health);
let modelsResponse: Response;
try {
modelsResponse = await fetchImpl(`${gatewayUrl}/v1/models`, {
headers: { authorization: `Bearer ${secrets.masterKey}` },
signal: AbortSignal.timeout(5_000),
});
} catch (error) {
last = `models: ${messageText(error)}`;
await delay(1_000);
continue;
}
if (!modelsResponse.ok) {
last = `models HTTP ${modelsResponse.status}`;
await delay(1_000);
continue;
}
let models: unknown;
try {
models = await modelsResponse.json();
} catch {
throw new Error("LiteLLM /v1/models readiness gate returned invalid JSON.");
}
if (!modelIds(models).includes(modelId)) {
throw new Error(`LiteLLM /v1/models readiness gate did not include model ${modelId}.`);
}
return;
}
throw new Error(`LiteLLM Daytona readiness gate timed out after ${STARTUP_TIMEOUT_MS}ms. Last: ${last}`);
}
function launchCommand(input: { modelId: string; reply: string; upstreamTokenId: string; controlTokenId: string }): string {
const model = Buffer.from(input.modelId).toString("base64");
const reply = Buffer.from(input.reply).toString("base64");
return `/app/.venv/bin/python3 - <<PYEOF
import subprocess
witness_log = open("${DAYTONA_WITNESS_LOG}", "ab", buffering=0)
subprocess.Popen(["/app/.venv/bin/python3", "${DAYTONA_WITNESS}", "--port", "${DAYTONA_WITNESS_PORT}", "--model-b64", "${model}", "--reply-b64", "${reply}", "--upstream-token-id", "${input.upstreamTokenId}", "--control-token-id", "${input.controlTokenId}"], stdin=subprocess.DEVNULL, stdout=witness_log, stderr=subprocess.STDOUT, start_new_session=True, close_fds=True)
proxy_log = open("${DAYTONA_LOG}", "ab", buffering=0)
subprocess.Popen(["/app/.venv/bin/litellm", "--config", "${DAYTONA_CONFIG}", "--host", "0.0.0.0", "--port", "${DAYTONA_PROXY_PORT}"], stdin=subprocess.DEVNULL, stdout=proxy_log, stderr=subprocess.STDOUT, start_new_session=True, close_fds=True)
PYEOF
echo detached`;
}
async function startDaytonaLiteLlm(
input: {
modelId: string;
reply: string;
maxInputTokens: number;
maxOutputTokens: number;
daytonaExec?: DaytonaExec;
fetchImpl?: typeof fetch;
},
secrets: LiteLlmSecrets,
): Promise<LiteLlmHandle> {
const exec = input.daytonaExec ?? defaultDaytonaExec;
const fetchImpl = input.fetchImpl ?? fetch;
const sandbox = liteLlmSandboxName();
let root = "";
let createAttempted = false;
try {
root = await realpath(await mkdtemp(join(tmpdir(), "omnirush-litellm-daytona-")));
const dockerfile = join(root, "Dockerfile");
await writeFile(dockerfile, DAYTONA_DOCKERFILE, { mode: 0o600 });
createAttempted = true;
await checkedExec(
exec,
[
"create",
"--name", sandbox,
"--dockerfile", dockerfile,
"--auto-stop", "60",
"--auto-delete", "0",
"--ttl", "120",
"--public",
"--target", "us",
],
`LiteLLM sandbox creation gate for ${sandbox}`,
{ timeoutMs: 300_000 },
);
await rm(root, { recursive: true, force: true });
root = "";
await waitForExecReady(exec, sandbox);
const config = liteLlmConfig(
input.modelId,
`http://127.0.0.1:${DAYTONA_WITNESS_PORT}/v1`,
secrets.masterKey,
secrets.upstreamKey,
input.maxInputTokens,
input.maxOutputTokens,
);
const uploads = [
...uploadCommands(config, DAYTONA_CONFIG),
...uploadCommands(DAYTONA_WITNESS_SOURCE, DAYTONA_WITNESS),
];
for (const [index, command] of uploads.entries()) {
await remoteExec(exec, sandbox, command, `LiteLLM upload ${index + 1}/${uploads.length} for ${sandbox}`);
}
await remoteExec(exec, sandbox, launchCommand({
modelId: input.modelId,
reply: input.reply,
upstreamTokenId: tokenId(secrets.upstreamKey),
controlTokenId: tokenId(secrets.controlKey),
}), `LiteLLM process launch for ${sandbox}`);
const [gatewayUrl, controlUrl] = await Promise.all([
previewUrl(exec, sandbox, DAYTONA_PROXY_PORT),
previewUrl(exec, sandbox, DAYTONA_WITNESS_PORT),
]);
await waitForDaytonaReady(fetchImpl, gatewayUrl, controlUrl, input.modelId, secrets);
let placementDisposed = false;
return makeHandle({
...secrets,
baseUrl: `${gatewayUrl}/v1`,
controlUrl,
fetchImpl,
async dispose(): Promise<void> {
if (placementDisposed) return;
await deleteSandboxes([sandbox], { exec, log: () => undefined });
placementDisposed = true;
},
});
} catch (error) {
const logs = createAttempted
? await remoteExec(
exec,
sandbox,
`tail -80 ${DAYTONA_WITNESS_LOG} 2>&1 || true; tail -80 ${DAYTONA_LOG} 2>&1 || true`,
`LiteLLM failure log capture for ${sandbox}`,
).then((result) => `${result.stdout}${result.stderr}`.slice(-4_000), () => "")
: "";
if (createAttempted) {
await deleteSandboxes([sandbox], { exec, log: () => undefined }).catch(() => undefined);
}
if (root) await rm(root, { recursive: true, force: true }).catch(() => undefined);
throw redactedError(
`${messageText(error)}${logs ? `\nDaytona logs:\n${logs}` : ""}`,
[secrets.masterKey, secrets.upstreamKey, secrets.controlKey],
);
}
}
export async function liteLlm(input: {
place: Place;
modelId: string;
reply: string;
database?: boolean;
maxInputTokens?: number;
maxOutputTokens?: number;
daytonaExec?: DaytonaExec;
fetchImpl?: typeof fetch;
}): Promise<LiteLlmHandle> {
const secrets: LiteLlmSecrets = {
masterKey: `sk-omnirush-master-${randomBytes(24).toString("hex")}`,
upstreamKey: `sk-omnirush-upstream-${randomBytes(24).toString("hex")}`,
controlKey: `sk-omnirush-control-${randomBytes(24).toString("hex")}`,
};
if (input.database && input.place.kind === "daytona") {
throw new SkipError("LiteLLM database mode currently requires docker placement");
}
const normalizedInput = {
...input,
maxInputTokens: positiveTokenLimit(input.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS, "maxInputTokens"),
maxOutputTokens: positiveTokenLimit(input.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS, "maxOutputTokens"),
};
return input.place.kind === "daytona"
? startDaytonaLiteLlm(normalizedInput, secrets)
: startLocalLiteLlm(normalizedInput, secrets);
}