Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Web UI: runtime console + KV session panel, with runtime/storage libs and tests (#32)

ZacharyZcR57730f6196e1
diff --git a/web/README.md b/web/README.md
index 0e9fce55..c867d01d 100644
--- a/web/README.md
+++ b/web/README.md
@@ -16,3 +16,12 @@ Local validation:
npm test
npm run build
```
+
+The test suite stays browser-light: API requests use a mocked `fetch`, while
+runtime capability and storage behavior are covered through pure helpers. It
+checks that `/health` is resolved next to (not below) the OpenAI `/v1` prefix,
+supports both boolean and numeric `scheduler.active` responses, and sends the
+colibrì-specific `cache_slot` field only when KV-slot support was advertised.
+
+The endpoint and selected model are persisted locally. API keys are intentionally
+memory-only; startup/persistence also removes the legacy `colibri.apiKey` value.
diff --git a/web/src/App.tsx b/web/src/App.tsx
index cd4475d5..2a3f323b 100644
--- a/web/src/App.tsx
+++ b/web/src/App.tsx
@@ -19,50 +19,111 @@ import { Badge } from "@/components/ui/badge"
import { Button } from "@/components/ui/button"
import { Input } from "@/components/ui/input"
import { Textarea } from "@/components/ui/textarea"
-import { listModels, streamChat, type ChatMessage } from "@/lib/api"
+import { getHealth, listModels, streamChat, type ChatMessage, type HealthResponse, type StreamChatResult } from "@/lib/api"
+import { activeRequests, supportsCacheSlots } from "@/lib/runtime"
+import { persistPublicSettings, stored } from "@/lib/storage"
import { cn } from "@/lib/utils"
-const stored = (key: string, fallback: string) => localStorage.getItem(key) || fallback
const message = (role: ChatMessage["role"], content: string): ChatMessage => ({ id: crypto.randomUUID(), role, content })
export default function App() {
- const [baseUrl, setBaseUrl] = useState(() => stored("colibri.baseUrl", "http://127.0.0.1:8000/v1"))
- const [apiKey, setApiKey] = useState(() => stored("colibri.apiKey", ""))
+ const [baseUrl, setBaseUrl] = useState(() => stored(localStorage, "colibri.baseUrl", "http://127.0.0.1:8000/v1"))
+ const [apiKey, setApiKey] = useState("")
const [models, setModels] = useState<string[]>([])
- const [model, setModel] = useState(() => stored("colibri.model", "glm-5.2-colibri"))
+ const [model, setModel] = useState(() => stored(localStorage, "colibri.model", "glm-5.2-colibri"))
const [temperature, setTemperature] = useState(0.7)
const [maxTokens, setMaxTokens] = useState(512)
const [thinking, setThinking] = useState(false)
- const [messages, setMessages] = useState<ChatMessage[]>([])
+ const [cacheSlot, setCacheSlot] = useState(0)
+ const [conversations, setConversations] = useState<Record<number, ChatMessage[]>>({ 0: [] })
+ const [health, setHealth] = useState<HealthResponse | null>(null)
+ const [healthError, setHealthError] = useState("")
+ const [lastRun, setLastRun] = useState<StreamChatResult | null>(null)
const [draft, setDraft] = useState("")
const [loading, setLoading] = useState(false)
const [connecting, setConnecting] = useState(false)
const [connected, setConnected] = useState(false)
const [error, setError] = useState("")
const abortRef = useRef<AbortController | null>(null)
+ const probeRef = useRef<AbortController | null>(null)
const bottomRef = useRef<HTMLDivElement>(null)
+ const messages = conversations[cacheSlot] || []
+ const kvSlots = Math.max(1, health?.kv_slots || 1)
+ const active = activeRequests(health)
+ const capacity = health?.scheduler?.capacity || kvSlots
+ const failures = health?.scheduler ? health.scheduler.rejected + health.scheduler.timed_out + health.scheduler.cancelled : 0
+
+ const updateMessages = (next: ChatMessage[] | ((current: ChatMessage[]) => ChatMessage[])) =>
+ setConversations((current) => ({
+ ...current,
+ [cacheSlot]: typeof next === "function" ? next(current[cacheSlot] || []) : next,
+ }))
+
+ useEffect(() => {
+ persistPublicSettings(localStorage, baseUrl, model)
+ }, [baseUrl, model])
+
+ useEffect(() => {
+ setConnected(false)
+ setHealth(null)
+ setHealthError("")
+ }, [baseUrl, apiKey])
+
+ useEffect(() => () => {
+ probeRef.current?.abort()
+ abortRef.current?.abort()
+ }, [])
+
+ useEffect(() => {
+ if (!connected) return
+ let disposed = false
+ const poll = async () => {
+ if (document.visibilityState === "hidden") return
+ try {
+ const result = await getHealth(baseUrl, apiKey)
+ if (!disposed) { setHealth(result); setHealthError("") }
+ } catch (cause) {
+ if (!disposed) setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable")
+ }
+ }
+ const timer = window.setInterval(() => void poll(), 5000)
+ return () => { disposed = true; window.clearInterval(timer) }
+ }, [apiKey, baseUrl, connected])
useEffect(() => {
- localStorage.setItem("colibri.baseUrl", baseUrl)
- localStorage.setItem("colibri.apiKey", apiKey)
- localStorage.setItem("colibri.model", model)
- }, [apiKey, baseUrl, model])
+ if (cacheSlot >= kvSlots) setCacheSlot(0)
+ }, [cacheSlot, kvSlots])
+
+ useEffect(() => setLastRun(null), [cacheSlot])
useEffect(() => bottomRef.current?.scrollIntoView({ behavior: "smooth" }), [messages])
const connect = async () => {
+ probeRef.current?.abort()
+ const controller = new AbortController()
+ probeRef.current = controller
setConnecting(true)
setError("")
try {
- const found = await listModels(baseUrl, apiKey)
+ const found = await listModels(baseUrl, apiKey, controller.signal)
setModels(found)
if (found.length && !found.includes(model)) setModel(found[0])
setConnected(true)
+ try {
+ setHealth(await getHealth(baseUrl, apiKey, controller.signal))
+ setHealthError("")
+ } catch (cause) {
+ if (!controller.signal.aborted) {
+ setHealth(null)
+ setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable")
+ }
+ }
} catch (cause) {
+ if (controller.signal.aborted) return
setConnected(false)
setError(cause instanceof Error ? cause.message : "Could not reach the server.")
} finally {
- setConnecting(false)
+ if (probeRef.current === controller) { probeRef.current = null; setConnecting(false) }
}
}
@@ -76,12 +137,12 @@ export default function App() {
const history = [...messages, user]
setDraft("")
setError("")
- setMessages([...history, assistant])
+ updateMessages([...history, assistant])
setLoading(true)
const controller = new AbortController()
abortRef.current = controller
try {
- await streamChat({
+ const result = await streamChat({
baseUrl,
apiKey,
model,
@@ -89,19 +150,21 @@ export default function App() {
temperature,
maxTokens,
enableThinking: thinking,
+ cacheSlot: supportsCacheSlots(health) ? cacheSlot : undefined,
signal: controller.signal,
onDelta: (delta) =>
- setMessages((current) => current.map((item) =>
+ updateMessages((current) => current.map((item) =>
item.id === assistant.id ? { ...item, content: item.content + delta } : item,
)),
})
+ setLastRun(result)
setConnected(true)
} catch (cause) {
if (controller.signal.aborted) {
- setMessages((current) => current.filter((item) => item.id !== assistant.id || item.content))
+ updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content))
} else {
setError(cause instanceof Error ? cause.message : "Generation failed.")
- setMessages((current) => current.filter((item) => item.id !== assistant.id || item.content))
+ updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content))
}
} finally {
abortRef.current = null
@@ -120,20 +183,36 @@ export default function App() {
<section className="side-section">
<div className="section-title"><Link2 className="size-3.5" /> Connection</div>
<label>API endpoint<Input value={baseUrl} onChange={(event) => setBaseUrl(event.target.value)} /></label>
- <label>API key<div className="relative"><KeyRound className="field-icon" /><Input className="pl-9" type="password" value={apiKey} placeholder="optional" onChange={(event) => setApiKey(event.target.value)} /></div></label>
- <Button variant="secondary" onClick={connect} disabled={connecting}>
+ <label>API key<div className="relative"><KeyRound className="field-icon" /><Input className="pl-9" type="password" value={apiKey} placeholder="optional" onChange={(event) => setApiKey(event.target.value)} /></div><span className="field-help">Kept in memory only · sent to this endpoint</span></label>
+ <Button type="button" variant="secondary" onClick={connect} disabled={connecting}>
{connecting ? <LoaderCircle className="size-4 animate-spin" /> : <RefreshCw className="size-4" />}
Probe server
</Button>
- <div className={cn("connection-state", connected && "connected")}><span />{connected ? "Engine reachable" : "Not connected"}</div>
+ <div className={cn("connection-state", connected && "connected")} aria-live="polite"><span />{connected ? "Engine reachable" : "Not connected"}</div>
+ </section>
+
+ <section className="side-section runtime-section" aria-live="polite">
+ <div className="section-title"><Activity className="size-3.5" /> Runtime</div>
+ {health?.scheduler ? <>
+ <div className="runtime-grid">
+ <div><span>Active</span><strong>{active}<small> / {capacity}</small></strong></div>
+ <div><span>Queued</span><strong>{health.scheduler.queued}<small> / {health.scheduler.max_queue}</small></strong></div>
+ <div><span>Completed</span><strong>{health.scheduler.completed}</strong></div>
+ <div><span>Failures</span><strong>{failures}</strong></div>
+ </div>
+ <div className="runtime-foot"><span className="runtime-dot" /> Scheduler online <code>{kvSlots} KV</code></div>
+ </> : <p className="runtime-unavailable">{connected ? (healthError || "Runtime metrics unavailable") : "Probe the server to inspect runtime state."}</p>}
</section>
<section className="side-section">
<div className="section-title"><SlidersHorizontal className="size-3.5" /> Inference</div>
<label>Model<select value={model} onChange={(event) => setModel(event.target.value)}>{models.length ? models.map((id) => <option key={id}>{id}</option>) : <option>{model}</option>}</select></label>
+ {health?.kv_slots && health.kv_slots > 1 ? <label>KV session<select value={cacheSlot} onChange={(event) => setCacheSlot(Number(event.target.value))} disabled={loading}>
+ {Array.from({ length: kvSlots }, (_, slot) => <option key={slot} value={slot}>Session {slot + 1}</option>)}
+ </select><span className="field-help">Isolated context · conversation follows the selected slot</span></label> : null}
<label><span className="label-line"><span>Temperature</span><code>{temperature.toFixed(1)}</code></span><input className="range" type="range" min="0" max="2" step="0.1" value={temperature} onChange={(event) => setTemperature(Number(event.target.value))} /></label>
- <label>Max output tokens<Input type="number" min={1} max={4096} value={maxTokens} onChange={(event) => setMaxTokens(Number(event.target.value))} /></label>
- <button className={cn("toggle-row", thinking && "active")} onClick={() => setThinking((value) => !value)}>
+ <label>Max output tokens<Input type="number" min={1} max={4096} value={maxTokens} onChange={(event) => { const value = Number(event.target.value); if (Number.isFinite(value)) setMaxTokens(Math.min(4096, Math.max(1, Math.round(value)))) }} /></label>
+ <button type="button" className={cn("toggle-row", thinking && "active")} aria-pressed={thinking} onClick={() => setThinking((value) => !value)}>
<span><BrainCircuit className="size-4" /> Reasoning</span><i><b /></i>
</button>
</section>
@@ -144,7 +223,7 @@ export default function App() {
<main className="chat-panel">
<header className="topbar">
<div><span className="eyebrow">ACTIVE MODEL</span><strong>{model}</strong></div>
- <div className="top-actions"><Badge><Activity className="size-3" /> stream</Badge><Button variant="ghost" size="sm" onClick={() => setMessages([])} disabled={!messages.length || loading}><Trash2 className="size-3.5" /> Clear</Button></div>
+ <div className="top-actions">{lastRun?.queueWaitMs != null ? <Badge>queue {Math.round(lastRun.queueWaitMs)}ms</Badge> : null}<Badge><Activity className="size-3" /> slot {cacheSlot + 1}</Badge><Button variant="ghost" size="sm" onClick={() => updateMessages([])} disabled={!messages.length || loading}><Trash2 className="size-3.5" /> Clear</Button></div>
</header>
<div className="conversation">
@@ -163,7 +242,7 @@ export default function App() {
{messages.map((item) => (
<article key={item.id} className={cn("message", item.role)}>
<div className="avatar">{item.role === "user" ? "Y" : <Feather className="size-4" />}</div>
- <div><div className="message-meta">{item.role === "user" ? "You" : "colibrì"}</div><div className="message-body">{item.content || <span className="typing"><i /><i /><i /></span>}</div></div>
+ <div><div className="message-meta">{item.role === "user" ? "You" : "colibrì"}</div><div className="message-body">{item.content || <span className="typing" aria-label="Generating"><i /><i /><i /></span>}</div></div>
</article>
))}
<div ref={bottomRef} />
@@ -172,9 +251,9 @@ export default function App() {
</div>
<div className="composer-wrap">
- {error && <div className="error-banner">{error}</div>}
+ {error && <div className="error-banner" role="alert">{error}</div>}
<div className="composer">
- <Textarea value={draft} onChange={(event) => setDraft(event.target.value)} placeholder="Message colibrì…" onKeyDown={(event) => { if (event.key === "Enter" && !event.shiftKey) { event.preventDefault(); void send() } }} />
+ <Textarea value={draft} onChange={(event) => setDraft(event.target.value)} placeholder="Message colibrì…" onKeyDown={(event) => { if (event.key === "Enter" && !event.shiftKey && !event.nativeEvent.isComposing) { event.preventDefault(); void send() } }} />
<div className="composer-foot"><span><MessageSquareText className="size-3.5" /> Enter to send · Shift+Enter for newline</span>{loading ? <Button variant="destructive" size="icon" aria-label="Stop generation" onClick={() => abortRef.current?.abort()}><CircleStop className="size-4" /></Button> : <Button size="icon" aria-label="Send message" disabled={!canSend} onClick={() => void send()}><ArrowUp className="size-4" /></Button>}</div>
</div>
</div>
diff --git a/web/src/index.css b/web/src/index.css
index b0f8ea13..43fb7a7d 100644
--- a/web/src/index.css
+++ b/web/src/index.css
@@ -25,7 +25,7 @@
--primary: #4ed6a5;
--primary-foreground: #052118;
--secondary: #151c20;
- --muted-foreground: #829095;
+ --muted-foreground: #96a4a9;
--border: #202a2f;
--input: #10171a;
--destructive: #ff766f;
@@ -36,9 +36,10 @@ html, body, #root { min-height: 100%; margin: 0; }
body { background: var(--background); color: var(--foreground); }
button, input, textarea, select { font: inherit; }
button { cursor: pointer; }
+button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-visible { outline: 2px solid var(--primary); outline-offset: 2px; }
.app-shell { min-height: 100vh; display: grid; grid-template-columns: 292px minmax(0, 1fr); background: radial-gradient(circle at 72% -20%, rgba(78,214,165,.08), transparent 34%), var(--background); }
-.sidebar { position: sticky; top: 0; height: 100vh; display: flex; flex-direction: column; gap: 30px; padding: 24px 20px; border-right: 1px solid var(--border); background: rgba(8,11,13,.86); backdrop-filter: blur(18px); }
+.sidebar { position: sticky; top: 0; height: 100vh; overflow-y: auto; display: flex; flex-direction: column; gap: 24px; padding: 24px 20px; border-right: 1px solid var(--border); background: rgba(8,11,13,.86); backdrop-filter: blur(18px); }
.brand-row { display: flex; align-items: center; gap: 12px; }
.brand-mark { display: grid; place-items: center; width: 38px; height: 38px; border: 1px solid rgba(78,214,165,.3); border-radius: 12px; color: var(--primary); background: rgba(78,214,165,.08); transform: rotate(-4deg); }
.brand-row h1 { margin: 0; font-family: Georgia, serif; font-size: 22px; font-style: italic; letter-spacing: -.02em; }
@@ -54,6 +55,12 @@ button { cursor: pointer; }
.connection-state { display: flex; align-items: center; gap: 7px; color: #69767b; font-size: 10px; }
.connection-state span { width: 6px; height: 6px; border-radius: 99px; background: #4b5559; }
.connection-state.connected { color: #8abfa9; }.connection-state.connected span { background: var(--primary); box-shadow: 0 0 10px var(--primary); }
+.runtime-section { gap: 10px; }
+.runtime-grid { display: grid; grid-template-columns: 1fr 1fr; overflow: hidden; border: 1px solid var(--border); border-radius: 10px; background: var(--card); }
+.runtime-grid div { display: grid; gap: 4px; padding: 10px; border-right: 1px solid var(--border); border-bottom: 1px solid var(--border); }.runtime-grid div:nth-child(2n) { border-right: 0; }.runtime-grid div:nth-last-child(-n+2) { border-bottom: 0; }
+.runtime-grid span { color: #7f8d92; font-size: 9px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; }.runtime-grid strong { font: 600 16px ui-monospace, SFMono-Regular, monospace; }.runtime-grid small { color: var(--muted-foreground); font-size: 10px; font-weight: 500; }
+.runtime-foot { display: flex; align-items: center; gap: 6px; color: #839197; font-size: 10px; }.runtime-foot code { margin-left: auto; color: var(--primary); }.runtime-dot { width: 6px; height: 6px; border-radius: 50%; background: var(--primary); box-shadow: 0 0 9px rgba(78,214,165,.65); }
+.runtime-unavailable { margin: 0; color: var(--muted-foreground); font-size: 11px; line-height: 1.5; }.field-help { color: #718086; font-size: 9px; font-weight: 400; line-height: 1.4; }
.toggle-row { display: flex; align-items: center; justify-content: space-between; height: 42px; padding: 0 11px; border: 1px solid var(--border); border-radius: 9px; color: #a9b4b8; background: var(--input); }
.toggle-row > span { display: flex; align-items: center; gap: 8px; font-size: 11px; font-weight: 600; }.toggle-row.active { border-color: rgba(78,214,165,.35); color: var(--foreground); }
.toggle-row i { width: 30px; height: 17px; padding: 2px; border-radius: 20px; background: #293136; transition: .2s; }.toggle-row i b { display: block; width: 13px; height: 13px; border-radius: 50%; background: #78858a; transition: .2s; }.toggle-row.active i { background: rgba(78,214,165,.28); }.toggle-row.active i b { transform: translateX(13px); background: var(--primary); }
@@ -61,7 +68,7 @@ button { cursor: pointer; }
.chat-panel { min-width: 0; height: 100vh; display: grid; grid-template-rows: 72px minmax(0, 1fr) auto; }
.topbar { display: flex; align-items: center; justify-content: space-between; padding: 0 32px; border-bottom: 1px solid var(--border); }
-.topbar > div:first-child { display: grid; gap: 3px; }.topbar strong { font: 500 14px ui-monospace, SFMono-Regular, monospace; }.eyebrow { color: #5e6d72; font-size: 9px; font-weight: 800; letter-spacing: .16em; }
+.topbar > div:first-child { min-width: 0; display: grid; gap: 3px; }.topbar strong { overflow: hidden; font: 500 14px ui-monospace, SFMono-Regular, monospace; text-overflow: ellipsis; white-space: nowrap; }.eyebrow { color: #718086; font-size: 9px; font-weight: 800; letter-spacing: .16em; }
.top-actions { display: flex; align-items: center; gap: 8px; }.top-actions [data-slot=badge] { gap: 5px; }
.conversation { overflow-y: auto; }
.empty-state { width: min(680px, calc(100% - 40px)); min-height: 100%; margin: 0 auto; display: flex; flex-direction: column; align-items: center; justify-content: center; padding: 56px 0; text-align: center; }
@@ -78,3 +85,4 @@ button { cursor: pointer; }
.app-shell { grid-template-columns: 1fr; }.sidebar { position: static; width: 100%; height: auto; display: grid; grid-template-columns: 1fr 1fr; border-right: 0; border-bottom: 1px solid var(--border); }.brand-row, .sidebar-foot { grid-column: 1/-1; }.chat-panel { height: 100svh; }.suggestions { grid-template-columns: 1fr; }.topbar, .composer-wrap { padding-left: 16px; padding-right: 16px; }
}
@media (max-width: 560px) { .sidebar { grid-template-columns: 1fr; }.brand-row, .sidebar-foot { grid-column: auto; }.empty-state h2 { font-size: 36px; }.message-list { width: calc(100% - 28px); }.composer-foot > span { display: none; } }
+@media (prefers-reduced-motion: reduce) { *, *::before, *::after { scroll-behavior: auto !important; animation-duration: .01ms !important; animation-iteration-count: 1 !important; transition-duration: .01ms !important; } }
diff --git a/web/src/lib/api.test.ts b/web/src/lib/api.test.ts
index fa0cb878..29e7eded 100644
--- a/web/src/lib/api.test.ts
+++ b/web/src/lib/api.test.ts
@@ -1,6 +1,8 @@
-import { describe, expect, it } from "vitest"
+import { afterEach, describe, expect, it, vi } from "vitest"
-import { extractSSE } from "./api"
+import { extractSSE, getHealth, serverEndpoint, streamChat } from "./api"
+
+afterEach(() => vi.unstubAllGlobals())
describe("extractSSE", () => {
it("keeps an incomplete frame for the next network chunk", () => {
@@ -15,3 +17,55 @@ describe("extractSSE", () => {
expect(parsed.rest).toBe("")
})
})
+
+describe("runtime API", () => {
+ it.each([
+ ["http://127.0.0.1:8000/v1", "http://127.0.0.1:8000/health"],
+ ["https://example.test/api/v1/", "https://example.test/api/health"],
+ ["https://example.test/api", "https://example.test/api/health"],
+ ])("resolves the health endpoint outside the OpenAI v1 prefix", (baseUrl, expected) => {
+ expect(serverEndpoint(baseUrl, "health")).toBe(expected)
+ })
+
+ it("requests health with the configured bearer credential", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(new Response(JSON.stringify({ status: "ok", scheduler: { active: true } })))
+ vi.stubGlobal("fetch", fetchMock)
+
+ await expect(getHealth("http://localhost:8000/v1/", "secret")).resolves.toMatchObject({ status: "ok" })
+ expect(fetchMock).toHaveBeenCalledWith("http://localhost:8000/health", expect.objectContaining({
+ headers: expect.objectContaining({ Authorization: "Bearer secret" }),
+ }))
+ })
+})
+
+describe("chat request extensions", () => {
+ const completedStream = () => new Response("data: [DONE]\n\n", {
+ headers: { "content-type": "text/event-stream" },
+ })
+
+ async function requestBody(cacheSlot?: number) {
+ const fetchMock = vi.fn().mockResolvedValue(completedStream())
+ vi.stubGlobal("fetch", fetchMock)
+ await streamChat({
+ baseUrl: "http://localhost:8000/v1",
+ apiKey: "",
+ model: "test-model",
+ messages: [],
+ temperature: 0,
+ maxTokens: 8,
+ enableThinking: false,
+ cacheSlot,
+ signal: new AbortController().signal,
+ onDelta: () => undefined,
+ })
+ return JSON.parse(fetchMock.mock.calls[0][1].body as string) as Record<string, unknown>
+ }
+
+ it("omits cache_slot for a generic OpenAI-compatible backend", async () => {
+ expect(await requestBody()).not.toHaveProperty("cache_slot")
+ })
+
+ it("sends cache_slot zero when colibrì advertises KV slots", async () => {
+ expect(await requestBody(0)).toMatchObject({ cache_slot: 0 })
+ })
+})
diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts
index 2d722f3e..1d5a854a 100644
--- a/web/src/lib/api.ts
+++ b/web/src/lib/api.ts
@@ -10,10 +10,46 @@ interface OpenAIError {
error?: { message?: string }
}
-function endpoint(baseUrl: string, path: string) {
+export interface SchedulerHealth {
+ active: boolean | number
+ capacity?: number
+ queued: number
+ max_queue: number
+ queue_timeout_seconds: number
+ admitted: number
+ completed: number
+ rejected: number
+ timed_out: number
+ cancelled: number
+}
+
+export interface HealthResponse {
+ status: string
+ scheduler?: SchedulerHealth
+ kv_slots?: number
+}
+
+export interface TokenUsage {
+ prompt_tokens: number
+ completion_tokens: number
+ total_tokens: number
+}
+
+export interface StreamChatResult {
+ finishReason: string | null
+ usage: TokenUsage | null
+ requestId: string | null
+ queueWaitMs: number | null
+}
+
+export function endpoint(baseUrl: string, path: string) {
return `${baseUrl.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`
}
+export function serverEndpoint(baseUrl: string, path: string) {
+ return endpoint(baseUrl.replace(/\/v1\/?$/, ""), path)
+}
+
function headers(apiKey: string) {
return {
"Content-Type": "application/json",
@@ -38,6 +74,12 @@ export async function listModels(baseUrl: string, apiKey: string, signal?: Abort
return (body.data || []).map((model) => model.id)
}
+export async function getHealth(baseUrl: string, apiKey = "", signal?: AbortSignal): Promise<HealthResponse> {
+ const response = await fetch(serverEndpoint(baseUrl, "health"), { headers: headers(apiKey), signal })
+ if (!response.ok) throw new Error(await responseError(response))
+ return (await response.json()) as HealthResponse
+}
+
export function extractSSE(buffer: string) {
const frames = buffer.split(/\r?\n\r?\n/)
const rest = frames.pop() || ""
@@ -50,7 +92,7 @@ export function extractSSE(buffer: string) {
return { data, rest }
}
-interface StreamChatOptions {
+export interface StreamChatOptions {
baseUrl: string
apiKey: string
model: string
@@ -58,11 +100,12 @@ interface StreamChatOptions {
temperature: number
maxTokens: number
enableThinking: boolean
+ cacheSlot?: number
signal: AbortSignal
onDelta: (text: string) => void
}
-export async function streamChat(options: StreamChatOptions) {
+export async function streamChat(options: StreamChatOptions): Promise<StreamChatResult> {
const response = await fetch(endpoint(options.baseUrl, "chat/completions"), {
method: "POST",
headers: headers(options.apiKey),
@@ -73,6 +116,7 @@ export async function streamChat(options: StreamChatOptions) {
temperature: options.temperature,
max_completion_tokens: options.maxTokens,
enable_thinking: options.enableThinking,
+ ...(options.cacheSlot === undefined ? {} : { cache_slot: options.cacheSlot }),
stream: true,
stream_options: { include_usage: true },
}),
@@ -83,14 +127,20 @@ export async function streamChat(options: StreamChatOptions) {
const reader = response.body.getReader()
const decoder = new TextDecoder()
let buffer = ""
+ let finishReason: string | null = null
+ let usage: TokenUsage | null = null
const consume = (data: string) => {
if (data === "[DONE]") return
const event = JSON.parse(data) as {
- choices?: Array<{ delta?: { content?: string } }>
+ choices?: Array<{ delta?: { content?: string }; finish_reason?: string | null }>
+ usage?: TokenUsage | null
}
- const text = event.choices?.[0]?.delta?.content
+ const choice = event.choices?.[0]
+ const text = choice?.delta?.content
if (text) options.onDelta(text)
+ if (choice?.finish_reason) finishReason = choice.finish_reason
+ if (event.usage) usage = event.usage
}
while (true) {
@@ -101,4 +151,13 @@ export async function streamChat(options: StreamChatOptions) {
parsed.data.forEach(consume)
if (done) break
}
+
+ const queueWaitHeader = response.headers.get("x-colibri-queue-wait-ms")
+ const parsedQueueWait = queueWaitHeader === null ? null : Number(queueWaitHeader)
+ return {
+ finishReason,
+ usage,
+ requestId: response.headers.get("x-request-id"),
+ queueWaitMs: parsedQueueWait !== null && Number.isFinite(parsedQueueWait) ? parsedQueueWait : null,
+ }
}
diff --git a/web/src/lib/runtime.test.ts b/web/src/lib/runtime.test.ts
new file mode 100644
index 00000000..8e084bea
--- /dev/null
+++ b/web/src/lib/runtime.test.ts
@@ -0,0 +1,41 @@
+import { describe, expect, it } from "vitest"
+
+import type { HealthResponse } from "./api"
+import { activeRequests, supportsCacheSlots } from "./runtime"
+
+const healthWithActive = (active: boolean | number): HealthResponse => ({
+ status: "ok",
+ scheduler: {
+ active,
+ queued: 0,
+ max_queue: 0,
+ queue_timeout_seconds: 0,
+ admitted: 0,
+ completed: 0,
+ rejected: 0,
+ timed_out: 0,
+ cancelled: 0,
+ },
+})
+
+describe("runtime capability normalization", () => {
+ it.each([
+ [true, 1],
+ [false, 0],
+ [3, 3],
+ [0, 0],
+ ] as const)("normalizes scheduler.active %s to %d", (active, expected) => {
+ expect(activeRequests(healthWithActive(active))).toBe(expected)
+ })
+
+ it("treats missing scheduler metrics as idle", () => {
+ expect(activeRequests({ status: "ok" })).toBe(0)
+ expect(activeRequests(null)).toBe(0)
+ })
+
+ it("only enables cache slots when the health response advertises them", () => {
+ expect(supportsCacheSlots({ status: "ok", kv_slots: 4 })).toBe(true)
+ expect(supportsCacheSlots({ status: "ok" })).toBe(false)
+ expect(supportsCacheSlots(null)).toBe(false)
+ })
+})
diff --git a/web/src/lib/runtime.ts b/web/src/lib/runtime.ts
new file mode 100644
index 00000000..ad229c17
--- /dev/null
+++ b/web/src/lib/runtime.ts
@@ -0,0 +1,9 @@
+import type { HealthResponse } from "./api"
+
+export function activeRequests(health: HealthResponse | null): number {
+ return Number(health?.scheduler?.active || 0)
+}
+
+export function supportsCacheSlots(health: HealthResponse | null): boolean {
+ return typeof health?.kv_slots === "number" && health.kv_slots > 0
+}
diff --git a/web/src/lib/storage.test.ts b/web/src/lib/storage.test.ts
new file mode 100644
index 00000000..1d256d78
--- /dev/null
+++ b/web/src/lib/storage.test.ts
@@ -0,0 +1,35 @@
+import { describe, expect, it, vi } from "vitest"
+
+import { persistPublicSettings, stored, type StringStorage } from "./storage"
+
+function memoryStorage(initial: Record<string, string> = {}): StringStorage & { values: Map<string, string> } {
+ const values = new Map(Object.entries(initial))
+ return {
+ values,
+ getItem: (key) => values.get(key) ?? null,
+ setItem: (key, value) => { values.set(key, value) },
+ removeItem: (key) => { values.delete(key) },
+ }
+}
+
+describe("browser settings persistence", () => {
+ it("persists endpoint and model but removes legacy API credentials", () => {
+ const storage = memoryStorage({ "colibri.apiKey": "legacy-secret" })
+ persistPublicSettings(storage, "https://localhost/v1", "test-model")
+ expect(Object.fromEntries(storage.values)).toEqual({
+ "colibri.baseUrl": "https://localhost/v1",
+ "colibri.model": "test-model",
+ })
+ })
+
+ it("does not attempt to write an API key", () => {
+ const storage = memoryStorage()
+ const setItem = vi.spyOn(storage, "setItem")
+ persistPublicSettings(storage, "http://localhost/v1", "test-model")
+ expect(setItem).not.toHaveBeenCalledWith("colibri.apiKey", expect.anything())
+ })
+
+ it("uses a fallback when storage access is unavailable", () => {
+ expect(stored({ getItem: () => { throw new Error("denied") } }, "key", "fallback")).toBe("fallback")
+ })
+})
diff --git a/web/src/lib/storage.ts b/web/src/lib/storage.ts
new file mode 100644
index 00000000..a0e6ed41
--- /dev/null
+++ b/web/src/lib/storage.ts
@@ -0,0 +1,19 @@
+export interface StringStorage {
+ getItem(key: string): string | null
+ setItem(key: string, value: string): void
+ removeItem(key: string): void
+}
+
+export function stored(storage: Pick<StringStorage, "getItem">, key: string, fallback: string) {
+ try { return storage.getItem(key) || fallback } catch { return fallback }
+}
+
+export function persistPublicSettings(storage: StringStorage, baseUrl: string, model: string) {
+ try {
+ storage.setItem("colibri.baseUrl", baseUrl)
+ storage.setItem("colibri.model", model)
+ // API credentials intentionally remain memory-only. Remove values left by
+ // older web releases whenever public settings are persisted.
+ storage.removeItem("colibri.apiKey")
+ } catch { /* restricted storage mode */ }
+}