Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
19.1 KB
Raw Blame
export type ChatRole = "system" | "user" | "assistant"

export interface ChatMessage {
  id: string
  role: ChatRole
  content: string
  /* Reasoning models stream their thinking on a separate delta field before
     the answer. Kept apart from `content` so it can be rendered as its own
     block and excluded from what is sent back as conversation history. */
  reasoning?: string
  /* Data URIs of the pictures attached to this turn. Kept on the message rather
     than on the draft, because the transcript is resent on every later turn and
     the model has to keep seeing what it was shown. */
  images?: string[]
  /* How an assistant turn's last generation ended: the server's finish_reason;
     "aborted" / "error" when the client stopped or lost the stream; or
     "incomplete" when the stream closed without a finish_reason, which colibri
     always sends last, so its absence means the reply was cut off. Kept on
     the message so it travels with the transcript through slot switches and
     archives; never sent to the server. */
  finish?: string
  /* Both turns of a text-to-image exchange, the prompt and its picture, carry
     mode "image". They stay in the conversation on screen but are never sent to
     a text model, which would read a question followed by an empty answer. */
  mode?: "image"
  /* The picture of an image turn: the request while it runs, then the result.
     `url` appears only once the image is finished. */
  generated?: GeneratedImage
}

interface OpenAIError {
  error?: { message?: string; param?: string | null }
}

export interface SchedulerHealth {
  active: boolean | number
  capacity?: number
  queued: number
  max_queue: number
  queue_timeout_seconds: number
  admitted: number
  completed: number
  rejected: number
  timed_out: number
  cancelled: number
}

export interface TiersHealth {
  vram: number
  ram: number
  disk: number
  vram_gb: number
  ram_gb: number
}

export interface HwinfoHealth {
  cores: number
  ram_total_gb: number
  ram_avail_gb: number
  gpus: number
  vram_total_gb: number
  cpu: string
  gpu: string
}

export interface HealthResponse {
  status: string
  scheduler?: SchedulerHealth
  kv_slots?: number
  tiers?: TiersHealth
  hwinfo?: HwinfoHealth
  /* Whether a message list ending on an assistant turn is continued rather than
     answered fresh (COLI_CONTINUE_ASSISTANT). Absent on older servers. */
  continue_assistant?: boolean
}

export interface ProfileTurn {
  wall_s: number
  prompt_tokens: number
  completion_tokens: number
  expert_disk_s: number
  expert_wait_s: number
  expert_matmul_s: number
  attention_s: number
  lm_head_s: number
  forwards: number
}

export interface ProfileResponse {
  seq: number
  turns: ProfileTurn[]
}

export interface TokenUsage {
  prompt_tokens: number
  completion_tokens: number
  total_tokens: number
}

export interface StreamChatResult {
  finishReason: string | null
  usage: TokenUsage | null
  requestId: string | null
  queueWaitMs: number | null
}

export function endpoint(baseUrl: string, path: string) {
  return `${baseUrl.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`
}

export function serverEndpoint(baseUrl: string, path: string) {
  return endpoint(baseUrl.replace(/\/v1\/?$/, ""), path)
}

function headers(apiKey: string) {
  return {
    "Content-Type": "application/json",
    ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
  }
}

async function responseError(response: Response) {
  const fallback = `${response.status} ${response.statusText}`
  try {
    const body = (await response.json()) as OpenAIError
    return body.error?.message || fallback
  } catch {
    return fallback
  }
}

export interface ModelInfo {
  id: string
  /* What a model does. colibri marks an image model with "image_generation"
     (it answers /v1/images/generations and refuses chat), a language model
     with "chat" and "systemone", and a decision model with "systemone" and
     "decision" (it answers only POST /v1/systemone). */
  capabilities?: string[]
}

export async function listModelInfo(baseUrl: string, apiKey: string, signal?: AbortSignal): Promise<ModelInfo[]> {
  const response = await fetch(endpoint(baseUrl, "models"), { headers: headers(apiKey), signal })
  if (!response.ok) throw new Error(await responseError(response))
  const body = (await response.json()) as { data?: Array<{ id: string; capabilities?: unknown }> }
  return (body.data || []).map(({ id, capabilities }) => Array.isArray(capabilities)
    ? { id, capabilities: capabilities.filter((item): item is string => typeof item === "string") }
    : { id })
}

export async function listModels(baseUrl: string, apiKey: string, signal?: AbortSignal) {
  return (await listModelInfo(baseUrl, apiKey, signal)).map((model) => model.id)
}

export function generatesImages(model: ModelInfo | undefined) {
  return !!model?.capabilities?.includes("image_generation")
}

export async function getHealth(baseUrl: string, apiKey = "", signal?: AbortSignal): Promise<HealthResponse> {
  const response = await fetch(serverEndpoint(baseUrl, "health"), { headers: headers(apiKey), signal })
  if (!response.ok) throw new Error(await responseError(response))
  return (await response.json()) as HealthResponse
}

export async function getProfile(baseUrl: string, apiKey = "", signal?: AbortSignal): Promise<ProfileResponse> {
  const response = await fetch(serverEndpoint(baseUrl, "profile"), { headers: headers(apiKey), signal })
  if (!response.ok) throw new Error(await responseError(response))
  return (await response.json()) as ProfileResponse
}

export function extractSSE(buffer: string) {
  const frames = buffer.split(/\r?\n\r?\n/)
  const rest = frames.pop() || ""
  const data = frames.flatMap((frame) =>
    frame
      .split(/\r?\n/)
      .filter((line) => line.startsWith("data:"))
      .map((line) => line.slice(5).trimStart()),
  )
  return { data, rest }
}

export interface SSEEvent {
  event: string
  data: string
}

/* extractSSE with the event names kept. The image endpoint tells progress,
   previews and the result apart by the `event:` line, not by the payload, so the
   name has to survive parsing. A frame without an event line is a "message" (the
   SSE default), which is how `data: [DONE]` arrives; comment lines such as
   ": keepalive" and frames without data are dropped, as the spec says. */
export function extractSSEEvents(buffer: string): { events: SSEEvent[]; rest: string } {
  const frames = buffer.split(/\r?\n\r?\n/)
  const rest = frames.pop() || ""
  const events: SSEEvent[] = []
  for (const frame of frames) {
    let event = "message"
    const data: string[] = []
    for (const line of frame.split(/\r?\n/)) {
      if (line.startsWith("event:")) event = line.slice(6).trim() || "message"
      else if (line.startsWith("data:")) data.push(line.slice(5).replace(/^ /, ""))
    }
    if (data.length) events.push({ event, data: data.join("\n") })
  }
  return { events, rest }
}

export interface StreamChatOptions {
  baseUrl: string
  apiKey: string
  model: string
  messages: ChatMessage[]
  temperature: number
  maxTokens: number
  enableThinking: boolean
  cacheSlot?: number
  signal: AbortSignal
  onDelta: (text: string) => void
  onReasoning?: (text: string) => void
}

export async function streamChat(options: StreamChatOptions): Promise<StreamChatResult> {
  const response = await fetch(endpoint(options.baseUrl, "chat/completions"), {
    method: "POST",
    headers: headers(options.apiKey),
    signal: options.signal,
    body: JSON.stringify({
      model: options.model,
      /* A turn with pictures goes out in the content-array form the API takes;
         a plain turn stays a string, so a text-only server sees exactly what it
         saw before this existed. */
      messages: options.messages.map(({ role, content, images }) => images?.length
        ? { role, content: [
            ...(content ? [{ type: "text", text: content }] : []),
            ...images.map(url => ({ type: "image_url", image_url: { url } })),
          ] }
        : { role, content }),
      temperature: options.temperature,
      max_completion_tokens: options.maxTokens,
      enable_thinking: options.enableThinking,
      ...(options.cacheSlot === undefined ? {} : { cache_slot: options.cacheSlot }),
      stream: true,
      stream_options: { include_usage: true },
    }),
  })
  if (!response.ok) throw new Error(await responseError(response))
  if (!response.body) throw new Error("The server returned an empty stream.")

  const reader = response.body.getReader()
  const decoder = new TextDecoder()
  let buffer = ""
  let finishReason: string | null = null
  let usage: TokenUsage | null = null

  const consume = (data: string) => {
    if (data === "[DONE]") return
    const event = JSON.parse(data) as {
      choices?: Array<{ delta?: { content?: string; reasoning_content?: string }; finish_reason?: string | null }>
      usage?: TokenUsage | null
    }
    const choice = event.choices?.[0]
    const text = choice?.delta?.content
    if (text) options.onDelta(text)
    const reasoning = choice?.delta?.reasoning_content
    if (reasoning) options.onReasoning?.(reasoning)
    if (choice?.finish_reason) finishReason = choice.finish_reason
    if (event.usage) usage = event.usage
  }

  while (true) {
    const { value, done } = await reader.read()
    buffer += decoder.decode(value, { stream: !done })
    const parsed = extractSSE(buffer)
    buffer = parsed.rest
    parsed.data.forEach(consume)
    if (done) break
  }

  const queueWaitHeader = response.headers.get("x-colibri-queue-wait-ms")
  const parsedQueueWait = queueWaitHeader === null ? null : Number(queueWaitHeader)
  return {
    finishReason,
    usage,
    requestId: response.headers.get("x-request-id"),
    queueWaitMs: parsedQueueWait !== null && Number.isFinite(parsedQueueWait) ? parsedQueueWait : null,
  }
}

/* System One mode: the model does not generate, it gives a probability to
 * every allowed option. The loop (photographing the shared prefix, one read per
 * option, the normalisation) lives in the gateway, behind POST /v1/systemone,
 * the one decision API for language models and decision models alike: the
 * page's question goes as one `choice` whose labels are the options, and the
 * reply is drawn as bars. The entropy, normalised over the options, is
 * computed here from the probabilities. */
export interface DecisionChoice {
  option: string
  p: number
}

export interface DecisionResponse {
  answer: string
  entropy: number
  choices: DecisionChoice[]
  /* input_tokens: the prompt read; output_tokens: the option tokens a language
     model read (a decision model reads everything as input); nothing is generated */
  usage: { input_tokens: number; output_tokens: number }
}

export async function askSystemOne(
  baseUrl: string,
  apiKey: string,
  model: string,
  state: string,
  question: string,
  options: string[],
  signal?: AbortSignal,
): Promise<DecisionResponse> {
  const response = await fetch(endpoint(baseUrl, "systemone"), {
    method: "POST",
    headers: headers(apiKey),
    body: JSON.stringify({
      model, state,
      questions: { q: { type: "choice", instructions: question, criteria: Object.fromEntries(options.map((o) => [o, null])) } },
    }),
    signal,
  })
  if (!response.ok) throw new Error(await responseError(response))
  const body = (await response.json()) as {
    answers?: { q?: { choice?: string; probabilities?: Record<string, number> } }
    usage?: { input_tokens?: number; output_tokens?: number }
  }
  const probabilities = body.answers?.q?.probabilities ?? {}
  const choices = options
    .map((option) => ({ option, p: probabilities[option] ?? 0 }))
    .sort((a, b) => b.p - a.p)
  const entropy = choices.length > 1
    ? -choices.reduce((sum, c) => sum + (c.p > 0 ? c.p * Math.log(c.p) : 0), 0) / Math.log(choices.length)
    : 0
  return {
    answer: body.answers?.q?.choice ?? choices[0]?.option ?? "",
    entropy,
    choices,
    usage: { input_tokens: body.usage?.input_tokens ?? 0, output_tokens: body.usage?.output_tokens ?? 0 },
  }
}

/* ---- text-to-image ----------------------------------------------------------
 * POST /v1/images/generations, in the OpenAI shape plus colibri's `steps`,
 * `seed` and the `colibri` block of the answer. With `stream: true` the gateway
 * sends named SSE events: progress while the engine works, partial_image when
 * it has a preview, completed with the same object the plain request returns. */

export type ImageStage = "encode" | "denoise" | "decode"

export interface ImageProgress {
  stage: ImageStage
  step: number
  steps: number
  elapsed: number
}

export interface ImageTimings {
  encode?: number
  denoise?: number
  decode?: number
}

export interface ImageGenerationResponse {
  created: number
  data: Array<{ b64_json?: string; url?: string; revised_prompt?: string | null }>
  colibri?: { width: number; height: number; seed: number; steps: number; timings?: ImageTimings }
}

/* One picture of the conversation. The request fields are known from the
   start; url, seconds and timings arrive with the result, and seed becomes the
   one the engine reports having used. */
export interface GeneratedImage {
  prompt: string
  width: number
  height: number
  steps: number
  seed: number
  url?: string
  seconds?: number
  timings?: ImageTimings
}

export interface GenerateImageOptions {
  baseUrl: string
  apiKey: string
  model: string
  prompt: string
  width: number
  height: number
  steps: number
  seed: number
  signal: AbortSignal
  onProgress?: (progress: ImageProgress) => void
  /* A preview as a data URI, small and noisy: the latent of the step so far. */
  onPartial?: (url: string, index: number) => void
  /* Streaming was not available and the picture is being made by one plain
     request: nothing more will arrive until it is done. */
  onPlainRequest?: () => void
}

/* The stream did not work as a stream (no body, or a close before the first
   event), so the same request is worth making once without it. Not raised for
   an HTTP error, an abort or a stream that broke halfway: those would only
   start the same generation over, or hide the server's own message. */
class StreamUnavailable extends Error {}

const IMAGE_EVENT = {
  progress: "image_generation.progress",
  partial: "image_generation.partial_image",
  completed: "image_generation.completed",
}

function imageRequest(options: GenerateImageOptions, stream: boolean) {
  return {
    method: "POST",
    headers: headers(options.apiKey),
    signal: options.signal,
    body: JSON.stringify({
      model: options.model,
      prompt: options.prompt,
      size: `${options.width}x${options.height}`,
      n: 1,
      response_format: "b64_json",
      steps: options.steps,
      seed: options.seed,
      stream,
    }),
  }
}

async function imageError(response: Response) {
  const fallback = `${response.status} ${response.statusText}`
  try {
    const body = (await response.json()) as OpenAIError
    if (response.status === 400 && body.error?.param === "stream") return new StreamUnavailable(body.error.message || fallback)
    return new Error(body.error?.message || fallback)
  } catch {
    return new Error(fallback)
  }
}

function streamError(data: string) {
  try {
    const body = JSON.parse(data) as OpenAIError & { message?: string }
    return new Error(body.error?.message || body.message || data)
  } catch {
    return new Error(data)
  }
}

async function streamImage(options: GenerateImageOptions): Promise<ImageGenerationResponse> {
  const response = await fetch(endpoint(options.baseUrl, "images/generations"), imageRequest(options, true))
  if (!response.ok) throw await imageError(response)
  /* A server that ignores `stream` answers with the plain object: that is the
     result, not a failure, and asking again would generate a second picture. */
  if ((response.headers.get("content-type") || "").includes("application/json")) {
    return (await response.json()) as ImageGenerationResponse
  }
  if (!response.body) throw new StreamUnavailable("The server returned an empty stream.")

  const reader = response.body.getReader()
  const decoder = new TextDecoder()
  let buffer = ""
  let seen = false
  let finished = false
  let result: ImageGenerationResponse | null = null

  const consume = ({ event, data }: SSEEvent) => {
    seen = true
    if (data === "[DONE]") { finished = true; return }
    if (event === "error") throw streamError(data)
    const payload = JSON.parse(data) as Record<string, unknown>
    if (payload.error) throw streamError(data)
    if (event === IMAGE_EVENT.progress) options.onProgress?.(payload as unknown as ImageProgress)
    else if (event === IMAGE_EVENT.partial && typeof payload.b64_json === "string") {
      options.onPartial?.(`data:image/png;base64,${payload.b64_json}`, Number(payload.partial_image_index) || 0)
    } else if (event === IMAGE_EVENT.completed) result = payload as unknown as ImageGenerationResponse
  }

  /* [DONE] ends the read even if the connection stays open; an error event
     cancels the body so the gateway sees the client leave. */
  try {
    while (!finished) {
      const { value, done } = await reader.read()
      const before = buffer.length
      buffer += decoder.decode(value, { stream: !done })
      /* A frame ends at a blank line. A chunk that brings none cannot complete
         one, so the pending frame (the finished image is megabytes of base64)
         is not split again: re-splitting it on every chunk was quadratic. The
         three characters before the chunk catch a boundary cut in half. */
      if (done || /\r?\n\r?\n/.test(buffer.slice(Math.max(0, before - 3)))) {
        const parsed = extractSSEEvents(buffer)
        buffer = parsed.rest
        parsed.events.forEach(consume)
      }
      if (done) break
    }
  } catch (cause) {
    reader.cancel().catch(() => undefined)
    throw cause
  }
  if (finished) reader.cancel().catch(() => undefined)
  if (result) return result
  if (!seen) throw new StreamUnavailable("The server closed the stream without sending anything.")
  throw new Error("The server closed the stream before the image was finished.")
}

async function requestImage(options: GenerateImageOptions): Promise<ImageGenerationResponse> {
  const response = await fetch(endpoint(options.baseUrl, "images/generations"), imageRequest(options, false))
  if (!response.ok) throw await imageError(response)
  return (await response.json()) as ImageGenerationResponse
}

/* Streams when it can, so the page can show the stage and the previews; falls
   back to one plain request when streaming is not available. */
export async function generateImage(options: GenerateImageOptions): Promise<GeneratedImage> {
  let answer: ImageGenerationResponse
  try {
    answer = await streamImage(options)
  } catch (cause) {
    if (!(cause instanceof StreamUnavailable) || options.signal.aborted) throw cause
    options.onPlainRequest?.()
    answer = await requestImage(options)
  }
  const picture = answer.data?.[0]
  const url = picture?.b64_json ? `data:image/png;base64,${picture.b64_json}` : picture?.url
  if (!url) throw new Error("The server answered without an image.")
  const info = answer.colibri
  return {
    prompt: options.prompt,
    width: info?.width ?? options.width,
    height: info?.height ?? options.height,
    steps: info?.steps ?? options.steps,
    seed: info?.seed ?? options.seed,
    url,
    ...(info?.timings ? { timings: info.timings } : {}),
  }
}