Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
docs: add Mintlify API reference
Hamed Rabah59babb88e414
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonnew file mode 100644index 00000000..d4c179be--- /dev/null+++ b/docs/api-reference/openapi.json@@ -0,0 +1,415 @@+{+ "openapi": "3.1.0",+ "info": {+ "title": "colibrì HTTP API",+ "version": "1.0.0",+ "description": "OpenAI- and Anthropic-compatible inference endpoints plus local scheduler and dashboard telemetry."+ },+ "servers": [+ {+ "url": "http://127.0.0.1:8000",+ "description": "Default local server"+ }+ ],+ "tags": [+ { "name": "Inference", "description": "Generate text through OpenAI- or Anthropic-compatible request shapes." },+ { "name": "Models", "description": "Inspect the model loaded by the persistent server." },+ { "name": "Operations", "description": "Inspect liveness, queue state, hardware, routing, and per-turn telemetry." }+ ],+ "paths": {+ "/v1/chat/completions": {+ "post": {+ "tags": ["Inference"],+ "summary": "Create a chat completion",+ "description": "Generate a response from a list of OpenAI-compatible messages. Set `stream` to receive server-sent events.",+ "operationId": "createChatCompletion",+ "security": [{ "bearerAuth": [] }, { "apiKeyAuth": [] }],+ "requestBody": {+ "required": true,+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/ChatCompletionRequest" }+ }+ }+ },+ "responses": {+ "200": {+ "description": "A JSON completion or an SSE stream when `stream` is true.",+ "headers": {+ "x-colibri-queue-wait-ms": {+ "description": "Milliseconds spent waiting for the engine admission queue.",+ "schema": { "type": "number" }+ }+ },+ "content": {+ "application/json": { "schema": { "$ref": "#/components/schemas/ChatCompletion" } },+ "text/event-stream": { "schema": { "type": "string" } }+ }+ },+ "400": { "$ref": "#/components/responses/BadRequest" },+ "401": { "$ref": "#/components/responses/Unauthorized" },+ "429": { "$ref": "#/components/responses/QueueFull" }+ }+ }+ },+ "/v1/completions": {+ "post": {+ "tags": ["Inference"],+ "summary": "Create a legacy completion",+ "description": "Generate text from one prompt using the legacy OpenAI completions shape.",+ "operationId": "createCompletion",+ "security": [{ "bearerAuth": [] }, { "apiKeyAuth": [] }],+ "requestBody": {+ "required": true,+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/CompletionRequest" }+ }+ }+ },+ "responses": {+ "200": {+ "description": "A JSON completion or an SSE stream when `stream` is true.",+ "content": {+ "application/json": { "schema": { "$ref": "#/components/schemas/Completion" } },+ "text/event-stream": { "schema": { "type": "string" } }+ }+ },+ "400": { "$ref": "#/components/responses/BadRequest" },+ "401": { "$ref": "#/components/responses/Unauthorized" },+ "429": { "$ref": "#/components/responses/QueueFull" }+ }+ }+ },+ "/v1/messages": {+ "post": {+ "tags": ["Inference"],+ "summary": "Create an Anthropic message",+ "description": "Generate a response with the Anthropic Messages request shape. Unsupported fields are rejected explicitly.",+ "operationId": "createMessage",+ "security": [{ "apiKeyAuth": [] }, { "bearerAuth": [] }],+ "requestBody": {+ "required": true,+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/AnthropicMessageRequest" }+ }+ }+ },+ "responses": {+ "200": {+ "description": "An Anthropic message or named SSE event stream.",+ "content": {+ "application/json": { "schema": { "$ref": "#/components/schemas/AnthropicMessage" } },+ "text/event-stream": { "schema": { "type": "string" } }+ }+ },+ "400": { "$ref": "#/components/responses/BadRequest" },+ "401": { "$ref": "#/components/responses/Unauthorized" },+ "429": { "$ref": "#/components/responses/QueueFull" }+ }+ }+ },+ "/v1/models": {+ "get": {+ "tags": ["Models"],+ "summary": "List loaded models",+ "description": "Return the model exposed by this persistent server process.",+ "operationId": "listModels",+ "security": [{ "bearerAuth": [] }, { "apiKeyAuth": [] }],+ "responses": {+ "200": {+ "description": "Model list",+ "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ModelList" } } }+ },+ "401": { "$ref": "#/components/responses/Unauthorized" }+ }+ }+ },+ "/v1/models/{model}": {+ "get": {+ "tags": ["Models"],+ "summary": "Retrieve the loaded model",+ "description": "Return metadata when the path matches the configured model identifier.",+ "operationId": "retrieveModel",+ "security": [{ "bearerAuth": [] }, { "apiKeyAuth": [] }],+ "parameters": [+ {+ "name": "model",+ "in": "path",+ "required": true,+ "schema": { "type": "string" },+ "description": "The identifier passed with `--model-id`."+ }+ ],+ "responses": {+ "200": {+ "description": "Model metadata",+ "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Model" } } }+ },+ "401": { "$ref": "#/components/responses/Unauthorized" },+ "404": { "$ref": "#/components/responses/NotFound" }+ }+ }+ },+ "/health": {+ "get": {+ "tags": ["Operations"],+ "summary": "Check server health",+ "description": "Always returns liveness. Authenticated requests also receive scheduler, KV-slot, tier, and hardware details when available.",+ "operationId": "getHealth",+ "security": [{}, { "bearerAuth": [] }, { "apiKeyAuth": [] }],+ "responses": {+ "200": {+ "description": "Server health",+ "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Health" } } }+ }+ }+ }+ },+ "/experts": {+ "get": {+ "tags": ["Operations"],+ "summary": "Read expert routing telemetry",+ "description": "Return an empty map without valid authentication, or the current expert map and routing hits for an authenticated local dashboard.",+ "operationId": "getExperts",+ "security": [{}, { "bearerAuth": [] }, { "apiKeyAuth": [] }],+ "responses": {+ "200": {+ "description": "Expert map and routing state",+ "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Experts" } } }+ }+ }+ }+ },+ "/profile": {+ "get": {+ "tags": ["Operations"],+ "summary": "Read generation profiles",+ "description": "Return an empty turn list without valid authentication, or recent per-turn timing and token telemetry for an authenticated local dashboard.",+ "operationId": "getProfile",+ "security": [{}, { "bearerAuth": [] }, { "apiKeyAuth": [] }],+ "responses": {+ "200": {+ "description": "Recent generation profiles",+ "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Profile" } } }+ }+ }+ }+ }+ },+ "components": {+ "securitySchemes": {+ "bearerAuth": {+ "type": "http",+ "scheme": "bearer",+ "description": "The value configured with `COLI_API_KEY` or `--api-key`."+ },+ "apiKeyAuth": {+ "type": "apiKey",+ "in": "header",+ "name": "x-api-key",+ "description": "Anthropic-compatible API-key header."+ }+ },+ "schemas": {+ "ChatMessage": {+ "type": "object",+ "required": ["role", "content"],+ "properties": {+ "role": { "type": "string", "enum": ["system", "user", "assistant", "tool"] },+ "content": {+ "oneOf": [+ { "type": "string" },+ { "type": "array", "items": { "type": "object", "additionalProperties": true } }+ ]+ },+ "name": { "type": "string" },+ "tool_call_id": { "type": "string" },+ "tool_calls": { "type": "array", "items": { "type": "object", "additionalProperties": true } }+ }+ },+ "ChatCompletionRequest": {+ "type": "object",+ "required": ["model", "messages"],+ "properties": {+ "model": { "type": "string", "example": "glm-5.2-colibri" },+ "messages": { "type": "array", "minItems": 1, "items": { "$ref": "#/components/schemas/ChatMessage" } },+ "stream": { "type": "boolean", "default": false },+ "stream_options": { "type": "object", "properties": { "include_usage": { "type": "boolean" } } },+ "max_tokens": { "type": "integer", "minimum": 1 },+ "max_completion_tokens": { "type": "integer", "minimum": 1 },+ "temperature": { "type": "number", "minimum": 0 },+ "top_p": { "type": "number", "minimum": 0, "maximum": 1 },+ "stop": {+ "oneOf": [+ { "type": "string" },+ { "type": "array", "maxItems": 4, "items": { "type": "string" } }+ ]+ },+ "tools": { "type": "array", "items": { "type": "object", "additionalProperties": true } },+ "tool_choice": { "oneOf": [{ "type": "string" }, { "type": "object", "additionalProperties": true }] },+ "reasoning_effort": { "type": "string" },+ "enable_thinking": { "type": "boolean", "description": "colibrì extension for model reasoning output." },+ "cache_slot": { "type": "integer", "minimum": 0, "maximum": 15, "description": "colibrì extension selecting an isolated KV context." },+ "x_colibri_ignore_leading_stop": { "type": "boolean", "description": "Ignore leading stop markers until non-whitespace output begins." }+ }+ },+ "CompletionRequest": {+ "type": "object",+ "required": ["model", "prompt"],+ "properties": {+ "model": { "type": "string" },+ "prompt": { "type": "string" },+ "stream": { "type": "boolean", "default": false },+ "max_tokens": { "type": "integer", "minimum": 1 },+ "temperature": { "type": "number", "minimum": 0 },+ "top_p": { "type": "number", "minimum": 0, "maximum": 1 },+ "stop": {+ "oneOf": [+ { "type": "string" },+ { "type": "array", "maxItems": 4, "items": { "type": "string" } }+ ]+ },+ "cache_slot": { "type": "integer", "minimum": 0, "maximum": 15 }+ }+ },+ "AnthropicMessageRequest": {+ "type": "object",+ "required": ["model", "max_tokens", "messages"],+ "properties": {+ "model": { "type": "string" },+ "max_tokens": { "type": "integer", "minimum": 1 },+ "messages": { "type": "array", "items": { "type": "object", "required": ["role", "content"], "additionalProperties": true } },+ "system": { "oneOf": [{ "type": "string" }, { "type": "array", "items": { "type": "object", "additionalProperties": true } }] },+ "stream": { "type": "boolean", "default": false },+ "temperature": { "type": "number", "minimum": 0 },+ "top_p": { "type": "number", "minimum": 0, "maximum": 1 },+ "tools": { "type": "array", "items": { "type": "object", "additionalProperties": true } },+ "tool_choice": { "type": "object", "additionalProperties": true },+ "thinking": { "type": "object", "additionalProperties": true }+ }+ },+ "Usage": {+ "type": "object",+ "properties": {+ "prompt_tokens": { "type": "integer" },+ "completion_tokens": { "type": "integer" },+ "total_tokens": { "type": "integer" }+ }+ },+ "ChatCompletion": {+ "type": "object",+ "properties": {+ "id": { "type": "string" },+ "object": { "type": "string", "const": "chat.completion" },+ "created": { "type": "integer" },+ "model": { "type": "string" },+ "choices": { "type": "array", "items": { "type": "object", "additionalProperties": true } },+ "usage": { "$ref": "#/components/schemas/Usage" }+ }+ },+ "Completion": {+ "type": "object",+ "properties": {+ "id": { "type": "string" },+ "object": { "type": "string", "const": "text_completion" },+ "created": { "type": "integer" },+ "model": { "type": "string" },+ "choices": { "type": "array", "items": { "type": "object", "additionalProperties": true } },+ "usage": { "$ref": "#/components/schemas/Usage" }+ }+ },+ "AnthropicMessage": {+ "type": "object",+ "properties": {+ "id": { "type": "string" },+ "type": { "type": "string", "const": "message" },+ "role": { "type": "string", "const": "assistant" },+ "content": { "type": "array", "items": { "type": "object", "additionalProperties": true } },+ "model": { "type": "string" },+ "stop_reason": { "type": ["string", "null"] },+ "usage": { "type": "object", "additionalProperties": true }+ }+ },+ "Model": {+ "type": "object",+ "properties": {+ "id": { "type": "string" },+ "object": { "type": "string", "const": "model" },+ "created": { "type": "integer" },+ "owned_by": { "type": "string" }+ }+ },+ "ModelList": {+ "type": "object",+ "properties": {+ "object": { "type": "string", "const": "list" },+ "data": { "type": "array", "items": { "$ref": "#/components/schemas/Model" } }+ }+ },+ "Health": {+ "type": "object",+ "required": ["status"],+ "properties": {+ "status": { "type": "string", "const": "ok" },+ "scheduler": { "type": "object", "additionalProperties": true },+ "kv_slots": { "type": "integer" },+ "tiers": { "type": "object", "additionalProperties": true },+ "hwinfo": { "type": "object", "additionalProperties": true }+ }+ },+ "Experts": {+ "type": "object",+ "required": ["rows", "cols", "map", "hits", "seq"],+ "properties": {+ "rows": { "type": "integer" },+ "cols": { "type": "integer" },+ "map": { "type": "string" },+ "hits": { "type": "string" },+ "seq": { "type": "integer" }+ }+ },+ "Profile": {+ "type": "object",+ "required": ["seq", "turns"],+ "properties": {+ "seq": { "type": "integer" },+ "turns": { "type": "array", "items": { "type": "object", "additionalProperties": true } }+ }+ },+ "Error": {+ "type": "object",+ "properties": {+ "error": {+ "type": "object",+ "properties": {+ "message": { "type": "string" },+ "type": { "type": ["string", "null"] },+ "param": { "type": ["string", "null"] },+ "code": { "type": ["string", "null"] }+ }+ }+ }+ }+ },+ "responses": {+ "BadRequest": {+ "description": "The request is invalid or asks for an unsupported capability.",+ "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } }+ },+ "Unauthorized": {+ "description": "The configured API key is missing or invalid.",+ "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } }+ },+ "QueueFull": {+ "description": "The bounded generation queue is full or the request expired while waiting.",+ "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } }+ },+ "NotFound": {+ "description": "The requested model or route does not exist.",+ "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } }+ }+ }+ }+}diff --git a/docs/api-reference/overview.mdx b/docs/api-reference/overview.mdxnew file mode 100644index 00000000..28d3cb16--- /dev/null+++ b/docs/api-reference/overview.mdx@@ -0,0 +1,58 @@+---+title: "API overview"+description: "Start colibrì's persistent HTTP server and connect OpenAI- or Anthropic-compatible clients."+---++`coli serve` keeps one model process loaded and exposes OpenAI- and Anthropic-compatible HTTP endpoints through a Python standard-library gateway.++## Start the server++```bash+COLI_MODEL=/nvme/glm52_i4 \+COLI_API_KEY=local-secret \+./coli serve --host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri+```++The default base URL is `http://127.0.0.1:8000`. OpenAI-compatible clients normally use `http://127.0.0.1:8000/v1`.++## Authenticate++When `COLI_API_KEY` is set, send either of these headers:++```http+Authorization: Bearer local-secret+```++```http+x-api-key: local-secret+```++<Note>+`GET /health`, `GET /experts`, and `GET /profile` remain reachable for local dashboards. Without valid authentication they return only public or empty data; authenticated requests include scheduler, routing, or per-turn telemetry.+</Note>++## Choose a protocol++<Tabs>+ <Tab title="OpenAI">+ Use `POST /v1/chat/completions` for messages or legacy `POST /v1/completions` for a text prompt. Both support JSON and server-sent event responses.+ </Tab>+ <Tab title="Anthropic">+ Use `POST /v1/messages`. The gateway translates supported message, thinking, and tool-use fields into the active engine's native prompt protocol.+ </Tab>+</Tabs>++## Concurrency and backpressure++One engine process generates one sequence at a time. Concurrent HTTP requests enter a bounded FIFO queue instead of loading duplicate model copies. Configure it with `--max-queue` and `--queue-timeout`, or `COLI_MAX_QUEUE` and `COLI_QUEUE_TIMEOUT`.++Saturated or expired requests receive HTTP `429`. Successful generation responses include `x-colibri-queue-wait-ms`.++## Compatibility boundaries++- Images, log probabilities, and token penalties return explicit errors instead of being ignored.+- Audio is accepted only by Inkling checkpoints with audio support.+- Tool calling depends on the active model engine.+- `stop_sequences`, `top_k`, and non-text Anthropic content blocks are rejected explicitly.++Read the [full API guide](/api) for engine-specific tool support, KV cache slots, coding-client examples, and performance expectations.diff --git a/docs/docs.json b/docs/docs.jsonnew file mode 100644index 00000000..4b9ffd21--- /dev/null+++ b/docs/docs.json@@ -0,0 +1,74 @@+{+ "$schema": "https://mintlify.com/docs.json",+ "theme": "mint",+ "name": "colibrì",+ "description": "Run frontier mixture-of-experts models on consumer and heterogeneous hardware.",+ "colors": {+ "primary": "#00afaf",+ "light": "#5fd7d7",+ "dark": "#008b8b"+ },+ "navigation": {+ "tabs": [+ {+ "tab": "Documentation",+ "groups": [+ {+ "group": "Get started",+ "pages": ["index", "quickstart"]+ },+ {+ "group": "Serve models",+ "pages": ["api-reference/overview", "api", "serve_protocol"]+ },+ {+ "group": "Model engines",+ "pages": ["deepseek-v4", "kimi_k3", "qwen36", "inkling"]+ },+ {+ "group": "Configure and tune",+ "pages": ["SETTINGS", "ENVIRONMENT", "tuning", "CACHE_ROUTE"]+ },+ {+ "group": "Hardware backends",+ "pages": ["cuda", "metal", "vulkan", "windows"]+ },+ {+ "group": "Contribute",+ "pages": ["benchmarks", "MAINTAINING-DOCS"]+ }+ ]+ },+ {+ "tab": "API Reference",+ "openapi": {+ "source": "./api-reference/openapi.json",+ "directory": "api-reference/endpoints"+ }+ }+ ]+ },+ "navbar": {+ "links": [+ {+ "label": "Website",+ "href": "https://justvugg.github.io/colibri"+ },+ {+ "label": "GitHub",+ "href": "https://github.com/JustVugg/colibri",+ "icon": "github"+ }+ ],+ "primary": {+ "type": "button",+ "label": "Quickstart",+ "href": "/quickstart"+ }+ },+ "footer": {+ "socials": {+ "github": "https://github.com/JustVugg/colibri"+ }+ }+}diff --git a/docs/index.mdx b/docs/index.mdxnew file mode 100644index 00000000..cc07f5f9--- /dev/null+++ b/docs/index.mdx@@ -0,0 +1,43 @@+---+title: "colibrì documentation"+description: "Run frontier mixture-of-experts models across storage, RAM, and VRAM."+---++colibrì is a dependency-light inference engine for running large mixture-of-experts models on consumer and heterogeneous hardware. It treats storage, RAM, and VRAM as one inference hierarchy while keeping model semantics explicit and measurable.++<CardGroup cols={2}>+ <Card title="Run your first model" icon="rocket" href="/quickstart">+ Build the engine, prepare a model, and start an interactive chat.+ </Card>+ <Card title="Expose an HTTP API" icon="server" href="/api-reference/overview">+ Serve OpenAI- and Anthropic-compatible endpoints from one persistent model process.+ </Card>+ <Card title="Explore the API" icon="brackets-curly" href="/api-reference/endpoints">+ Browse request schemas, responses, authentication, and live examples by endpoint.+ </Card>+ <Card title="Tune the memory hierarchy" icon="gauge-high" href="/tuning">+ Measure and tune placement across VRAM, RAM, and storage without silently changing precision.+ </Card>+</CardGroup>++## What you can run++The repository currently includes engines for GLM, DeepSeek, Kimi, Qwen, Inkling, and OLMoE families. Each engine uses the same `coli chat`, `coli serve`, and `coli web` front end.++<Note>+colibrì is an inference engine and research platform, not a hosted service. Throughput depends heavily on model format, memory placement, storage, and prompt size.+</Note>++## Common entry points++| Command | Use it for |+| --- | --- |+| `coli chat` | Interactive terminal inference |+| `coli serve` | OpenAI- and Anthropic-compatible HTTP endpoints |+| `coli web` | HTTP endpoints plus the local web dashboard |+| `coli doctor` | Environment and dependency diagnostics |+| `coli tune` | Measured placement and performance tuning |++<Warning>+Set `COLI_API_KEY` before exposing the HTTP server beyond a trusted local machine. Keep raw browser and telemetry endpoints behind the same network boundary.+</Warning>