Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
19.2 KB
Raw Blame
#!/usr/bin/env python3
"""colibri as a Model Context Protocol server, over stdio.

An AI coding assistant starts this process and talks JSON-RPC 2.0 to it, one
JSON message per line on stdin and stdout (the MCP stdio transport). Nothing
else may reach stdout: diagnostics go to stderr.

Tools, all thin wrappers over setup_flow.py (the same code `coli setup` runs):

    detect_hardware   RAM, disk, CPU features, GPUs (Vulkan, NVIDIA)
    recommend_models  the catalog against this machine, with the default marked
    install           build or fetch the engine and download a model, as a
                      background job that survives this server; progress is
                      reported with notifications/progress while waiting
    start / stop      the configured server (background)
    status            setup, install progress, server state, URLs, tok/s
    logs              the tail of the server, install or build log

Run it with `coli mcp`, or `python3 c/mcp_server.py`. docs/MCP_SERVER.md has
the client configuration.
"""
import json
import os
import subprocess
import sys
import threading
import time
import traceback

HERE = os.path.dirname(os.path.realpath(__file__))
if HERE not in sys.path:
    sys.path.insert(0, HERE)

import setup_catalog  # noqa: E402
import setup_flow  # noqa: E402
import setup_hw  # noqa: E402

SUPPORTED_PROTOCOLS = ("2025-06-18", "2025-03-26", "2024-11-05")
SERVER_NAME = "colibri"

PARSE_ERROR, INVALID_REQUEST, METHOD_NOT_FOUND, INVALID_PARAMS, INTERNAL_ERROR = (
    -32700, -32600, -32601, -32602, -32603)

INSTRUCTIONS = (
    "colibri runs large mixture-of-experts language models locally. Typical flow: "
    "detect_hardware, recommend_models, install (a background job: follow it with status), "
    "start, then give the user the browser URL from the result. stop shuts the server down. "
    "Downloads are tens to hundreds of GB: confirm the model with the user before install.")


def _version():
    try:
        from version import __version__
        return __version__
    except ImportError:
        return "0"


def _schema(properties=None, required=()):
    return {"type": "object", "properties": properties or {}, "required": list(required),
            "additionalProperties": False}


TOOLS = [
    {"name": "detect_hardware", "title": "Detect hardware",
     "description": ("RAM (total and free), free disk where models go, CPU name and features, "
                     "and GPUs: Vulkan devices with type and memory, NVIDIA cards via nvidia-smi."),
     "inputSchema": _schema({"target_dir": {"type": "string",
                                            "description": "folder whose free space to report "
                                                           "(default ~/colibri-models)"}})},
    {"name": "recommend_models", "title": "Recommend models",
     "description": ("Every downloadable model against this machine: download size, RAM needed, "
                     "whether it fits and why, with the recommended default marked."),
     "inputSchema": _schema({"target_dir": {"type": "string"},
                             "include_unfit": {"type": "boolean", "default": False}})},
    {"name": "install", "title": "Install a model",
     "description": ("Build or fetch the engine (Vulkan/CUDA when a GPU is usable), download the "
                     "model with resume, write the run configuration, and by default start the "
                     "server in the background. Runs as a background job; returns at once unless "
                     "wait_seconds is set. Calling it again resumes an interrupted install."),
     "inputSchema": _schema({
         "model": {"type": "string", "description": "catalog id from recommend_models "
                                                    "(default: the recommendation)"},
         "target_dir": {"type": "string", "description": "folder for models"},
         "model_dir": {"type": "string", "description": "use a model already on disk"},
         "backend": {"type": "string", "enum": ["auto", "cpu", "vulkan", "cuda"]},
         "port": {"type": "integer", "minimum": 1, "maximum": 65535},
         "start": {"type": "boolean", "default": True},
         "wait_seconds": {"type": "integer", "minimum": 0, "maximum": 86400, "default": 0}})},
    {"name": "start", "title": "Start colibri",
     "description": "Start the configured server in the background; returns its URLs.",
     "inputSchema": _schema({"open_browser": {"type": "boolean", "default": False},
                             "wait_seconds": {"type": "integer", "minimum": 0, "maximum": 7200,
                                              "default": 0}})},
    {"name": "stop", "title": "Stop colibri",
     "description": "Stop the configured server and its engine.",
     "inputSchema": _schema()},
    {"name": "status", "title": "Status",
     "description": ("Configured model and engine, install progress, server state (stopped, "
                     "loading, ready), browser/OpenAI/Anthropic URLs, tok/s of the last answer."),
     "inputSchema": _schema()},
    {"name": "logs", "title": "Logs",
     "description": "The last lines of the server, install or build log.",
     "inputSchema": _schema({"which": {"type": "string", "enum": ["server", "install", "build"],
                                       "default": "server"},
                             "lines": {"type": "integer", "minimum": 1, "maximum": 5000,
                                       "default": 100}})},
]


class ToolError(Exception):
    """A tool-level failure: reported as a result with isError, not a protocol error."""


def _log(*parts):
    print("[colibri-mcp]", *parts, file=sys.stderr, flush=True)


def _check_args(name, args):
    tool = next((t for t in TOOLS if t["name"] == name), None)
    if tool is None:
        raise KeyError(name)
    if not isinstance(args, dict):
        raise ValueError("arguments must be an object")
    props = tool["inputSchema"]["properties"]
    for key, value in args.items():
        if key not in props:
            raise ValueError(f"unknown argument {key!r} for {name}")
        kind = props[key].get("type")
        ok = {"string": isinstance(value, str),
              "integer": isinstance(value, int) and not isinstance(value, bool),
              "boolean": isinstance(value, bool)}.get(kind, True)
        if not ok:
            raise ValueError(f"{name}.{key} must be a {kind}")
        if "enum" in props[key] and value not in props[key]["enum"]:
            raise ValueError(f"{name}.{key} must be one of {props[key]['enum']}")
        if kind == "integer":
            low, high = props[key].get("minimum"), props[key].get("maximum")
            if (low is not None and value < low) or (high is not None and value > high):
                raise ValueError(f"{name}.{key} is out of range")
    return args


# ---------------------------------------------------------------- tools


def tool_detect_hardware(args, notify):
    target = args.get("target_dir") or setup_flow.default_models_root()
    report = setup_hw.detect(target)
    report["summary"] = setup_hw.format_report(report)
    return report


def tool_recommend_models(args, notify):
    target = args.get("target_dir") or setup_flow.default_models_root()
    hw = setup_hw.detect(target, probe_gpu=False)
    rows = setup_flow._recommend_for(hw, args.get("target_dir"))
    models = [setup_catalog.as_dict(r) for r in rows if r["fits"] or args.get("include_unfit")]
    default = next((m["id"] for m in models if m["recommended"]), None)
    return {"recommended": default, "fits_means": setup_catalog.FITS_EXPLAINED,
            "ram_total_gb": round((hw["memory"].get("total") or 0) / 1e9, 1),
            "disk_free_gb": round((setup_hw.disk_free(target) or 0) / 1e9, 1),
            "target_dir": target, "models": models}


def _install_running():
    state = setup_flow.read_state() or {}
    if state.get("phase") in ("ready", "error", "interrupted", "done", "started", None):
        return None
    return state if setup_flow._pid_alive(state.get("pid")) else None


def _coli():
    return os.path.join(HERE, "coli")


_INSTALL_LOCK = threading.Lock()


def _launch_install(args):
    running = _install_running()
    if running:
        raise ToolError(f"an install is already running (pid {running.get('pid')}, phase "
                        f"{running.get('phase')}); follow it with status")
    if args.get("model"):
        try:
            setup_catalog.by_id(args["model"])
        except KeyError:
            raise ToolError(f"unknown model {args['model']!r}; recommend_models lists the ids")
    cmd = [sys.executable, _coli(), "setup", "--yes", "--no-browser"]
    if args.get("model"):
        cmd += ["--model", args["model"]]
    if args.get("model_dir"):
        cmd += ["--model-dir", args["model_dir"]]
    if args.get("target_dir"):
        cmd += ["--dir", args["target_dir"]]
    if args.get("backend"):
        cmd += ["--backend", args["backend"]]
    if args.get("port"):
        cmd += ["--port", str(args["port"])]
    cmd += ["--background"] if args.get("start", True) else ["--no-start"]
    log = setup_flow.log_path("install")
    os.makedirs(os.path.dirname(log), exist_ok=True)
    handle = open(log, "a", encoding="utf-8", errors="replace")
    handle.write(f"\n--- install {time.strftime('%Y-%m-%d %H:%M:%S')}: {' '.join(cmd)}\n")
    handle.flush()
    kwargs = {"stdin": subprocess.DEVNULL, "stdout": handle, "stderr": subprocess.STDOUT,
              "env": dict(os.environ, PYTHONUNBUFFERED="1")}
    if sys.platform == "win32":
        kwargs["creationflags"] = 0x00000200 | 0x08000000   # new group, hidden console
    else:
        kwargs["start_new_session"] = True   # survives the assistant closing this server
    process = setup_flow.spawn_detached(cmd, **kwargs)
    handle.close()
    # The job's own pid: the child rewrites this file as it goes, with the same pid.
    setup_flow._write_json(setup_flow.state_path(),
                           {"phase": "starting", "pid": process.pid, "updated": time.time()})
    return process, log, cmd


def tool_install(args, notify):
    # tools/call runs in independent threads: admission and publishing the
    # spawned job's state must be one operation, or two calls can both see no
    # running install. Waiting/progress is outside the lock so status remains
    # responsive and a second install receives the existing-job error promptly.
    with _INSTALL_LOCK:
        process, log, cmd = _launch_install(args)
    wait = int(args.get("wait_seconds") or 0)
    result = {"job": {"pid": process.pid, "log": log, "command": cmd}}
    deadline = time.time() + wait
    while wait and time.time() < deadline:
        code = process.poll()
        state = setup_flow.read_state() or {}
        if state.get("phase") == "download" and state.get("total"):
            notify(state.get("done", 0), state["total"],
                   f"downloading {state.get('model')}: {state.get('done', 0) / 1e9:.1f} of "
                   f"{state['total'] / 1e9:.1f} GB")
        elif state.get("phase"):
            notify(None, None, f"install: {state['phase']}")
        if code is not None:
            break
        time.sleep(1.0)
    result["exit_code"] = process.poll()
    result["install"] = setup_flow.read_state()
    if result["exit_code"] not in (None, 0):
        result["log_tail"] = setup_flow.tail_file(log, 30)
        raise ToolError(json.dumps(result, indent=2))
    result["next"] = ("the install continues in the background: call status to follow it"
                      if result["exit_code"] is None else "installed: call status for the URLs")
    return result


def _stderr_out(*parts):
    _log(*parts)


def tool_start(args, notify):
    cfg = setup_flow.load_config()
    if not cfg or not setup_flow.config_ready(cfg):
        raise ToolError("nothing is installed yet (or the install has not finished): call install")
    status = setup_flow.start_server(cfg, background=True,
                                     open_browser=bool(args.get("open_browser")), out=_stderr_out)
    wait = int(args.get("wait_seconds") or 0)
    deadline = time.time() + wait
    while wait and time.time() < deadline and status.get("state") != "ready":
        time.sleep(2.0)
        status = setup_flow.server_status(setup_flow.load_config() or cfg)
        notify(None, None, f"server: {status['state']}")
        if status["state"] == "stopped":
            status["log_tail"] = setup_flow.tail_file(setup_flow.log_path("serve"), 30)
            raise ToolError(json.dumps(status, indent=2))
    return status


def tool_stop(args, notify):
    cfg = setup_flow.load_config()
    if not cfg:
        raise ToolError("nothing is configured; there is nothing to stop")
    return setup_flow.stop_server(cfg, out=_stderr_out)


def tool_status(args, notify):
    return setup_flow.status_report()


def tool_logs(args, notify):
    which = {"server": "serve"}.get(args.get("which", "server"), args.get("which", "server"))
    path = setup_flow.log_path(which)
    return {"log": path, "text": setup_flow.tail_file(path, int(args.get("lines") or 100))}


HANDLERS = {"detect_hardware": tool_detect_hardware, "recommend_models": tool_recommend_models,
            "install": tool_install, "start": tool_start, "stop": tool_stop,
            "status": tool_status, "logs": tool_logs}


# ---------------------------------------------------------------- protocol


class Server:
    def __init__(self, out=None):
        self.out = out or sys.stdout
        self.lock = threading.Lock()
        self.initialized = False
        self.threads = []

    def send(self, message):
        line = json.dumps(message, separators=(",", ":"), ensure_ascii=True)
        with self.lock:
            self.out.write(line + "\n")
            self.out.flush()

    def error(self, msg_id, code, message):
        self.send({"jsonrpc": "2.0", "id": msg_id, "error": {"code": code, "message": message}})

    def result(self, msg_id, result):
        self.send({"jsonrpc": "2.0", "id": msg_id, "result": result})

    def handle_line(self, line):
        line = line.strip()
        if not line:
            return
        try:
            message = json.loads(line)
        except ValueError:
            self.error(None, PARSE_ERROR, "parse error")
            return
        if isinstance(message, list):
            for item in message:
                self.handle(item)
            return
        self.handle(message)

    def handle(self, message):
        if not isinstance(message, dict) or message.get("jsonrpc") != "2.0":
            self.error(message.get("id") if isinstance(message, dict) else None,
                       INVALID_REQUEST, "invalid request")
            return
        method = message.get("method")
        msg_id = message.get("id")
        is_request = "id" in message
        if method is None:
            return                      # a response to something we never asked: ignore
        params = message.get("params") or {}
        if not is_request:
            return                      # notifications: initialized, cancelled, ...
        if method == "initialize":
            requested = params.get("protocolVersion") if isinstance(params, dict) else None
            version = requested if requested in SUPPORTED_PROTOCOLS else SUPPORTED_PROTOCOLS[0]
            self.initialized = True
            self.result(msg_id, {"protocolVersion": version,
                                 "capabilities": {"tools": {"listChanged": False}},
                                 "serverInfo": {"name": SERVER_NAME, "title": "colibri",
                                                "version": _version()},
                                 "instructions": INSTRUCTIONS})
        elif method == "ping":
            self.result(msg_id, {})
        elif method == "tools/list":
            self.result(msg_id, {"tools": TOOLS})
        elif method == "tools/call":
            # Keep EOF draining limited to calls still in flight, rather than
            # retaining every completed worker for the lifetime of the session.
            self.threads = [worker for worker in self.threads if worker.is_alive()]
            thread = threading.Thread(target=self.call_tool, args=(msg_id, params), daemon=True)
            self.threads.append(thread)
            thread.start()
        elif method in ("resources/list", "prompts/list"):
            self.result(msg_id, {method.split("/")[0]: []})
        else:
            self.error(msg_id, METHOD_NOT_FOUND, f"method not found: {method}")

    def call_tool(self, msg_id, params):
        if not isinstance(params, dict) or not isinstance(params.get("name"), str):
            self.error(msg_id, INVALID_PARAMS, "tools/call needs a tool name")
            return
        name = params["name"]
        token = (params.get("_meta") or {}).get("progressToken") if isinstance(params.get("_meta"), dict) else None
        last = {"value": 0}

        def notify(done, total, text):
            # MCP requires progress to increase with every notification: bytes
            # when a download reports them, else one step past the last value.
            if token is None:
                return
            value = done if done is not None and done > last["value"] else last["value"] + 1
            last["value"] = value
            payload = {"progressToken": token, "progress": value}
            if total and total >= value:
                payload["total"] = total
            if text:
                payload["message"] = text
            self.send({"jsonrpc": "2.0", "method": "notifications/progress", "params": payload})

        try:
            args = _check_args(name, params.get("arguments", {}))
        except KeyError:
            self.error(msg_id, INVALID_PARAMS, f"unknown tool: {name}")
            return
        except ValueError as error:
            self.error(msg_id, INVALID_PARAMS, str(error))
            return
        try:
            data = HANDLERS[name](args, notify)
            self.result(msg_id, {"content": [{"type": "text", "text": json.dumps(data, indent=2)}],
                                 "structuredContent": data if isinstance(data, dict) else {"result": data},
                                 "isError": False})
        except (ToolError, setup_flow.SetupError) as error:
            self.result(msg_id, {"content": [{"type": "text", "text": str(error)}], "isError": True})
        except Exception as error:   # a tool bug must not take the server down
            _log(traceback.format_exc())
            self.result(msg_id, {"content": [{"type": "text",
                                              "text": f"{type(error).__name__}: {error}"}],
                                 "isError": True})

    def serve(self, stream=None):
        stream = stream or sys.stdin
        for line in stream:
            self.handle_line(line)
        # stdin closed: let the calls in flight answer (a stop drains its engine)
        # before exiting, within the minute a client gives a server to go.
        deadline = time.time() + 60
        for thread in self.threads:
            thread.join(timeout=max(0.0, deadline - time.time()))


def main():
    if hasattr(sys.stdin, "reconfigure"):
        try:
            sys.stdin.reconfigure(encoding="utf-8")
            sys.stdout.reconfigure(encoding="utf-8", newline="\n")
        except (AttributeError, OSError, ValueError):
            pass
    _log(f"colibri MCP server {_version()} on stdio")
    Server().serve()
    return 0


if __name__ == "__main__":
    sys.exit(main())