Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
17.7 KB
Raw Blame
"""edge0-8b engine: Ling 3.0 hybrid (MLA + MoE, 8B tier).

Port of the deployment's ling ``engine.py`` hybrid-prerouter path:

* the vendored ``bailing_hybrid.py`` model already owns its prerouter
  heads and consumes ``prerouter_cache`` logits inside
  ``BailingSparseMoE``'s ``_select_from_logits`` (sigmoid + group-limited
  top-k, routed-scaling 2.5) — the engine only feeds the cache and
  submits the staged fills.
* step boundary: ONE stacked head batch -> ONE tolist -> per-consumer
  ``stage_experts`` + ``pg_cache[consumer]`` logits; the next forward's
  consuming MoE blocks re-select from those cached logits, so staged
  set == routing set (zero drops) by construction.
* prefill — E3b whole-layer load-drop (``prefill_full_layers``).
"""

from __future__ import annotations

import os
from types import SimpleNamespace

from edge0.backends import core

from edge0.backends.mlx._impl.bailing_hybrid import Model as BailingModel
from edge0.backends.mlx._impl.bailing_hybrid import ModelArgs as BailingArgs
from edge0.backends import io
from edge0.backends.mlx.io import load_model, load_tokenizer
from edge0.engine.base import Edge0Engine
from edge0.engine.hooks import (
    make_history_prefetch,
    make_prefill_before_layer,
)
from edge0.prerouter.install import install_prerouter
from edge0.prerouter.stager import LingPrerouterStager
from edge0.streaming.install import install_streaming_experts
from edge0.streaming.mmap import SafetensorsMmap


def _get_model_classes(config):
    """mlx-lm class hook: serve the vendored bailing backbone."""
    return BailingModel, BailingArgs


def load_installed(model_dir: str, cfg):
    """Load the ling skeleton and install streaming twins, LoRA and the
    hybrid prerouter weights (shared by ``build_model``/``build_engine``).

    Returns ``(model, model_config, shards, installs)``.
    """
    model, model_config = load_model(
        model_dir, lazy=True, strict=False,
        model_config={"model_type": "bailing_hybrid",
                      "prerouter_enabled": cfg.prerouter is not None,
                      "prerouter_start_layer":
                          getattr(cfg.prerouter, "start_layer", 7),
                      "prerouter_hidden":
                          getattr(cfg.prerouter, "hidden", 512)},
        get_model_classes=_get_model_classes)
    shards = [SafetensorsMmap(os.path.join(
        os.fspath(model_dir), "model.safetensors"))]
    spec = cfg.moe_spec
    opts = cfg.options
    n_layers = model_config["num_hidden_layers"]
    installed = install_streaming_experts(
        model, shards, spec, options=opts, num_layers=n_layers)
    all_stream = {li: t for li, t in enumerate(installed) if t is not None}
    # Zero-drop layer scoping (quality invariant): staged slots are only
    # coherent for layers whose ROUTE comes from the prerouter (consumer
    # li >= start_layer + 1: owner li-1 predicts li, and the consuming
    # block re-selects from the SAME cached logits the stager staged
    # from — set == routing, drops == 0, output bit-parity with the
    # exact path).  Layers at/below start_layer route with their own
    # gate every token; their slots cannot be predicted, so they run
    # the exact path.  History-staging them instead (qwen-style
    # actuals-as-prediction) misaligns one step behind the gate and
    # measurably degrades output (17% drop rate, garbled text).
    first_pg_consumer = (
        getattr(cfg.prerouter, "start_layer", 7) + 1
        if cfg.prerouter else 0)
    if not getattr(cfg, "history_slots", False):
        # Scoped default: only the consumers keep staged slots.  Switch-off
        # takes the same rule to its limit -- with ``--no-prerouter`` there is
        # no prediction anywhere, so NO layer keeps slots and the off path is
        # the exact path (gate routing, on-demand loads), not history staging
        # (measured ``staged_dropped = 3593`` over a whole run when the slots
        # stayed on with the prerouter off).  Legacy ``history_slots=True``
        # deliberately leaves every layer staged (slots filled from the
        # previous token's actuals), which is the behaviour that silently
        # zeroes the routed experts outside the slot set.
        for li, t in all_stream.items():
            if cfg.prerouter is None or li < first_pg_consumer:
                t._staged_mode = False
    stream_layers = {li: t for li, t in all_stream.items() if t._staged_mode}

    if cfg.lora:
        from edge0.adapters.lora import install_lora
        install_lora(model, cfg.lora, r=cfg.lora_r, alpha=cfg.lora_alpha)

    pg_state = pg_stager = None
    if cfg.prerouter and cfg.prerouter.weights_file:
        pg_state, heads = install_prerouter(
            model=model, spec=spec, pspec=cfg.prerouter, n_layers=n_layers)
        pg_stager = LingPrerouterStager(
            model=model, spec=spec, pspec=cfg.prerouter,
            state=pg_state, stream_layers=stream_layers,
            top_k=cfg.prerouter_top_k)
        print(f"[edge0-8b] prerouter installed: {len(heads)} heads, "
              f"start={cfg.prerouter.start_layer}, "
              f"K={cfg.prerouter_top_k}", flush=True)
    installs = dict(all_stream_layers=all_stream,
                    stream_layers=stream_layers,
                    first_pg_consumer=first_pg_consumer,
                    pg_state=pg_state, pg_stager=pg_stager)
    return model, model_config, shards, installs


class Ling8BEngine(Edge0Engine):
    """Streaming Ling-3.0 engine (staged decode + hybrid prerouter)."""

    name = "edge0-8b"

    def __init__(self, model_dir: str, cfg, tokenizer=None,
                 think: bool = False):
        # Deployment parity: the ling server's THINK_MODE knob becomes the
        # engine's ``think`` flag (start_server.sh exports THINK_MODE=0).
        self.think = think
        self._chat_tpl = None
        # NAN_BANG_COLLAPSE_FIX.md parity: hidden clip default 1000 unless
        # the deployer overrides LING_HIDDEN_CLIP explicitly.  One fp16
        # overflow inside a layer otherwise poisons the whole net into
        # all-NaN logits -> argmax fallback token 0 ('!') death spiral.
        if "LING_HIDDEN_CLIP" not in os.environ:
            os.environ["LING_HIDDEN_CLIP"] = "1000"
        super().__init__(model_dir, cfg, tokenizer=tokenizer)

    def _build(self):
        cfg = self.cfg
        (self.model, self.model_config, self.shards,
         inst) = load_installed(cfg.model_dir, cfg)
        self._all_stream_layers = inst["all_stream_layers"]
        self._stream_layers = inst["stream_layers"]
        self._first_pg_consumer = inst["first_pg_consumer"]
        self._pg_state = inst["pg_state"]
        self._pg_stager = inst["pg_stager"]
        opts = cfg.options
        if self._tok is None:
            try:
                self._tok = load_tokenizer(cfg.model_dir)
            except Exception:  # noqa: BLE001 — tokenizer optional for CLI
                pass

        self._prefill_before_layer = make_prefill_before_layer(
            self._all_stream_layers,
            full_layer=bool(getattr(opts, "full_layer_prefill", False)),
            full_n=getattr(opts, "prefill_full_layers", 0),
            hot_n=getattr(opts, "prefill_hot", 0), hot_window=1)
        self._history_prefetch = make_history_prefetch(
            self._all_stream_layers, enabled=cfg.prefetch_history)

        self.cache = self.model.make_cache()
        # start_server.sh LING_PREWARM=1 parity (opt-in): warm the OS page
        # cache over the whole checkpoint (madvise + sequential read) and
        # run a tiny dummy prefill+step so the first real request runs at
        # near-steady-state speed (kernel JIT + LRU + hot pins warm).
        if os.environ.get("EDGE0_PREWARM", os.environ.get(
                "LING_PREWARM", "0")) == "1":
            self._prewarm()

    # ---- startup warm-up --------------------------------------------------

    def _prewarm(self):
        """Page-cache + kernel warm-up.

        1. madvise(WILLNEED) + a full sequential read over every shard —
           removes per-expert page-fault cost from the first request.
        2. A dummy prefill + one decode step — materializes attention /
           router / shared weights, warms the LRU and hot pins, and
           pre-compiles the single-token decode kernels.
        KV state is reset afterwards; only page-cache warmth remains.
        """
        import time as _t
        t0 = _t.perf_counter()
        try:
            for shard in self.shards:
                shard.advise_willneed() if hasattr(
                    shard, "advise_willneed") else None
                shard.seq_read() if hasattr(shard, "seq_read") else None
        except Exception:  # noqa: BLE001 — advisory only
            pass
        dummy = [self._tok.bos_token_id or 0] if self._tok else [0]
        try:
            self.reset()
            self.prefill(dummy * 4)
            self.step(dummy[0])
            self.reset()
        except Exception:  # noqa: BLE001 — advisory only
            pass
        print(f"[edge0-8b] prewarm done in "
              f"{_t.perf_counter() - t0:.1f}s", flush=True)

    # ---- forward ----------------------------------------------------------

    def _forward(self, ids, intra_stage: bool = True) -> core.array:
        inputs = core.array(ids)[None, :]
        opts = self.cfg.options
        prefill_multi = len(ids) > 1 and self._prefill_active
        full_layer = bool(
            prefill_multi and self._all_stream_layers
            and opts.full_layer_prefill)
        h = self.model.model(
            inputs, cache=self.cache,
            before_layer_cb=(self._prefill_before_layer
                             if (prefill_multi and (full_layer
                                                    or opts.prefill_hot))
                             else None),
            after_layer_cb=None,
            async_eval_per_layer=bool(prefill_multi and full_layer),
            prerouter_cache=(self._pg_stager.pg_cache
                             if self._pg_stager is not None else None))
        logits = self.model.lm_head(h[0, -1])
        core.eval(logits)
        # Step boundary: ONE head batch (stacked einsum) -> fills.  The
        # host-side tolist only happens when a consumer actually needs the
        # expert IDs (``stream_layers`` non-empty); with staged loading off
        # the routing cache is filled lazily and the sync is skipped.
        if (self._pg_stager is not None and not self._prefill_active
                and len(ids) == 1):
            self._pg_stager.stage_all()
        return logits

    # ---- step hooks -------------------------------------------------------

    def _step_pre(self, token_id: int) -> None:
        if self._pg_stager is not None:
            # Legacy ``history_slots=True``: the layers without an incoming
            # prediction (li < start+1) are still staged, so fill them from
            # their previous actuals.  In the scoped default they are
            # ``_staged_mode=False`` and run the exact path here -- no
            # staging and no prefetch (the previous token's set covers only a
            # small fraction of the next route, so the reads are wasted).
            for li, exp in self._all_stream_layers.items():
                if li < self._first_pg_consumer:
                    if exp._staged_mode and exp.last_used:
                        exp.stage_experts(list(exp.last_used))
        elif self._history_prefetch is not None:
            self._history_prefetch()

    def _step_post(self, token_id: int) -> None:
        if self._pg_stager is not None:
            # Consumers: the prerouter's ``stage_all`` already submitted
            # their next set; only record actuals (drop accounting + hot
            # pins).  Non-consumers run the exact path in the scoped default
            # (``_staged_mode=False``), where this bookkeeping is dead
            # weight; legacy ``history_slots`` stages them from their
            # actuals like the no-prerouter path.
            for li, exp in self._all_stream_layers.items():
                if li < self._first_pg_consumer:
                    if not exp._staged_mode:
                        continue
                    exp.sync_actuals()
                    if exp.last_used:
                        exp.stage_experts(list(exp.last_used))
                        exp.swap_staged()
                elif exp._staged_mode:
                    exp.sync_actuals()
            return
        # No prerouter: history staging -- each layer's next set is its
        # previous actuals (adjacent-token expert locality).
        for exp in self._stream_layers.values():
            exp.sync_actuals()
            if exp.last_used:
                exp.stage_experts(list(exp.last_used))
                exp.swap_staged()

    # ---- prefill / reset --------------------------------------------------

    def _prefill_end(self) -> None:
        for exp in self._all_stream_layers.values():
            exp.clear_full_layer()
        if self._pg_stager is not None:
            # Deployment parity: prefill's tail stages the FIRST decode
            # token's expert sets for the prerouter consumers (engine.py:919
            # calls stage_all right after prefill).  Without this the first
            # decode step has no prerouter logits, falls back to the true
            # gate for one step, and the whole prediction chain shifts by one
            # token (observed: [23982, 2862, ...] vs correct [23982, 4264, ...]).
            self._pg_stager.stage_all()
            if getattr(self.cfg, "history_slots", False):
                # Legacy: the non-consumer layers are staged too and get the
                # last prefill token's actual top-k.  MUST NOT touch the
                # consumers -- ``stage_from_prefill`` would overwrite the
                # prerouter's submissions and misalign the first step.
                for li, exp in self._stream_layers.items():
                    if li < self._first_pg_consumer:
                        exp.stage_from_prefill()
        elif self._stream_layers:
            # No prerouter: history staging is the only predictor (legacy).
            for exp in self._stream_layers.values():
                exp.stage_from_prefill()

    def _reset_state(self) -> None:
        self.cache = self.model.make_cache()
        if self._pg_state is not None:
            self._pg_stager.reset()
            self._pg_state.reset()
            # Drop the per-block/per-layer feature captures as well, or the
            # NEXT request's first ``stage_all`` runs the heads on this
            # request's last token: ``prev_topk_oh`` is only ever written by
            # ``_note`` at the step boundary, so it would feed the previous
            # request's last top-k as the "prev token" feature and stage
            # stale predictions that the first decode step then routes with.
            for owner in self._pg_state.owners:
                block = self.cfg.moe_spec.block_of(self.model, owner)
                layer = self.cfg.moe_spec.layer_of(self.model, owner)
                block.last_topk = None
                block.prev_topk_oh = None
                layer.m_in_cache = None
        # Per-request reset of the streaming layers' staged state: a leftover
        # ``_staged_state`` from the previous request defeats the staged
        # path's "fill not ready -> exact path" fallback (``st`` would never
        # be None) and would serve the previous request's expert bundles.
        for exp in getattr(self, "_all_stream_layers", {}).values():
            exp.reset()

    def _lm_logits(self, h: core.array) -> core.array:
        return self.model.lm_head(h[0, -1])

    # ---- chat template (deployment parity) -------------------------------

    def _chat_template(self):
        """Render the deployment's ``chat_template.jinja`` (same file and
        renderer as the ling server's ``encode_chat``, engine.py)."""
        from jinja2 import BaseLoader, Environment, StrictUndefined

        if self._chat_tpl is None:
            src = open(
                os.path.join(self.dir, "chat_template.jinja"),
                encoding="utf-8",
            ).read()
            self._chat_tpl = Environment(
                loader=BaseLoader(), undefined=StrictUndefined,
                autoescape=False,
            ).from_string(src)
        return self._chat_tpl

    def encode_chat(self, messages, think=None, tools=None) -> list:
        """Tokenize chat messages with the deployment chat template.

        ``think`` mirrors THINK_MODE: True renders "detailed thinking on"
        (the model answers with a reasoning preamble), False renders
        "detailed thinking off" (direct answer).  Defaults to the
        engine's ``think`` flag. ``tools`` is the OpenAI-style function
        list from the request, forwarded to the template's own ``# Tools``
        system-prompt section and ``<tool_call>`` format instructions.
        """
        if think is None:
            think = self.think
        msgs = [
            SimpleNamespace(
                role=m.get("role"),
                content=m.get("content") or "",
                reasoning_content=m.get("reasoning_content") or "",
                tool_calls=m.get("tool_calls"),
            )
            for m in messages
        ]
        text = self._chat_template().render(
            messages=msgs,
            add_generation_prompt=True,
            enable_thinking=bool(think),
            tools=tools,
        )
        # transformers encode would prepend/append special tokens by
        # default; the template text is already complete.
        return self._tok.encode(text, add_special_tokens=False)

    def parse_tool_calls(self, text: str):
        """Split generated ``text`` into (content, OpenAI tool_calls) per
        this checkpoint's chat template ``<tool_call>name<arg_key>...``
        dialect (see edge0.server.tool_calls)."""
        from edge0.server.tool_calls import parse_ling_tool_calls
        return parse_ling_tool_calls(text)