Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
"""Prefill / decode callback factories.
All callbacks are ports of the deployment engines' proven hooks
(``engine_qwen.py`` / ling ``engine.py``); each is a pure function
building a callback from a ``dict[int, StreamingSwitchGLU]`` so the
same machinery serves both model families.
"""
from __future__ import annotations
def make_prefill_before_layer(all_stream_layers, *, full_layer: bool = True,
full_n: int = 0,
hot_n: int = 0, hot_window: int = 4):
"""Before-layer prefill hook: whole-layer load-drop (E3b) + the
sliding hot-expert window.
Before layer ``li`` runs: drop layer ``li-1``'s whole-layer set (its
async_eval was already submitted, so the GPU queue keeps the arrays
alive until evaluated), then either load layer ``li``'s full stacked
tensors (``full_layer`` and ``full_n`` unset / ``li < full_n``) or
build the hot-stack window (``hot_n``): numpy backing for layers
``li..li+ahead``, GPU materialization for the window, dematerialize
trailing layers.
``full_layer=False`` switches the whole-layer load-drop off (the hot
window, if requested, still runs). Callers should not install this
hook at all when both ``full_layer`` and ``hot_n`` are off, so a
disabled prefill keeps the plain per-expert on-demand path.
"""
w = max(1, hot_window)
ahead = max(1, w // 2)
def before_layer(li: int) -> None:
exp_old = all_stream_layers.get(li - 1)
if exp_old is not None:
exp_old.clear_full_layer()
if full_n and li >= full_n:
return
if hot_n:
for lj in range(li, min(li + ahead + 1,
len(all_stream_layers))):
exp_w = all_stream_layers.get(lj)
if exp_w is None:
continue
exp_w.load_hot_layer(hot_n) # numpy backing (cheap)
exp_w.materialize_hot() # GPU window
for lj in range(0, max(0, li - (w - ahead))):
exp_t = all_stream_layers.get(lj)
if exp_t is not None:
exp_t.dematerialize_hot()
if not full_n:
return
if not full_layer:
return
exp = all_stream_layers.get(li)
if exp is not None:
exp.load_full_layer()
return before_layer
def make_intra_after_layer(all_stream_layers, enabled: bool):
"""Intra-step staging: right after layer ``li`` runs, submit the next
step's layer-``li`` fill from ``last_used`` (the router's actuals from
the previous step, already on the CPU — no mid-forward tolist, which
segfaults in mlx 0.30.4 when a pool thread forces a GPU sync while the
main thread is still building the graph)."""
if not enabled:
return None
def after_layer(li: int, hidden_states=None) -> None:
exp = all_stream_layers.get(li)
if exp is None or not exp.last_used:
return
exp.stage_experts(list(exp.last_used))
return after_layer
def make_history_prefetch(all_stream_layers, enabled: bool):
"""Zero-cost decode prefetch: submit each layer's previous-token
expert set (``last_used``) so the SSD loads overlap this step's GPU
compute. The no-prerouter replacement for next-token prediction —
exploits adjacent-token expert locality (~0.4 overlap measured)."""
if not enabled:
return None
def history_prefetch() -> None:
for li in range(1, len(all_stream_layers)):
e = all_stream_layers.get(li)
if e is None or not e.last_used:
continue
e.prefetch(list(e.last_used))
return history_prefetch