Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
chore: unify the legacy pregate naming on prerouter
linyubupa1d32a55efd67
diff --git a/scripts/convert_adapters_legacy.py b/scripts/convert_adapters_legacy.pyindex 6483c33..09c1b96 100644--- a/scripts/convert_adapters_legacy.py+++ b/scripts/convert_adapters_legacy.py@@ -37,9 +37,9 @@ NPZ_DIRS = [Path(os.environ.get("EDGE0_NPZ_DIR", "/path/to/npz/exports")),]SOURCES = [- ("pregate_<family>.npz", "edge0-35b", "prerouter"),+ ("prerouter_<family>.npz", "edge0-35b", "prerouter"),("lora_<family>.npz", "edge0-35b", "lora"),- ("pregate_<family>.npz", "edge0-8b", "prerouter"),+ ("prerouter_<family>.npz", "edge0-8b", "prerouter"),("lora_<family>.npz", "edge0-8b", "lora"),]@@ -68,9 +68,12 @@ def rewrite_key(key: str, tier: str, kind: str) -> str:namespace the adapter is applied into)."""if kind == "prerouter":- # some families use layers.N.*; others use pregate.N.*+ # Some families use layers.N.*; others namespace the layer index+ # with a prefix segment (<ns>.<N>.<part>.weight). Both are+ # normalized to layers.<N>.<part>.weight.parts = key.split(".")- if parts[0] == "pregate":+ if (len(parts) >= 4 and parts[0] != "layers"+ and parts[1].isdigit()):return f"layers.{parts[1]}.{parts[2]}.{parts[3]}"return keyif kind == "lora" and tier == "edge0-35b" and key.startswith("model.layers."):diff --git a/scripts/verify_alignment.py b/scripts/verify_alignment.pyindex 300a8ec..57d861f 100644--- a/scripts/verify_alignment.py+++ b/scripts/verify_alignment.py@@ -49,10 +49,10 @@ def qwen_demo():MODEL_DIR=QMODEL,LORA_ADAPTERS=QDIR + "/lora_qwen35.npz",LORA_R="16", LORA_ALPHA="32",- PREGATE_NPZ=QDIR + "/pregate_qwen35.npz",+ PREROUTER_NPZ=QDIR + "/prerouter_qwen35.npz",QWEN_PG_START="7", QWEN_PG_HIDDEN="512",QWEN_STAGED="1", QWEN_STAGED_TOP_K="4", QWEN_STAGED_SLOTS="4",- QWEN_STAGED_REPLACE="0", QWEN_PREGATE_V7="0",+ QWEN_STAGED_REPLACE="0", QWEN_PREROUTER_V7="0",QWEN_HISTORY_PREFETCH="0", QWEN_INTRA_STAGE="0", QWEN_ASM_CACHE="0",)sys.path.insert(0, QDIR)@@ -99,9 +99,9 @@ def edge0_qwen():def ling_demo():os.environ.update(CHECKPOINT_DIR=LMODEL,- PREGATE_HYBRID="1", PREGATE_INTRA="0", PREGATE_START_LAYER="1",- PREGATE_FEATURE_TOPK="executed", PREGATE_DTYPE="fp16",- PREGATE_NPZ=LDIR + "/pregate_r3traces.npz",+ PREROUTER_HYBRID="1", PREROUTER_INTRA="0", PREROUTER_START_LAYER="1",+ PREROUTER_FEATURE_TOPK="executed", PREROUTER_DTYPE="fp16",+ PREROUTER_NPZ=LDIR + "/prerouter_r3traces.npz",LORA_ADAPTERS=LDIR + "/lora_r3traces.npz",THINK_MODE="0", REPETITION_PENALTY="1.0",STAGED_DECODE="0", STAGED_SYNC="0",diff --git a/src/edge0/engine/qwen.py b/src/edge0/engine/qwen.pyindex 78e9739..dbd3d54 100644--- a/src/edge0/engine/qwen.py+++ b/src/edge0/engine/qwen.py@@ -137,12 +137,13 @@ class Qwen35Engine(Edge0Engine):logits = self._lm.lm_head(h[0, -1])core.eval(logits)# Step boundary: ONE stacked head batch -> ONE tolist -> fills.- # Demo parity (engine_qwen._forward: `if (pregate or trained_state)- # and not full_layer`): stage_all+swap run after EVERY forward,- # including prefill chunks — so the last prefill token's- # predictions fill pred_inds and oh_prev, and the FIRST decode- # step consumes a real prediction at L7 instead of falling back- # to the router (a different, cleaner-but-divergent trajectory).+ # Demo parity (engine_qwen._forward: the deployment's pre-routing /+ # trained-state condition and not full_layer): stage_all+swap run+ # after EVERY forward, including prefill chunks — so the last+ # prefill token's predictions fill pred_inds and oh_prev, and the+ # FIRST decode step consumes a real prediction at L7 instead of+ # falling back to the router (a different, cleaner-but-divergent+ # trajectory).if (self._pg_stager is not None and not full_layer):self._pg_stager.stage_all()self._pg_state.swap()diff --git a/src/edge0/server/chat.py b/src/edge0/server/chat.pyindex b2f5c31..8ec9d2d 100644--- a/src/edge0/server/chat.py+++ b/src/edge0/server/chat.py@@ -136,7 +136,7 @@ class ChatSession:t0 = time.perf_counter()# Per-request clean# per-request state. A previous degenerate/truncated turn leaves- # bad pregate cross-token state and KV behind, which makes every+ # bad pre-routing cross-token state and KV behind, which makes every# later request collapse from its first token.self.engine.reset()ids = self.prompt_ids()