Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
round9 (35b) + round6 (10b) adapters; co-located layout; strip vision tower
linyubupab73af008b2a6
diff --git a/scripts/strip_vision_weights.py b/scripts/strip_vision_weights.pynew file mode 100644index 0000000..ce87800--- /dev/null+++ b/scripts/strip_vision_weights.py@@ -0,0 +1,94 @@+#!/usr/bin/env python3+"""Strip the unused vision-tower weights from an edge0-35b checkpoint.++The qwen35 checkpoint ships a 333-tensor / ~0.9GB vision tower+(``vision_tower.*``) that the text-only engine never loads — the model's+sanitize() already skips these keys, so removing them from the+safetensors shards is pure disk/memory savings. Rewrites shard 1 in+place (backup first), updates model.safetensors.index.json.++Usage: python scripts/strip_vision_weights.py <model_dir> [--no-backup]+"""+from __future__ import annotations++import argparse+import json+import shutil+import struct+from pathlib import Path+++def read_shard(path: Path):+ with open(path, "rb") as f:+ n = struct.unpack("<Q", f.read(8))[0]+ hdr = json.loads(f.read(n))+ data_start = 8 + n+ return hdr, data_start+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("model_dir")+ ap.add_argument("--no-backup", action="store_true")+ args = ap.parse_args()+ d = Path(args.model_dir)+ idx_path = d / "model.safetensors.index.json"+ idx = json.load(open(idx_path))+ wm = idx["weight_map"]+ vs = {k for k in wm if "visual" in k or "vision" in k}+ if not vs:+ print("no vision weights found — nothing to strip")+ return 0++ shards = sorted({wm[k] for k in vs})+ for shard_name in shards:+ path = d / shard_name+ hdr, data_start = read_shard(path)+ drop = {k for k in vs if wm[k] == shard_name}+ keep_hdr = {k: v for k, v in hdr.items()+ if k not in drop and "data_offsets" in v}+ # rewrite: new offsets (contiguous from original data_start)+ out = bytearray()+ new_hdr = {}+ off = 0+ with open(path, "rb") as f:+ f.seek(data_start)+ for k in sorted(keep_hdr, key=lambda k: keep_hdr[k]["data_offsets"][0]):+ e = dict(keep_hdr[k])+ b0, b1 = e["data_offsets"]+ f.seek(data_start + b0)+ blob = f.read(b1 - b0)+ out += blob+ e["data_offsets"] = [off, off + len(blob)]+ off += len(blob)+ new_hdr[k] = e+ # non-tensor entries (e.g. __metadata__) pass through+ for k, v in hdr.items():+ if k not in keep_hdr and "data_offsets" not in v and k not in drop:+ new_hdr[k] = v+ hdr_bytes = json.dumps(new_hdr).encode()+ pad = (8 - (len(hdr_bytes) % 8)) % 8+ hdr_bytes += b" " * pad+ if not args.no_backup:+ shutil.copy2(path, path.with_suffix(".safetensors.bak_vision"))+ with open(path, "wb") as f:+ f.write(struct.pack("<Q", len(hdr_bytes)))+ f.write(hdr_bytes)+ f.write(out)+ saved = sum(hdr[k]["data_offsets"][1] - hdr[k]["data_offsets"][0]+ for k in drop)+ print(f"{shard_name}: dropped {len(drop)} tensors, "+ f"{saved/1e9:.2f} GB, new size {path.stat().st_size/1e9:.2f} GB")++ # index: remove vision keys+ idx["weight_map"] = {k: v for k, v in wm.items() if k not in vs}+ if not args.no_backup:+ shutil.copy2(idx_path, idx_path.with_suffix(".json.bak_vision"))+ json.dump(idx, open(idx_path, "w"), indent=2)+ print(f"index: {len(idx['weight_map'])} keys remain "+ f"({len(vs)} vision keys removed)")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/src/edge0/models/base.py b/src/edge0/models/base.pyindex ad3bac8..e0ca224 100644--- a/src/edge0/models/base.py+++ b/src/edge0/models/base.py@@ -25,8 +25,14 @@ from edge0.streaming.options import LayerOptionsARTIFACTS_DIR = Path(__file__).resolve().parents[3] / "artifacts"-def artifact(name: str) -> str:- """Absolute path of one converted adapter file under ``artifacts/``."""+def artifact(name: str, model_dir: str | None = None) -> str:+ """Path of one adapter file: the model's own directory first+ (v7-deploy layout — model and its adapters side by side in ONE+ dir), falling back to ``artifacts/`` (legacy conversion cache)."""+ if model_dir:+ cand = Path(model_dir) / name+ if cand.is_file():+ return str(cand)return str(ARTIFACTS_DIR / name)diff --git a/src/edge0/models/edge0_10b/__init__.py b/src/edge0/models/edge0_10b/__init__.pyindex 155bf24..1d41db4 100644--- a/src/edge0/models/edge0_10b/__init__.py+++ b/src/edge0/models/edge0_10b/__init__.py@@ -50,12 +50,13 @@ class Ling10BConfig(ModelConfig):kind=RouterKind.SIGMOID_GROUP,start_layer=7, hidden=512, dtype="fp16",feature_topk="executed",- owners=tuple(range(6, 23)),- weights_file=artifact("prerouter_edge0_10b.safetensors"),+ owners=tuple(range(7, 23)),+ weights_file=artifact(+ "prerouter_edge0_10b_round6.safetensors", model_dir),patch_call=False,),prerouter_top_k=8,- lora=artifact("lora_edge0_10b.safetensors"),+ lora=artifact("lora_edge0_10b_round6.safetensors", model_dir),lora_r=16, lora_alpha=32.0,gen=GenerationConfig(temperature=0.7, top_p=0.95, top_k=64,diff --git a/src/edge0/models/edge0_35b/__init__.py b/src/edge0/models/edge0_35b/__init__.pyindex 90805a4..ff39fb5 100644--- a/src/edge0/models/edge0_35b/__init__.py+++ b/src/edge0/models/edge0_35b/__init__.py@@ -49,11 +49,12 @@ class Qwen35Config(ModelConfig):kind=RouterKind.SOFTMAX_TOPK,start_layer=7, hidden=512, dtype="fp16",feature_topk="executed",- weights_file=artifact("prerouter_edge0_35b_k4.safetensors"),+ weights_file=artifact(+ "prerouter_edge0_35b.safetensors", model_dir),patch_call=True,),prerouter_top_k=4,- lora=artifact("lora_edge0_35b_k4.safetensors"),+ lora=artifact("lora_edge0_35b.safetensors", model_dir),lora_r=16, lora_alpha=32.0,gen=GenerationConfig(temperature=0.6, top_p=0.95, top_k=64,