Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
fix(cli,models): unify serving port to 8000; repair convert-adapters and demo.py
Binb4d43c113635
diff --git a/.gitignore b/.gitignoreindex b41702c..623ec53 100644--- a/.gitignore+++ b/.gitignore@@ -13,6 +13,11 @@ dist/# Trained adapter artifacts (never committed; see README)artifacts/+# Downloaded model checkpoints (never committed; see README quick start).+# Anchored so the source package python/src/edge0/models/ is not matched.+/models/+/python/models/+# macOS.DS_Storediff --git a/docs/models/edge0-35b.md b/docs/models/edge0-35b.mdindex 99d611b..e6c12af 100644--- a/docs/models/edge0-35b.md+++ b/docs/models/edge0-35b.md@@ -22,7 +22,7 @@ The performance profile is based on benchmarks of the current release adapter ve| Prefill chunk | 2048 || Hot window | 4 || Streaming prefetch history | on (`prefetch_history=True`) |-| Serving port | 8085 |+| Serving port | 8000 || Measured throughput | 14.9–17.7 tok/s (M4 Pro) || Measured peak activation memory | ≈ 2.9 GiB |@@ -42,7 +42,7 @@ This tier uses the `LayerOptions.staged_k4()` preset:### Serving via the CLI```bash-edge0 serve /path/to/checkpoint --host 127.0.0.1 --port 8085+edge0 serve /path/to/checkpoint --host 127.0.0.1 --port 8000```Optional arguments:@@ -99,7 +99,7 @@ cfg = AutoConfig.from_pretrained("/path/to/checkpoint", name="edge0-35b")### Non-streaming chat```bash-curl -s http://127.0.0.1:8085/v1/chat/completions \+curl -s http://127.0.0.1:8000/v1/chat/completions \-H 'Content-Type: application/json' \-d '{"model": "edge0-35b",@@ -131,7 +131,7 @@ Optional request fields: `model`, `messages` (with `role`/`content`; content sup### Streaming chat (requires Flask)```bash-curl -N http://127.0.0.1:8085/v1/chat/completions \+curl -N http://127.0.0.1:8000/v1/chat/completions \-H 'Content-Type: application/json' \-d '{"model": "edge0-35b",@@ -152,7 +152,7 @@ from edge0 import AutoEngineengine = AutoEngine.from_pretrained("/path/to/checkpoint",name="edge0-35b",- port=9090, # override the default port 8085+ port=9090, # override the default port 8000target_tok_s=14.0, # override the acceptance throughput targetprerouter=None, # disable the prerouterlora="", # disable LoRAdiff --git a/docs/models/edge0-8b.md b/docs/models/edge0-8b.mdindex 55c2db0..a71943f 100644--- a/docs/models/edge0-8b.md+++ b/docs/models/edge0-8b.md@@ -22,7 +22,7 @@ The performance profile is based on benchmarks of the current release adapter ve| Prefill chunk | 2048 || Hot window | 1 || Streaming prefetch history | on (`prefetch_history=True`) |-| Serving port | 8083 |+| Serving port | 8000 || Measured throughput | 23.9–25.3 tok/s (M4 Pro) || Measured peak activation memory | ≈ 1.0 GiB |@@ -41,7 +41,7 @@ This tier uses the `LayerOptions.prod_k8()` preset (aligned with the reference d### Serving via the CLI```bash-edge0 serve /path/to/checkpoint --host 127.0.0.1 --port 8083+edge0 serve /path/to/checkpoint --host 127.0.0.1 --port 8000```Optional arguments:@@ -99,7 +99,7 @@ cfg = AutoConfig.from_pretrained("/path/to/checkpoint", name="edge0-8b")### Non-streaming chat```bash-curl -s http://127.0.0.1:8083/v1/chat/completions \+curl -s http://127.0.0.1:8000/v1/chat/completions \-H 'Content-Type: application/json' \-d '{"model": "edge0-8b",@@ -131,7 +131,7 @@ Optional request fields: `model`, `messages` (with `role`/`content`; content sup### Streaming chat (requires Flask)```bash-curl -N http://127.0.0.1:8083/v1/chat/completions \+curl -N http://127.0.0.1:8000/v1/chat/completions \-H 'Content-Type: application/json' \-d '{"model": "edge0-8b",@@ -152,7 +152,7 @@ from edge0 import AutoEngineengine = AutoEngine.from_pretrained("/path/to/checkpoint",name="edge0-8b",- port=9083, # override the default port 8083+ port=9083, # override the default port 8000target_tok_s=35.0, # override the acceptance throughput targetprerouter=None, # disable the prerouterlora="", # disable LoRAdiff --git a/python/examples/demo.py b/python/examples/demo.pyindex 686700f..236e9b6 100644--- a/python/examples/demo.py+++ b/python/examples/demo.py@@ -40,6 +40,18 @@ def main() -> int:"already run that path)")args = ap.parse_args()+ # ``--model <tier>`` resolves through the tier env var (same contract+ # as ``edge0 demo``); ``--model-dir`` always wins.+ if not args.model_dir and args.model:+ import os++ from edge0.cli import TIER_ENV+ env = TIER_ENV.get(args.model)+ args.model_dir = os.environ.get(env, "") if env else ""+ if not args.model_dir:+ ap.error(f"--model {args.model} needs ${env or '?'} set to the "+ f"checkpoint directory (or pass --model-dir)")+kw = ({"prerouter": None} if args.no_prerouterelse demo_kwargs(args.model_dir, args.model))engine = AutoEngine.from_pretrained(args.model_dir, name=args.model, **kw)diff --git a/python/src/edge0/cli.py b/python/src/edge0/cli.pyindex d2cbb73..5b3ec54 100644--- a/python/src/edge0/cli.py+++ b/python/src/edge0/cli.py@@ -209,15 +209,22 @@ def cmd_serve(args) -> int:def cmd_convert(args) -> int:- import runpy- sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve()- .parents[2] / "scripts"))- runpy.run_path(- str(__import__("pathlib").Path(__file__).resolve().parents[2]- / "scripts" / "convert_adapters_legacy.py"),- run_name="__main__",- )- return 0+ from pathlib import Path++ scripts_dir = Path(__file__).resolve().parents[2] / "scripts"+ if str(scripts_dir) not in sys.path:+ sys.path.insert(0, str(scripts_dir))+ # Delegate to the legacy converter's own main(argv) with a clean argv,+ # so the subcommand name never leaks into its argparse (runpy would+ # re-parse sys.argv and choke on "convert-adapters").+ from convert_adapters_legacy import main as _convert_main++ argv: list[str] = []+ if getattr(args, "force", False):+ argv.append("--force")+ if getattr(args, "npz_dir", None):+ argv += ["--npz-dir", args.npz_dir]+ return _convert_main(argv)def _add_engine_flags(p) -> None:@@ -290,6 +297,11 @@ def _build_parser() -> argparse.ArgumentParser:p = sub.add_parser("convert-adapters",help="one-shot legacy npz -> safetensors migration")+ p.add_argument("--force", action="store_true",+ help="reconvert even if the artifact exists")+ p.add_argument("--npz-dir", default=None,+ help="directory holding the legacy npz exports "+ "(default: $EDGE0_NPZ_DIR)")p.set_defaults(fn=cmd_convert)return apdiff --git a/python/src/edge0/models/edge0_35b/__init__.py b/python/src/edge0/models/edge0_35b/__init__.pyindex 3f27819..0e5bd40 100644--- a/python/src/edge0/models/edge0_35b/__init__.py+++ b/python/src/edge0/models/edge0_35b/__init__.py@@ -10,7 +10,8 @@ checkpoint):* trained prerouter: 33 heads (owners 6..38), start_layer 7,hidden 512, fp16, consumed directly by the patched MoE block atdecode (patch_call=True, cross-token staged decode, K=4).-* serving: port 8085; acceptance ≈13 tok/s, peak active ≈3.3 GB.+* serving: port 8000 (the shared CLI default); acceptance ≈13 tok/s,+ peak active ≈3.3 GB."""from __future__ import annotations@@ -64,7 +65,6 @@ class Qwen35Config(ModelConfig):hot_window=4,intra_staging=False,prefetch_history=True,- port=8085,target_tok_s=13.0,peak_active_mem_mb=3400.0,)diff --git a/python/src/edge0/models/edge0_8b/__init__.py b/python/src/edge0/models/edge0_8b/__init__.pyindex 7fcc724..5453580 100644--- a/python/src/edge0/models/edge0_8b/__init__.py+++ b/python/src/edge0/models/edge0_8b/__init__.py@@ -11,7 +11,8 @@ Family facts (production profile, verified against the reference checkpoint):(explicit owners 1..22), start_layer 1, hidden 512, fp16,feature_topk "executed", consumed inside ``BailingSparseMoE``from ``prerouter_cache`` logits (patch_call=False).-* serving: port 8083; acceptance ≈33 tok/s, peak active ≈1.4 GB.+* serving: port 8000 (the shared CLI default); acceptance ≈33 tok/s,+ peak active ≈1.4 GB."""from __future__ import annotations@@ -72,7 +73,6 @@ class Ling8BConfig(ModelConfig):hot_window=1,intra_staging=False,prefetch_history=True,- port=8083,target_tok_s=33.0,peak_active_mem_mb=1400.0,)diff --git a/python/tests/test_registry.py b/python/tests/test_registry.pyindex 9cf7932..d3a61d9 100644--- a/python/tests/test_registry.py+++ b/python/tests/test_registry.py@@ -65,7 +65,7 @@ def test_qwen35_profile():name="edge0-35b", history_slots=True).history_slots is Trueassert cfg.gen.temperature == 0.7assert 248046 in cfg.gen.eos_ids- assert cfg.port == 8085+ assert cfg.port == 8000 # unified serving port (CLI default)def test_ling8b_profile():@@ -82,7 +82,7 @@ def test_ling8b_profile():assert cfg.prerouter.patch_call is Falseassert cfg.gen.repetition_penalty == 1.1assert 156895 in cfg.gen.eos_ids- assert cfg.port == 8083+ assert cfg.port == 8000 # unified serving port (CLI default)def test_override_and_reject():