Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

fix(cli,models): unify serving port to 8000; repair convert-adapters and demo.py

Binb4d43c113635
diff --git a/.gitignore b/.gitignore
index b41702c..623ec53 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,11 @@ dist/
# Trained adapter artifacts (never committed; see README)
artifacts/
+# Downloaded model checkpoints (never committed; see README quick start).
+# Anchored so the source package python/src/edge0/models/ is not matched.
+/models/
+/python/models/
+
# macOS
.DS_Store
diff --git a/docs/models/edge0-35b.md b/docs/models/edge0-35b.md
index 99d611b..e6c12af 100644
--- a/docs/models/edge0-35b.md
+++ b/docs/models/edge0-35b.md
@@ -22,7 +22,7 @@ The performance profile is based on benchmarks of the current release adapter ve
| Prefill chunk | 2048 |
| Hot window | 4 |
| Streaming prefetch history | on (`prefetch_history=True`) |
-| Serving port | 8085 |
+| Serving port | 8000 |
| Measured throughput | 14.9–17.7 tok/s (M4 Pro) |
| Measured peak activation memory | ≈ 2.9 GiB |
@@ -42,7 +42,7 @@ This tier uses the `LayerOptions.staged_k4()` preset:
### Serving via the CLI
```bash
-edge0 serve /path/to/checkpoint --host 127.0.0.1 --port 8085
+edge0 serve /path/to/checkpoint --host 127.0.0.1 --port 8000
```
Optional arguments:
@@ -99,7 +99,7 @@ cfg = AutoConfig.from_pretrained("/path/to/checkpoint", name="edge0-35b")
### Non-streaming chat
```bash
-curl -s http://127.0.0.1:8085/v1/chat/completions \
+curl -s http://127.0.0.1:8000/v1/chat/completions \
-H 'Content-Type: application/json' \
-d '{
"model": "edge0-35b",
@@ -131,7 +131,7 @@ Optional request fields: `model`, `messages` (with `role`/`content`; content sup
### Streaming chat (requires Flask)
```bash
-curl -N http://127.0.0.1:8085/v1/chat/completions \
+curl -N http://127.0.0.1:8000/v1/chat/completions \
-H 'Content-Type: application/json' \
-d '{
"model": "edge0-35b",
@@ -152,7 +152,7 @@ from edge0 import AutoEngine
engine = AutoEngine.from_pretrained(
"/path/to/checkpoint",
name="edge0-35b",
- port=9090, # override the default port 8085
+ port=9090, # override the default port 8000
target_tok_s=14.0, # override the acceptance throughput target
prerouter=None, # disable the prerouter
lora="", # disable LoRA
diff --git a/docs/models/edge0-8b.md b/docs/models/edge0-8b.md
index 55c2db0..a71943f 100644
--- a/docs/models/edge0-8b.md
+++ b/docs/models/edge0-8b.md
@@ -22,7 +22,7 @@ The performance profile is based on benchmarks of the current release adapter ve
| Prefill chunk | 2048 |
| Hot window | 1 |
| Streaming prefetch history | on (`prefetch_history=True`) |
-| Serving port | 8083 |
+| Serving port | 8000 |
| Measured throughput | 23.9–25.3 tok/s (M4 Pro) |
| Measured peak activation memory | ≈ 1.0 GiB |
@@ -41,7 +41,7 @@ This tier uses the `LayerOptions.prod_k8()` preset (aligned with the reference d
### Serving via the CLI
```bash
-edge0 serve /path/to/checkpoint --host 127.0.0.1 --port 8083
+edge0 serve /path/to/checkpoint --host 127.0.0.1 --port 8000
```
Optional arguments:
@@ -99,7 +99,7 @@ cfg = AutoConfig.from_pretrained("/path/to/checkpoint", name="edge0-8b")
### Non-streaming chat
```bash
-curl -s http://127.0.0.1:8083/v1/chat/completions \
+curl -s http://127.0.0.1:8000/v1/chat/completions \
-H 'Content-Type: application/json' \
-d '{
"model": "edge0-8b",
@@ -131,7 +131,7 @@ Optional request fields: `model`, `messages` (with `role`/`content`; content sup
### Streaming chat (requires Flask)
```bash
-curl -N http://127.0.0.1:8083/v1/chat/completions \
+curl -N http://127.0.0.1:8000/v1/chat/completions \
-H 'Content-Type: application/json' \
-d '{
"model": "edge0-8b",
@@ -152,7 +152,7 @@ from edge0 import AutoEngine
engine = AutoEngine.from_pretrained(
"/path/to/checkpoint",
name="edge0-8b",
- port=9083, # override the default port 8083
+ port=9083, # override the default port 8000
target_tok_s=35.0, # override the acceptance throughput target
prerouter=None, # disable the prerouter
lora="", # disable LoRA
diff --git a/python/examples/demo.py b/python/examples/demo.py
index 686700f..236e9b6 100644
--- a/python/examples/demo.py
+++ b/python/examples/demo.py
@@ -40,6 +40,18 @@ def main() -> int:
"already run that path)")
args = ap.parse_args()
+ # ``--model <tier>`` resolves through the tier env var (same contract
+ # as ``edge0 demo``); ``--model-dir`` always wins.
+ if not args.model_dir and args.model:
+ import os
+
+ from edge0.cli import TIER_ENV
+ env = TIER_ENV.get(args.model)
+ args.model_dir = os.environ.get(env, "") if env else ""
+ if not args.model_dir:
+ ap.error(f"--model {args.model} needs ${env or '?'} set to the "
+ f"checkpoint directory (or pass --model-dir)")
+
kw = ({"prerouter": None} if args.no_prerouter
else demo_kwargs(args.model_dir, args.model))
engine = AutoEngine.from_pretrained(args.model_dir, name=args.model, **kw)
diff --git a/python/src/edge0/cli.py b/python/src/edge0/cli.py
index d2cbb73..5b3ec54 100644
--- a/python/src/edge0/cli.py
+++ b/python/src/edge0/cli.py
@@ -209,15 +209,22 @@ def cmd_serve(args) -> int:
def cmd_convert(args) -> int:
- import runpy
- sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve()
- .parents[2] / "scripts"))
- runpy.run_path(
- str(__import__("pathlib").Path(__file__).resolve().parents[2]
- / "scripts" / "convert_adapters_legacy.py"),
- run_name="__main__",
- )
- return 0
+ from pathlib import Path
+
+ scripts_dir = Path(__file__).resolve().parents[2] / "scripts"
+ if str(scripts_dir) not in sys.path:
+ sys.path.insert(0, str(scripts_dir))
+ # Delegate to the legacy converter's own main(argv) with a clean argv,
+ # so the subcommand name never leaks into its argparse (runpy would
+ # re-parse sys.argv and choke on "convert-adapters").
+ from convert_adapters_legacy import main as _convert_main
+
+ argv: list[str] = []
+ if getattr(args, "force", False):
+ argv.append("--force")
+ if getattr(args, "npz_dir", None):
+ argv += ["--npz-dir", args.npz_dir]
+ return _convert_main(argv)
def _add_engine_flags(p) -> None:
@@ -290,6 +297,11 @@ def _build_parser() -> argparse.ArgumentParser:
p = sub.add_parser("convert-adapters",
help="one-shot legacy npz -> safetensors migration")
+ p.add_argument("--force", action="store_true",
+ help="reconvert even if the artifact exists")
+ p.add_argument("--npz-dir", default=None,
+ help="directory holding the legacy npz exports "
+ "(default: $EDGE0_NPZ_DIR)")
p.set_defaults(fn=cmd_convert)
return ap
diff --git a/python/src/edge0/models/edge0_35b/__init__.py b/python/src/edge0/models/edge0_35b/__init__.py
index 3f27819..0e5bd40 100644
--- a/python/src/edge0/models/edge0_35b/__init__.py
+++ b/python/src/edge0/models/edge0_35b/__init__.py
@@ -10,7 +10,8 @@ checkpoint):
* trained prerouter: 33 heads (owners 6..38), start_layer 7,
hidden 512, fp16, consumed directly by the patched MoE block at
decode (patch_call=True, cross-token staged decode, K=4).
-* serving: port 8085; acceptance ≈13 tok/s, peak active ≈3.3 GB.
+* serving: port 8000 (the shared CLI default); acceptance ≈13 tok/s,
+ peak active ≈3.3 GB.
"""
from __future__ import annotations
@@ -64,7 +65,6 @@ class Qwen35Config(ModelConfig):
hot_window=4,
intra_staging=False,
prefetch_history=True,
- port=8085,
target_tok_s=13.0,
peak_active_mem_mb=3400.0,
)
diff --git a/python/src/edge0/models/edge0_8b/__init__.py b/python/src/edge0/models/edge0_8b/__init__.py
index 7fcc724..5453580 100644
--- a/python/src/edge0/models/edge0_8b/__init__.py
+++ b/python/src/edge0/models/edge0_8b/__init__.py
@@ -11,7 +11,8 @@ Family facts (production profile, verified against the reference checkpoint):
(explicit owners 1..22), start_layer 1, hidden 512, fp16,
feature_topk "executed", consumed inside ``BailingSparseMoE``
from ``prerouter_cache`` logits (patch_call=False).
-* serving: port 8083; acceptance ≈33 tok/s, peak active ≈1.4 GB.
+* serving: port 8000 (the shared CLI default); acceptance ≈33 tok/s,
+ peak active ≈1.4 GB.
"""
from __future__ import annotations
@@ -72,7 +73,6 @@ class Ling8BConfig(ModelConfig):
hot_window=1,
intra_staging=False,
prefetch_history=True,
- port=8083,
target_tok_s=33.0,
peak_active_mem_mb=1400.0,
)
diff --git a/python/tests/test_registry.py b/python/tests/test_registry.py
index 9cf7932..d3a61d9 100644
--- a/python/tests/test_registry.py
+++ b/python/tests/test_registry.py
@@ -65,7 +65,7 @@ def test_qwen35_profile():
name="edge0-35b", history_slots=True).history_slots is True
assert cfg.gen.temperature == 0.7
assert 248046 in cfg.gen.eos_ids
- assert cfg.port == 8085
+ assert cfg.port == 8000 # unified serving port (CLI default)
def test_ling8b_profile():
@@ -82,7 +82,7 @@ def test_ling8b_profile():
assert cfg.prerouter.patch_call is False
assert cfg.gen.repetition_penalty == 1.1
assert 156895 in cfg.gen.eos_ids
- assert cfg.port == 8083
+ assert cfg.port == 8000 # unified serving port (CLI default)
def test_override_and_reject():