Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
feat(bench): add community submission and JSON output
Demetrios Agourakis6e67349ff16e
diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.mdindex 3c583fb..77e283f 100644--- a/docs/benchmarks/README.md+++ b/docs/benchmarks/README.md@@ -15,6 +15,38 @@ numbers are welcome here — **every reporter is credited by handle**.BENCH_LONG=1 .venv/bin/python examples/bench.py edge0-8b```+ To print the discussion's ready-to-paste submission block:++ ```bash+ .venv/bin/python examples/bench.py edge0-8b --submission \+ --reporter @your-handle --storage "internal SSD" --on-battery no+ ```++ `--submission` selects the standard long prompt automatically and retains+ the benchmark's two timed runs, per-run warmup, sampling, mean decode rate,+ and maximum MLX allocator peak. Prefill is the mean of the two per-run+ rates. It rejects `BENCH_PROMPT` rather than silently submitting a custom+ workload. The default invocation and return fields remain unchanged.++ Add `--json` to emit one JSON object on stdout, with Python runtime and+ benchmark progress redirected to stderr:++ ```bash+ .venv/bin/python examples/bench.py edge0-8b --submission --json \+ --reporter @your-handle --notes "quiet system" > submission.json+ ```++ `--json` requires `--submission`. Hardware model, chip, system RAM, macOS+ and installed MLX version are detected when available. Unknown text fields+ remain empty and unknown RAM is JSON `null`; storage and power state are+ supplied by the reporter, not guessed. `--notes` accepts measurement context.+ Non-default token/warmup counts and sampling environment overrides are+ recorded in the notes; use the defaults for community comparisons.++ `peak_active_gib` here is `mlx.core.get_peak_memory()`, **not** process RSS+ or a Windows process working-set peak. JSON uses the existing submission+ block fields; this command does not append to or change `results.csv`.+2. Post your numbers in the pinned[Hardware Benchmarks discussion](https://github.com/Edge0-AI/Edge0/discussions/142),using the submission block there.diff --git a/python/examples/bench.py b/python/examples/bench.pyindex e2b5058..5256b0a 100644--- a/python/examples/bench.py+++ b/python/examples/bench.py@@ -2,7 +2,7 @@"""edge0 speed + memory benchmark (any tier).Runs the same measurement shape as the deployment's bench: prefill ->-sampled warmup steps -> timed sampled decode -> report tok/s and MLX+greedy warmup steps -> timed sampled decode -> report tok/s and MLXallocator peak memory (`mlx.core.get_peak_memory`). Works for bothtiers; the tier is auto-detected from the checkpoint (or forced via themodel name).@@ -11,6 +11,8 @@ Usage:python examples/bench.py /path/to/model [--ntok 200] [--warmup 10]python examples/bench.py edge0-35b # via $EDGE0_35B_MODELBENCH_LONG=1 python examples/bench.py edge0-35b # ~3k-token prefill+ python examples/bench.py edge0-35b --submission --reporter @you+ python examples/bench.py edge0-35b --submission --json > submission.jsonEnv (sampling knobs, defaults follow the tier's GenerationConfig):BENCH_TEMP / BENCH_PROMPT / BENCH_SEED@@ -22,14 +24,15 @@ Env (sampling knobs, defaults follow the tier's GenerationConfig):from __future__ import annotationsimport argparse+from contextlib import nullcontext, redirect_stdout+from importlib.metadata import PackageNotFoundError, version+import jsonimport os+import platform+import subprocessimport sysimport time-from edge0 import AutoEngine-from edge0.backends import core-from edge0.sampling import sample-PROMPTS = {"edge0-35b": "什么是混合专家模型(MoE)?它和普通 Transformer 有什么区别?","edge0-8b": "9.11 和 9.8 哪个大?请仔细比较。",@@ -70,9 +73,12 @@ def _prompt_for(engine) -> str:return PROMPTS.get(engine.name, DEFAULT_PROMPT)-def run_bench(engine, ntok: int, warmup: int) -> dict:+def run_bench(engine, ntok: int, warmup: int, *, submission: bool = False) -> dict:+ from edge0.backends import core+ from edge0.sampling import sample+tok = engine._tok- prompt = _prompt_for(engine)+ prompt = _long_prompt() if submission else _prompt_for(engine)if hasattr(engine, "encode_chat"):ids = engine.encode_chat([{"role": "user", "content": prompt}], think=False)@@ -102,7 +108,7 @@ def run_bench(engine, ntok: int, warmup: int) -> dict:core.random.seed(int(seed))history = list(ids)- # warmup: sampled, untimed (first tokens are dominated by expert+ # warmup: greedy, untimed (first tokens are dominated by expert# cold-starts; the timed window is the steady state)for _ in range(warmup):tid = int(core.argmax(logits, axis=-1).item())@@ -123,6 +129,8 @@ def run_bench(engine, ntok: int, warmup: int) -> dict:ntok=len(out), peak_gib=peak_gib,tok_s=len(out) / t_decode if t_decode else 0.0))pf_tps = (len(ids) / t_prefill) if t_prefill else 0.0+ if submission:+ results[-1]["prefill_tok_s"] = pf_tpsprint(f"prompt={len(ids)} tok prefill={t_prefill:.2f}s "f"({pf_tps:.0f} tok/s) "f"decode={len(out)}/{t_decode:.2f}s "@@ -132,7 +140,72 @@ def run_bench(engine, ntok: int, warmup: int) -> dict:max_peak = max(r["peak_gib"] for r in results)print(f"[bench] mean tok/s={mean_ts:.1f} mlx_allocator_peak<={max_peak:.2f} GiB",flush=True)- return {"runs": results, "mean_tok_s": mean_ts, "peak_gib": max_peak}+ report = {"runs": results, "mean_tok_s": mean_ts, "peak_gib": max_peak}+ if submission:+ report["mean_prefill_tok_s"] = sum(r["prefill_tok_s"] for r in results) / len(results)+ return report+++def _sysctl(name: str) -> str:+ try:+ return subprocess.run(["sysctl", "-n", name], capture_output=True,+ text=True, check=True, timeout=3).stdout.strip()+ except (OSError, subprocess.SubprocessError):+ return ""+++def _hardware_metadata() -> dict:+ try:+ mlx_version = version("mlx")+ except PackageNotFoundError:+ mlx_version = ""+ ram_gb = None+ device = chip = ""+ if platform.system() == "Darwin":+ device = _sysctl("hw.model")+ chip = _sysctl("machdep.cpu.brand_string")+ try:+ memory = int(_sysctl("hw.memsize"))+ if memory > 0:+ ram_gb = memory / (1024 ** 3)+ except ValueError:+ pass+ if platform.system() == "Darwin":+ os_name = "macOS " + platform.mac_ver()[0]+ else:+ os_name = platform.system() + " " + platform.release()+ return {"device": device, "chip": chip, "ram_gb": ram_gb,+ "os": os_name.strip(), "mlx_version": mlx_version}+++def _single_line(value) -> str:+ return " ".join(str(value).split())+++def _submission_block(tier: str, report: dict, args) -> dict:+ metadata = _hardware_metadata()+ notes = [args.notes, "BENCH_LONG; 2 timed runs; "+ f"{args.warmup} greedy warmup steps/run; {args.ntok} sampled decode steps/run; "+ "memory=MLX allocator peak (not process RSS)"]+ for key in ("BENCH_TEMP", "BENCH_SEED"):+ if key in os.environ:+ notes.append(f"{key}={os.environ[key]}")+ return {+ "tier": tier,+ "runtime": "python-mlx",+ "device": _single_line(metadata.get("device", "")),+ "chip": _single_line(metadata.get("chip", "")),+ "ram_gb": metadata.get("ram_gb"),+ "os": _single_line(metadata.get("os", "")),+ "storage": _single_line(args.storage),+ "mlx_version": _single_line(metadata.get("mlx_version", "")),+ "decode_tok_s": report["mean_tok_s"],+ "prefill_tok_s": report["mean_prefill_tok_s"],+ "peak_active_gib": report["peak_gib"],+ "on_battery": args.on_battery,+ "reporter": _single_line(args.reporter),+ "notes": _single_line("; ".join(note for note in notes if note)),+ }def main() -> int:@@ -141,23 +214,62 @@ def main() -> int:ap.add_argument("--ntok", type=int,default=int(os.environ.get("BENCH_NTOK", "200")))ap.add_argument("--warmup", type=int, default=10)+ ap.add_argument("--submission", action="store_true",+ help="use BENCH_LONG and print the community submission block")+ ap.add_argument("--json", action="store_true",+ help="with --submission, emit only JSON on stdout; progress goes to stderr")+ ap.add_argument("--reporter", default="", help="GitHub handle for the submission")+ ap.add_argument("--storage", default="", help="storage containing the checkpoint (not auto-detected)")+ ap.add_argument("--on-battery", choices=("yes", "no"), default="",+ help="power state during measurement (not auto-detected)")+ ap.add_argument("--notes", default="", help="background load, throttling or other context")args = ap.parse_args()+ if args.json and not args.submission:+ ap.error("--json requires --submission")+ if args.ntok <= 0 or args.warmup < 0:+ ap.error("--ntok must be positive and --warmup must be nonnegative")+ if args.submission and os.environ.get("BENCH_PROMPT"):+ ap.error("--submission requires the standard long prompt; unset BENCH_PROMPT")# tier name -> checkpoint dir via $EDGE0_<TIER>_MODEL (cli parity)model = args.modelname = None- if model.startswith("edge0-"):- from edge0.cli import TIER_ENV- env = os.environ.get(TIER_ENV.get(model, ""))- if not env:- raise SystemExit(- f"set {TIER_ENV.get(model, 'EDGE0_<TIER>_MODEL')} to the "- f"checkpoint directory for {model}, or pass the dir itself")- model = env- engine = AutoEngine.from_pretrained(model, name=name)- print(f"[bench] tier={engine.name} ntok={args.ntok} warmup={args.warmup}",- flush=True)- run_bench(engine, args.ntok, args.warmup)- engine.close()+ with redirect_stdout(sys.stderr) if args.json else nullcontext():+ from edge0 import AutoEngine++ if model.startswith("edge0-"):+ from edge0.cli import TIER_ENV+ env = os.environ.get(TIER_ENV.get(model, ""))+ if not env:+ raise SystemExit(+ f"set {TIER_ENV.get(model, 'EDGE0_<TIER>_MODEL')} to the "+ f"checkpoint directory for {model}, or pass the dir itself")+ model = env+ engine = AutoEngine.from_pretrained(model, name=name)+ try:+ print(f"[bench] tier={engine.name} ntok={args.ntok} warmup={args.warmup}",+ flush=True)+ if args.submission:+ report = run_bench(engine, args.ntok, args.warmup, submission=True)+ else:+ report = run_bench(engine, args.ntok, args.warmup)+ tier = engine.name+ finally:+ engine.close()+ if args.submission:+ block = _submission_block(tier, report, args)+ payload = json.dumps(block, ensure_ascii=False, allow_nan=False)+ if args.json:+ print(payload)+ else:+ print()+ for key, value in block.items():+ if value is None:+ text = ""+ elif isinstance(value, float):+ text = f"{value:.2f}"+ else:+ text = str(value)+ print(f"{key + ':':<18} {text}".rstrip())return 0diff --git a/python/tests/test_bench_submission.py b/python/tests/test_bench_submission.pynew file mode 100644index 0000000..cb1510a--- /dev/null+++ b/python/tests/test_bench_submission.py@@ -0,0 +1,232 @@+"""CLI/report contracts for the MLX benchmark harness; no scalar numerical oracle."""++import importlib.util+import json+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import Mock++import pytest+++def bench():+ path = Path(__file__).resolve().parents[1] / "examples" / "bench.py"+ spec = importlib.util.spec_from_file_location("bench_example", path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def invocation(bench, monkeypatch):+ from edge0 import AutoEngine++ for key in ("BENCH_PROMPT", "BENCH_LONG", "BENCH_NTOK"):+ monkeypatch.delenv(key, raising=False)+ engine = SimpleNamespace(name="edge0-35b", close=Mock())+ factory = Mock(return_value=engine)+ monkeypatch.setattr(AutoEngine, "from_pretrained", factory)+ report = {"runs": [], "mean_tok_s": 7.5, "peak_gib": 3.25,+ "mean_prefill_tok_s": 125.5}++ def measured(*args, **kwargs):+ print("measurement progress")+ return report++ run = Mock(side_effect=measured)+ monkeypatch.setattr(bench, "run_bench", run)+ return engine, factory, run, report+++def test_submission_json_is_one_document_and_matches_template(+ bench, invocation, monkeypatch, capsys):+ engine, factory, run, report = invocation+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {+ "device": "Mac fixture", "chip": "Apple fixture", "ram_gb": None,+ "os": "macOS fixture", "mlx_version": "fixture"})+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission", "--json",+ "--reporter", "@tester", "--storage", "external SSD",+ "--on-battery", "no", "--notes", "first\nsecond"])+ assert bench.main() == 0+ output = capsys.readouterr()+ data = json.loads(output.out)+ assert len(output.out.splitlines()) == 1+ assert "measurement progress" in output.err+ assert data["tier"] == engine.name+ assert data["runtime"] == "python-mlx"+ assert data["decode_tok_s"] == report["mean_tok_s"]+ assert data["prefill_tok_s"] == report["mean_prefill_tok_s"]+ assert data["peak_active_gib"] == report["peak_gib"]+ assert data["ram_gb"] is None+ assert data["reporter"] == "@tester"+ assert data["on_battery"] == "no"+ assert data["storage"] == "external SSD"+ assert "first second" in data["notes"]+ assert "MLX allocator" in data["notes"]+ assert set(data) == {"tier", "runtime", "device", "chip", "ram_gb", "os", "storage",+ "mlx_version", "decode_tok_s", "prefill_tok_s", "peak_active_gib",+ "on_battery", "reporter", "notes"}+ run.assert_called_once_with(engine, 200, 10, submission=True)+ factory.assert_called_once_with("checkpoint", name=None)+ engine.close.assert_called_once()+++def test_submission_text_has_pasteable_colon_fields(bench, invocation, monkeypatch, capsys):+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {})+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission"])+ assert bench.main() == 0+ text = capsys.readouterr().out+ assert "tier:" in text and "runtime:" in text and "reporter:" in text+ assert "None" not in text+++def test_default_cli_retains_existing_benchmark_call(bench, invocation, monkeypatch, capsys):+ engine, _, run, _ = invocation+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint"])+ assert bench.main() == 0+ run.assert_called_once_with(engine, 200, 10)+ engine.close.assert_called_once()+ output = capsys.readouterr()+ assert "[bench] tier=edge0-35b" in output.out+ assert "measurement progress" in output.out+ assert output.err == ""++[email protected]("flags", [["--json"], ["--submission", "--ntok", "0"],+ ["--submission", "--warmup", "-1"],+ ["--submission", "--on-battery", "maybe"]])+def test_invalid_flags_fail_before_loading_model(bench, invocation, monkeypatch, flags):+ _, factory, _, _ = invocation+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", *flags])+ with pytest.raises(SystemExit) as error:+ bench.main()+ assert error.value.code == 2+ factory.assert_not_called()+++def test_submission_rejects_custom_prompt_before_model_load(bench, invocation, monkeypatch):+ _, factory, _, _ = invocation+ monkeypatch.setenv("BENCH_PROMPT", "custom prompt")+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission"])+ with pytest.raises(SystemExit) as error:+ bench.main()+ assert error.value.code == 2+ factory.assert_not_called()+++def test_json_failure_does_not_emit_partial_document_and_closes_engine(+ bench, invocation, monkeypatch, capsys):+ engine, _, run, _ = invocation+ run.side_effect = RuntimeError("fixture failure")+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission", "--json"])+ with pytest.raises(RuntimeError, match="fixture failure"):+ bench.main()+ assert capsys.readouterr().out == ""+ engine.close.assert_called_once()+++def test_json_routes_engine_load_and_cleanup_progress_to_stderr(+ bench, invocation, monkeypatch, capsys):+ engine, factory, _, _ = invocation++ def load(*args, **kwargs):+ print("loading fixture")+ return engine++ factory.side_effect = load+ engine.close.side_effect = lambda: print("closing fixture")+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {})+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission", "--json"])+ assert bench.main() == 0+ output = capsys.readouterr()+ assert json.loads(output.out)["tier"] == engine.name+ assert "loading fixture" in output.err and "closing fixture" in output.err+++def test_json_rejects_nonfinite_measurements(bench, invocation, monkeypatch, capsys):+ _, _, _, report = invocation+ report["mean_tok_s"] = float("nan")+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {})+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission", "--json"])+ with pytest.raises(ValueError):+ bench.main()+ assert capsys.readouterr().out == ""++[email protected]("submission", [False, True])+def test_mlx_harness_retains_reset_warmup_decode_shape(bench, monkeypatch, submission):+ from edge0.backends import core++ for key in ("BENCH_TEMP", "BENCH_SEED", "BENCH_PROMPT", "BENCH_LONG"):+ monkeypatch.delenv(key, raising=False)++ class TinyMLXEngine:+ name = "edge0-35b"+ _tok = None+ cfg = SimpleNamespace(gen=SimpleNamespace(temperature=0.7, top_k=2, top_p=1.0,+ repetition_penalty=1.0))++ def __init__(self):+ self.resets = 0+ self.steps = 0+ self.prefills = []+ self.prompts = []+ self.logits = core.zeros((4,))++ def encode_chat(self, messages, *, think):+ assert think is False+ self.prompts.append(messages[0]["content"])+ return [1, 2, 3]++ def reset(self):+ self.resets += 1++ def prefill(self, ids):+ self.prefills.append(list(ids))+ core.eval(self.logits)++ def next_logits(self):+ return self.logits++ def step(self, token):+ self.steps += 1+ return self.logits++ engine = TinyMLXEngine()+ report = bench.run_bench(engine, 2, 1, submission=submission)+ assert engine.resets == 2+ assert engine.steps == 6+ assert engine.prefills == [[1, 2, 3], [1, 2, 3]]+ assert engine.prompts == [bench._long_prompt() if submission else bench.PROMPTS[engine.name]]+ assert len(report["runs"]) == 2+ keys = {"runs", "mean_tok_s", "peak_gib"}+ run_keys = {"prefill_s", "decode_s", "ntok", "peak_gib", "tok_s"}+ if submission:+ keys.add("mean_prefill_tok_s")+ run_keys.add("prefill_tok_s")+ assert set(report) == keys+ assert all(set(run) == run_keys for run in report["runs"])+ json.dumps(report, allow_nan=False)+++def test_submission_metadata_failure_leaves_unknown_values(bench, monkeypatch):+ def unavailable(*args, **kwargs):+ raise FileNotFoundError("fixture")++ monkeypatch.setattr(bench.platform, "system", lambda: "Darwin")+ monkeypatch.setattr(bench.platform, "mac_ver", lambda: ("fixture", (), ""))+ monkeypatch.setattr(bench.subprocess, "run", unavailable)+ monkeypatch.setattr(bench, "version", Mock(side_effect=bench.PackageNotFoundError))+ assert bench._hardware_metadata() == {"device": "", "chip": "", "ram_gb": None,+ "os": "macOS fixture", "mlx_version": ""}+++def test_submission_alias_uses_existing_environment_resolution(bench, invocation, monkeypatch):+ engine, factory, _, _ = invocation+ monkeypatch.setenv("EDGE0_35B_MODEL", "fixture-checkpoint")+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {})+ monkeypatch.setattr("sys.argv", ["bench.py", "edge0-35b", "--submission", "--json"])+ assert bench.main() == 0+ factory.assert_called_once_with("fixture-checkpoint", name=None)+ engine.close.assert_called_once()