Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

feat(bench): add community submission and JSON output

Demetrios Agourakis6e67349ff16e
diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md
index 3c583fb..77e283f 100644
--- a/docs/benchmarks/README.md
+++ b/docs/benchmarks/README.md
@@ -15,6 +15,38 @@ numbers are welcome here — **every reporter is credited by handle**.
BENCH_LONG=1 .venv/bin/python examples/bench.py edge0-8b
```
+ To print the discussion's ready-to-paste submission block:
+
+ ```bash
+ .venv/bin/python examples/bench.py edge0-8b --submission \
+ --reporter @your-handle --storage "internal SSD" --on-battery no
+ ```
+
+ `--submission` selects the standard long prompt automatically and retains
+ the benchmark's two timed runs, per-run warmup, sampling, mean decode rate,
+ and maximum MLX allocator peak. Prefill is the mean of the two per-run
+ rates. It rejects `BENCH_PROMPT` rather than silently submitting a custom
+ workload. The default invocation and return fields remain unchanged.
+
+ Add `--json` to emit one JSON object on stdout, with Python runtime and
+ benchmark progress redirected to stderr:
+
+ ```bash
+ .venv/bin/python examples/bench.py edge0-8b --submission --json \
+ --reporter @your-handle --notes "quiet system" > submission.json
+ ```
+
+ `--json` requires `--submission`. Hardware model, chip, system RAM, macOS
+ and installed MLX version are detected when available. Unknown text fields
+ remain empty and unknown RAM is JSON `null`; storage and power state are
+ supplied by the reporter, not guessed. `--notes` accepts measurement context.
+ Non-default token/warmup counts and sampling environment overrides are
+ recorded in the notes; use the defaults for community comparisons.
+
+ `peak_active_gib` here is `mlx.core.get_peak_memory()`, **not** process RSS
+ or a Windows process working-set peak. JSON uses the existing submission
+ block fields; this command does not append to or change `results.csv`.
+
2. Post your numbers in the pinned
[Hardware Benchmarks discussion](https://github.com/Edge0-AI/Edge0/discussions/142),
using the submission block there.
diff --git a/python/examples/bench.py b/python/examples/bench.py
index e2b5058..5256b0a 100644
--- a/python/examples/bench.py
+++ b/python/examples/bench.py
@@ -2,7 +2,7 @@
"""edge0 speed + memory benchmark (any tier).
Runs the same measurement shape as the deployment's bench: prefill ->
-sampled warmup steps -> timed sampled decode -> report tok/s and MLX
+greedy warmup steps -> timed sampled decode -> report tok/s and MLX
allocator peak memory (`mlx.core.get_peak_memory`). Works for both
tiers; the tier is auto-detected from the checkpoint (or forced via the
model name).
@@ -11,6 +11,8 @@ Usage:
python examples/bench.py /path/to/model [--ntok 200] [--warmup 10]
python examples/bench.py edge0-35b # via $EDGE0_35B_MODEL
BENCH_LONG=1 python examples/bench.py edge0-35b # ~3k-token prefill
+ python examples/bench.py edge0-35b --submission --reporter @you
+ python examples/bench.py edge0-35b --submission --json > submission.json
Env (sampling knobs, defaults follow the tier's GenerationConfig):
BENCH_TEMP / BENCH_PROMPT / BENCH_SEED
@@ -22,14 +24,15 @@ Env (sampling knobs, defaults follow the tier's GenerationConfig):
from __future__ import annotations
import argparse
+from contextlib import nullcontext, redirect_stdout
+from importlib.metadata import PackageNotFoundError, version
+import json
import os
+import platform
+import subprocess
import sys
import time
-from edge0 import AutoEngine
-from edge0.backends import core
-from edge0.sampling import sample
-
PROMPTS = {
"edge0-35b": "什么是混合专家模型(MoE)?它和普通 Transformer 有什么区别?",
"edge0-8b": "9.11 和 9.8 哪个大?请仔细比较。",
@@ -70,9 +73,12 @@ def _prompt_for(engine) -> str:
return PROMPTS.get(engine.name, DEFAULT_PROMPT)
-def run_bench(engine, ntok: int, warmup: int) -> dict:
+def run_bench(engine, ntok: int, warmup: int, *, submission: bool = False) -> dict:
+ from edge0.backends import core
+ from edge0.sampling import sample
+
tok = engine._tok
- prompt = _prompt_for(engine)
+ prompt = _long_prompt() if submission else _prompt_for(engine)
if hasattr(engine, "encode_chat"):
ids = engine.encode_chat(
[{"role": "user", "content": prompt}], think=False)
@@ -102,7 +108,7 @@ def run_bench(engine, ntok: int, warmup: int) -> dict:
core.random.seed(int(seed))
history = list(ids)
- # warmup: sampled, untimed (first tokens are dominated by expert
+ # warmup: greedy, untimed (first tokens are dominated by expert
# cold-starts; the timed window is the steady state)
for _ in range(warmup):
tid = int(core.argmax(logits, axis=-1).item())
@@ -123,6 +129,8 @@ def run_bench(engine, ntok: int, warmup: int) -> dict:
ntok=len(out), peak_gib=peak_gib,
tok_s=len(out) / t_decode if t_decode else 0.0))
pf_tps = (len(ids) / t_prefill) if t_prefill else 0.0
+ if submission:
+ results[-1]["prefill_tok_s"] = pf_tps
print(f"prompt={len(ids)} tok prefill={t_prefill:.2f}s "
f"({pf_tps:.0f} tok/s) "
f"decode={len(out)}/{t_decode:.2f}s "
@@ -132,7 +140,72 @@ def run_bench(engine, ntok: int, warmup: int) -> dict:
max_peak = max(r["peak_gib"] for r in results)
print(f"[bench] mean tok/s={mean_ts:.1f} mlx_allocator_peak<={max_peak:.2f} GiB",
flush=True)
- return {"runs": results, "mean_tok_s": mean_ts, "peak_gib": max_peak}
+ report = {"runs": results, "mean_tok_s": mean_ts, "peak_gib": max_peak}
+ if submission:
+ report["mean_prefill_tok_s"] = sum(r["prefill_tok_s"] for r in results) / len(results)
+ return report
+
+
+def _sysctl(name: str) -> str:
+ try:
+ return subprocess.run(["sysctl", "-n", name], capture_output=True,
+ text=True, check=True, timeout=3).stdout.strip()
+ except (OSError, subprocess.SubprocessError):
+ return ""
+
+
+def _hardware_metadata() -> dict:
+ try:
+ mlx_version = version("mlx")
+ except PackageNotFoundError:
+ mlx_version = ""
+ ram_gb = None
+ device = chip = ""
+ if platform.system() == "Darwin":
+ device = _sysctl("hw.model")
+ chip = _sysctl("machdep.cpu.brand_string")
+ try:
+ memory = int(_sysctl("hw.memsize"))
+ if memory > 0:
+ ram_gb = memory / (1024 ** 3)
+ except ValueError:
+ pass
+ if platform.system() == "Darwin":
+ os_name = "macOS " + platform.mac_ver()[0]
+ else:
+ os_name = platform.system() + " " + platform.release()
+ return {"device": device, "chip": chip, "ram_gb": ram_gb,
+ "os": os_name.strip(), "mlx_version": mlx_version}
+
+
+def _single_line(value) -> str:
+ return " ".join(str(value).split())
+
+
+def _submission_block(tier: str, report: dict, args) -> dict:
+ metadata = _hardware_metadata()
+ notes = [args.notes, "BENCH_LONG; 2 timed runs; "
+ f"{args.warmup} greedy warmup steps/run; {args.ntok} sampled decode steps/run; "
+ "memory=MLX allocator peak (not process RSS)"]
+ for key in ("BENCH_TEMP", "BENCH_SEED"):
+ if key in os.environ:
+ notes.append(f"{key}={os.environ[key]}")
+ return {
+ "tier": tier,
+ "runtime": "python-mlx",
+ "device": _single_line(metadata.get("device", "")),
+ "chip": _single_line(metadata.get("chip", "")),
+ "ram_gb": metadata.get("ram_gb"),
+ "os": _single_line(metadata.get("os", "")),
+ "storage": _single_line(args.storage),
+ "mlx_version": _single_line(metadata.get("mlx_version", "")),
+ "decode_tok_s": report["mean_tok_s"],
+ "prefill_tok_s": report["mean_prefill_tok_s"],
+ "peak_active_gib": report["peak_gib"],
+ "on_battery": args.on_battery,
+ "reporter": _single_line(args.reporter),
+ "notes": _single_line("; ".join(note for note in notes if note)),
+ }
def main() -> int:
@@ -141,23 +214,62 @@ def main() -> int:
ap.add_argument("--ntok", type=int,
default=int(os.environ.get("BENCH_NTOK", "200")))
ap.add_argument("--warmup", type=int, default=10)
+ ap.add_argument("--submission", action="store_true",
+ help="use BENCH_LONG and print the community submission block")
+ ap.add_argument("--json", action="store_true",
+ help="with --submission, emit only JSON on stdout; progress goes to stderr")
+ ap.add_argument("--reporter", default="", help="GitHub handle for the submission")
+ ap.add_argument("--storage", default="", help="storage containing the checkpoint (not auto-detected)")
+ ap.add_argument("--on-battery", choices=("yes", "no"), default="",
+ help="power state during measurement (not auto-detected)")
+ ap.add_argument("--notes", default="", help="background load, throttling or other context")
args = ap.parse_args()
+ if args.json and not args.submission:
+ ap.error("--json requires --submission")
+ if args.ntok <= 0 or args.warmup < 0:
+ ap.error("--ntok must be positive and --warmup must be nonnegative")
+ if args.submission and os.environ.get("BENCH_PROMPT"):
+ ap.error("--submission requires the standard long prompt; unset BENCH_PROMPT")
# tier name -> checkpoint dir via $EDGE0_<TIER>_MODEL (cli parity)
model = args.model
name = None
- if model.startswith("edge0-"):
- from edge0.cli import TIER_ENV
- env = os.environ.get(TIER_ENV.get(model, ""))
- if not env:
- raise SystemExit(
- f"set {TIER_ENV.get(model, 'EDGE0_<TIER>_MODEL')} to the "
- f"checkpoint directory for {model}, or pass the dir itself")
- model = env
- engine = AutoEngine.from_pretrained(model, name=name)
- print(f"[bench] tier={engine.name} ntok={args.ntok} warmup={args.warmup}",
- flush=True)
- run_bench(engine, args.ntok, args.warmup)
- engine.close()
+ with redirect_stdout(sys.stderr) if args.json else nullcontext():
+ from edge0 import AutoEngine
+
+ if model.startswith("edge0-"):
+ from edge0.cli import TIER_ENV
+ env = os.environ.get(TIER_ENV.get(model, ""))
+ if not env:
+ raise SystemExit(
+ f"set {TIER_ENV.get(model, 'EDGE0_<TIER>_MODEL')} to the "
+ f"checkpoint directory for {model}, or pass the dir itself")
+ model = env
+ engine = AutoEngine.from_pretrained(model, name=name)
+ try:
+ print(f"[bench] tier={engine.name} ntok={args.ntok} warmup={args.warmup}",
+ flush=True)
+ if args.submission:
+ report = run_bench(engine, args.ntok, args.warmup, submission=True)
+ else:
+ report = run_bench(engine, args.ntok, args.warmup)
+ tier = engine.name
+ finally:
+ engine.close()
+ if args.submission:
+ block = _submission_block(tier, report, args)
+ payload = json.dumps(block, ensure_ascii=False, allow_nan=False)
+ if args.json:
+ print(payload)
+ else:
+ print()
+ for key, value in block.items():
+ if value is None:
+ text = ""
+ elif isinstance(value, float):
+ text = f"{value:.2f}"
+ else:
+ text = str(value)
+ print(f"{key + ':':<18} {text}".rstrip())
return 0
diff --git a/python/tests/test_bench_submission.py b/python/tests/test_bench_submission.py
new file mode 100644
index 0000000..cb1510a
--- /dev/null
+++ b/python/tests/test_bench_submission.py
@@ -0,0 +1,232 @@
+"""CLI/report contracts for the MLX benchmark harness; no scalar numerical oracle."""
+
+import importlib.util
+import json
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+
+
+def bench():
+ path = Path(__file__).resolve().parents[1] / "examples" / "bench.py"
+ spec = importlib.util.spec_from_file_location("bench_example", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def invocation(bench, monkeypatch):
+ from edge0 import AutoEngine
+
+ for key in ("BENCH_PROMPT", "BENCH_LONG", "BENCH_NTOK"):
+ monkeypatch.delenv(key, raising=False)
+ engine = SimpleNamespace(name="edge0-35b", close=Mock())
+ factory = Mock(return_value=engine)
+ monkeypatch.setattr(AutoEngine, "from_pretrained", factory)
+ report = {"runs": [], "mean_tok_s": 7.5, "peak_gib": 3.25,
+ "mean_prefill_tok_s": 125.5}
+
+ def measured(*args, **kwargs):
+ print("measurement progress")
+ return report
+
+ run = Mock(side_effect=measured)
+ monkeypatch.setattr(bench, "run_bench", run)
+ return engine, factory, run, report
+
+
+def test_submission_json_is_one_document_and_matches_template(
+ bench, invocation, monkeypatch, capsys):
+ engine, factory, run, report = invocation
+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {
+ "device": "Mac fixture", "chip": "Apple fixture", "ram_gb": None,
+ "os": "macOS fixture", "mlx_version": "fixture"})
+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission", "--json",
+ "--reporter", "@tester", "--storage", "external SSD",
+ "--on-battery", "no", "--notes", "first\nsecond"])
+ assert bench.main() == 0
+ output = capsys.readouterr()
+ data = json.loads(output.out)
+ assert len(output.out.splitlines()) == 1
+ assert "measurement progress" in output.err
+ assert data["tier"] == engine.name
+ assert data["runtime"] == "python-mlx"
+ assert data["decode_tok_s"] == report["mean_tok_s"]
+ assert data["prefill_tok_s"] == report["mean_prefill_tok_s"]
+ assert data["peak_active_gib"] == report["peak_gib"]
+ assert data["ram_gb"] is None
+ assert data["reporter"] == "@tester"
+ assert data["on_battery"] == "no"
+ assert data["storage"] == "external SSD"
+ assert "first second" in data["notes"]
+ assert "MLX allocator" in data["notes"]
+ assert set(data) == {"tier", "runtime", "device", "chip", "ram_gb", "os", "storage",
+ "mlx_version", "decode_tok_s", "prefill_tok_s", "peak_active_gib",
+ "on_battery", "reporter", "notes"}
+ run.assert_called_once_with(engine, 200, 10, submission=True)
+ factory.assert_called_once_with("checkpoint", name=None)
+ engine.close.assert_called_once()
+
+
+def test_submission_text_has_pasteable_colon_fields(bench, invocation, monkeypatch, capsys):
+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {})
+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission"])
+ assert bench.main() == 0
+ text = capsys.readouterr().out
+ assert "tier:" in text and "runtime:" in text and "reporter:" in text
+ assert "None" not in text
+
+
+def test_default_cli_retains_existing_benchmark_call(bench, invocation, monkeypatch, capsys):
+ engine, _, run, _ = invocation
+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint"])
+ assert bench.main() == 0
+ run.assert_called_once_with(engine, 200, 10)
+ engine.close.assert_called_once()
+ output = capsys.readouterr()
+ assert "[bench] tier=edge0-35b" in output.out
+ assert "measurement progress" in output.out
+ assert output.err == ""
+
+
[email protected]("flags", [["--json"], ["--submission", "--ntok", "0"],
+ ["--submission", "--warmup", "-1"],
+ ["--submission", "--on-battery", "maybe"]])
+def test_invalid_flags_fail_before_loading_model(bench, invocation, monkeypatch, flags):
+ _, factory, _, _ = invocation
+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", *flags])
+ with pytest.raises(SystemExit) as error:
+ bench.main()
+ assert error.value.code == 2
+ factory.assert_not_called()
+
+
+def test_submission_rejects_custom_prompt_before_model_load(bench, invocation, monkeypatch):
+ _, factory, _, _ = invocation
+ monkeypatch.setenv("BENCH_PROMPT", "custom prompt")
+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission"])
+ with pytest.raises(SystemExit) as error:
+ bench.main()
+ assert error.value.code == 2
+ factory.assert_not_called()
+
+
+def test_json_failure_does_not_emit_partial_document_and_closes_engine(
+ bench, invocation, monkeypatch, capsys):
+ engine, _, run, _ = invocation
+ run.side_effect = RuntimeError("fixture failure")
+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission", "--json"])
+ with pytest.raises(RuntimeError, match="fixture failure"):
+ bench.main()
+ assert capsys.readouterr().out == ""
+ engine.close.assert_called_once()
+
+
+def test_json_routes_engine_load_and_cleanup_progress_to_stderr(
+ bench, invocation, monkeypatch, capsys):
+ engine, factory, _, _ = invocation
+
+ def load(*args, **kwargs):
+ print("loading fixture")
+ return engine
+
+ factory.side_effect = load
+ engine.close.side_effect = lambda: print("closing fixture")
+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {})
+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission", "--json"])
+ assert bench.main() == 0
+ output = capsys.readouterr()
+ assert json.loads(output.out)["tier"] == engine.name
+ assert "loading fixture" in output.err and "closing fixture" in output.err
+
+
+def test_json_rejects_nonfinite_measurements(bench, invocation, monkeypatch, capsys):
+ _, _, _, report = invocation
+ report["mean_tok_s"] = float("nan")
+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {})
+ monkeypatch.setattr("sys.argv", ["bench.py", "checkpoint", "--submission", "--json"])
+ with pytest.raises(ValueError):
+ bench.main()
+ assert capsys.readouterr().out == ""
+
+
[email protected]("submission", [False, True])
+def test_mlx_harness_retains_reset_warmup_decode_shape(bench, monkeypatch, submission):
+ from edge0.backends import core
+
+ for key in ("BENCH_TEMP", "BENCH_SEED", "BENCH_PROMPT", "BENCH_LONG"):
+ monkeypatch.delenv(key, raising=False)
+
+ class TinyMLXEngine:
+ name = "edge0-35b"
+ _tok = None
+ cfg = SimpleNamespace(gen=SimpleNamespace(temperature=0.7, top_k=2, top_p=1.0,
+ repetition_penalty=1.0))
+
+ def __init__(self):
+ self.resets = 0
+ self.steps = 0
+ self.prefills = []
+ self.prompts = []
+ self.logits = core.zeros((4,))
+
+ def encode_chat(self, messages, *, think):
+ assert think is False
+ self.prompts.append(messages[0]["content"])
+ return [1, 2, 3]
+
+ def reset(self):
+ self.resets += 1
+
+ def prefill(self, ids):
+ self.prefills.append(list(ids))
+ core.eval(self.logits)
+
+ def next_logits(self):
+ return self.logits
+
+ def step(self, token):
+ self.steps += 1
+ return self.logits
+
+ engine = TinyMLXEngine()
+ report = bench.run_bench(engine, 2, 1, submission=submission)
+ assert engine.resets == 2
+ assert engine.steps == 6
+ assert engine.prefills == [[1, 2, 3], [1, 2, 3]]
+ assert engine.prompts == [bench._long_prompt() if submission else bench.PROMPTS[engine.name]]
+ assert len(report["runs"]) == 2
+ keys = {"runs", "mean_tok_s", "peak_gib"}
+ run_keys = {"prefill_s", "decode_s", "ntok", "peak_gib", "tok_s"}
+ if submission:
+ keys.add("mean_prefill_tok_s")
+ run_keys.add("prefill_tok_s")
+ assert set(report) == keys
+ assert all(set(run) == run_keys for run in report["runs"])
+ json.dumps(report, allow_nan=False)
+
+
+def test_submission_metadata_failure_leaves_unknown_values(bench, monkeypatch):
+ def unavailable(*args, **kwargs):
+ raise FileNotFoundError("fixture")
+
+ monkeypatch.setattr(bench.platform, "system", lambda: "Darwin")
+ monkeypatch.setattr(bench.platform, "mac_ver", lambda: ("fixture", (), ""))
+ monkeypatch.setattr(bench.subprocess, "run", unavailable)
+ monkeypatch.setattr(bench, "version", Mock(side_effect=bench.PackageNotFoundError))
+ assert bench._hardware_metadata() == {"device": "", "chip": "", "ram_gb": None,
+ "os": "macOS fixture", "mlx_version": ""}
+
+
+def test_submission_alias_uses_existing_environment_resolution(bench, invocation, monkeypatch):
+ engine, factory, _, _ = invocation
+ monkeypatch.setenv("EDGE0_35B_MODEL", "fixture-checkpoint")
+ monkeypatch.setattr(bench, "_hardware_metadata", lambda: {})
+ monkeypatch.setattr("sys.argv", ["bench.py", "edge0-35b", "--submission", "--json"])
+ assert bench.main() == 0
+ factory.assert_called_once_with("fixture-checkpoint", name=None)
+ engine.close.assert_called_once()