Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
11.5 KB
Raw Blame
#!/usr/bin/env python3
"""edge0 speed + memory benchmark (any tier).

Runs the same measurement shape as the deployment's bench: prefill ->
greedy warmup steps -> timed sampled decode -> report tok/s and MLX
allocator peak memory (`mlx.core.get_peak_memory`).  Works for both
tiers; the tier is auto-detected from the checkpoint (or forced via the
model name).

Usage:
    python examples/bench.py /path/to/model [--ntok 200] [--warmup 10]
    python examples/bench.py edge0-35b        # via $EDGE0_35B_MODEL
    BENCH_LONG=1 python examples/bench.py edge0-35b   # ~3k-token prefill
    python examples/bench.py edge0-35b --submission --reporter @you
    python examples/bench.py edge0-35b --submission --json > submission.json

Env (sampling knobs, defaults follow the tier's GenerationConfig):
    BENCH_TEMP / BENCH_PROMPT / BENCH_SEED
    BENCH_LONG=1  use a ~3k-token synthetic prompt so the prefill timing
                  is meaningful (short prompts only measure fixed
                  overhead; set BENCH_PROMPT for your own long text)
"""

from __future__ import annotations

import argparse
from contextlib import nullcontext, redirect_stdout
from importlib.metadata import PackageNotFoundError, version
import json
import os
import platform
import subprocess
import sys
import time

PROMPTS = {
    "edge0-35b": "什么是混合专家模型(MoE)?它和普通 Transformer 有什么区别?",
    "edge0-8b": "9.11 和 9.8 哪个大?请仔细比较。",
}
DEFAULT_PROMPT = "什么是混合专家模型(MoE)?简单介绍一下。"

# Synthetic long-context prompt (~3k tokens after tokenization): two
# non-repetitive paragraphs alternating 40 times, prefixed with a task
# instruction.  Prefill timing on a few dozen tokens only measures fixed
# overhead; a long prompt exercises the real chunked prefill path
# (per-layer expert loading, KV growth).
_PARA_ZH = (
    "流式推理框架的设计要点在于把磁盘带宽、缓存层级与计算核心三者重叠"
    "起来。混合专家模型每一层只在少数专家上激活,路由器给出的稀疏选择"
    "恰好为按需加载提供了天然的批粒度。若能在前一步的隐状态基础上预判"
    "下一步的专家集合,固态硬盘的读取延迟就能被计算完全掩盖,这对边缘"
    "设备上的大模型部署具有直接意义。")
_PARA_EN = (
    "The design of a streaming inference framework hinges on overlapping "
    "disk bandwidth, cache hierarchy, and compute. A mixture-of-experts "
    "layer activates only a handful of experts per token, and the router "
    "sparsity provides a natural granularity for on-demand loading. If "
    "the expert set of the next step can be predicted from the previous "
    "hidden state, SSD read latency hides behind compute entirely, which "
    "matters for large-model deployment on edge hardware.")


def _long_prompt() -> str:
    paras = [_PARA_ZH if i % 2 == 0 else _PARA_EN for i in range(40)]
    return "请仔细阅读以下材料并用一句话总结其核心思想:" + " ".join(paras)


def _prompt_for(engine) -> str:
    if os.environ.get("BENCH_PROMPT"):
        return os.environ["BENCH_PROMPT"]
    if os.environ.get("BENCH_LONG") == "1":
        return _long_prompt()
    return PROMPTS.get(engine.name, DEFAULT_PROMPT)


def run_bench(engine, ntok: int, warmup: int, *, submission: bool = False) -> dict:
    from edge0.backends import core
    from edge0.sampling import sample

    tok = engine._tok
    prompt = _long_prompt() if submission else _prompt_for(engine)
    if hasattr(engine, "encode_chat"):
        ids = engine.encode_chat(
            [{"role": "user", "content": prompt}], think=False)
    else:
        ids = list(tok(tok.apply_chat_template(
            [{"role": "user", "content": prompt}],
            tokenize=False, add_generation_prompt=True,
            enable_thinking=False))["input_ids"])

    gen = getattr(engine.cfg, "gen", None)
    temperature = float(os.environ.get(
        "BENCH_TEMP", getattr(gen, "temperature", 0.7)))
    top_k = getattr(gen, "top_k", 64)
    top_p = getattr(gen, "top_p", 0.95)
    rp = getattr(gen, "repetition_penalty", 1.0)
    seed = os.environ.get("BENCH_SEED")

    results = []
    for run in range(2):
        engine.reset()
        core.reset_peak_memory()
        t0 = time.perf_counter()
        engine.prefill(ids)
        t_prefill = time.perf_counter() - t0
        logits = engine.next_logits()
        if seed is not None:
            core.random.seed(int(seed))

        history = list(ids)
        # warmup: greedy, untimed (first tokens are dominated by expert
        # cold-starts; the timed window is the steady state)
        for _ in range(warmup):
            tid = int(core.argmax(logits, axis=-1).item())
            history.append(tid)
            logits = engine.step(tid)
        out: list[int] = []
        t0 = time.perf_counter()
        for _ in range(ntok):
            tid = sample(logits, temperature=temperature, top_k=top_k,
                         top_p=top_p, repetition_penalty=rp,
                         history=history, seed=None)
            history.append(tid)
            out.append(tid)
            logits = engine.step(tid)
        t_decode = time.perf_counter() - t0
        peak_gib = core.get_peak_memory() / (1024 ** 3)
        results.append(dict(prefill_s=t_prefill, decode_s=t_decode,
                            ntok=len(out), peak_gib=peak_gib,
                            tok_s=len(out) / t_decode if t_decode else 0.0))
        pf_tps = (len(ids) / t_prefill) if t_prefill else 0.0
        if submission:
            results[-1]["prefill_tok_s"] = pf_tps
        print(f"prompt={len(ids)} tok  prefill={t_prefill:.2f}s "
              f"({pf_tps:.0f} tok/s)  "
              f"decode={len(out)}/{t_decode:.2f}s  "
              f"tok/s={results[-1]['tok_s']:.1f}  "
              f"mlx_allocator_peak={peak_gib:.2f} GiB", flush=True)
    mean_ts = sum(r["tok_s"] for r in results) / len(results)
    max_peak = max(r["peak_gib"] for r in results)
    print(f"[bench] mean tok/s={mean_ts:.1f}  mlx_allocator_peak<={max_peak:.2f} GiB",
          flush=True)
    report = {"runs": results, "mean_tok_s": mean_ts, "peak_gib": max_peak}
    if submission:
        report["mean_prefill_tok_s"] = sum(r["prefill_tok_s"] for r in results) / len(results)
    return report


def _sysctl(name: str) -> str:
    try:
        return subprocess.run(["sysctl", "-n", name], capture_output=True,
                              text=True, check=True, timeout=3).stdout.strip()
    except (OSError, subprocess.SubprocessError):
        return ""


def _hardware_metadata() -> dict:
    try:
        mlx_version = version("mlx")
    except PackageNotFoundError:
        mlx_version = ""
    ram_gb = None
    device = chip = ""
    if platform.system() == "Darwin":
        device = _sysctl("hw.model")
        chip = _sysctl("machdep.cpu.brand_string")
        try:
            memory = int(_sysctl("hw.memsize"))
            if memory > 0:
                ram_gb = memory / (1024 ** 3)
        except ValueError:
            pass
    if platform.system() == "Darwin":
        os_name = "macOS " + platform.mac_ver()[0]
    else:
        os_name = platform.system() + " " + platform.release()
    return {"device": device, "chip": chip, "ram_gb": ram_gb,
            "os": os_name.strip(), "mlx_version": mlx_version}


def _single_line(value) -> str:
    return " ".join(str(value).split())


def _submission_block(tier: str, report: dict, args) -> dict:
    metadata = _hardware_metadata()
    notes = [args.notes, "BENCH_LONG; 2 timed runs; "
             f"{args.warmup} greedy warmup steps/run; {args.ntok} sampled decode steps/run; "
             "memory=MLX allocator peak (not process RSS)"]
    for key in ("BENCH_TEMP", "BENCH_SEED"):
        if key in os.environ:
            notes.append(f"{key}={os.environ[key]}")
    return {
        "tier": tier,
        "runtime": "python-mlx",
        "device": _single_line(metadata.get("device", "")),
        "chip": _single_line(metadata.get("chip", "")),
        "ram_gb": metadata.get("ram_gb"),
        "os": _single_line(metadata.get("os", "")),
        "storage": _single_line(args.storage),
        "mlx_version": _single_line(metadata.get("mlx_version", "")),
        "decode_tok_s": report["mean_tok_s"],
        "prefill_tok_s": report["mean_prefill_tok_s"],
        "peak_active_gib": report["peak_gib"],
        "on_battery": args.on_battery,
        "reporter": _single_line(args.reporter),
        "notes": _single_line("; ".join(note for note in notes if note)),
    }


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("model", help="checkpoint dir or tier name")
    ap.add_argument("--ntok", type=int,
                    default=int(os.environ.get("BENCH_NTOK", "200")))
    ap.add_argument("--warmup", type=int, default=10)
    ap.add_argument("--submission", action="store_true",
                    help="use BENCH_LONG and print the community submission block")
    ap.add_argument("--json", action="store_true",
                    help="with --submission, emit only JSON on stdout; progress goes to stderr")
    ap.add_argument("--reporter", default="", help="GitHub handle for the submission")
    ap.add_argument("--storage", default="", help="storage containing the checkpoint (not auto-detected)")
    ap.add_argument("--on-battery", choices=("yes", "no"), default="",
                    help="power state during measurement (not auto-detected)")
    ap.add_argument("--notes", default="", help="background load, throttling or other context")
    args = ap.parse_args()
    if args.json and not args.submission:
        ap.error("--json requires --submission")
    if args.ntok <= 0 or args.warmup < 0:
        ap.error("--ntok must be positive and --warmup must be nonnegative")
    if args.submission and os.environ.get("BENCH_PROMPT"):
        ap.error("--submission requires the standard long prompt; unset BENCH_PROMPT")
    # tier name -> checkpoint dir via $EDGE0_<TIER>_MODEL (cli parity)
    model = args.model
    name = None
    with redirect_stdout(sys.stderr) if args.json else nullcontext():
        from edge0 import AutoEngine

        if model.startswith("edge0-"):
            from edge0.cli import TIER_ENV
            env = os.environ.get(TIER_ENV.get(model, ""))
            if not env:
                raise SystemExit(
                    f"set {TIER_ENV.get(model, 'EDGE0_<TIER>_MODEL')} to the "
                    f"checkpoint directory for {model}, or pass the dir itself")
            model = env
        engine = AutoEngine.from_pretrained(model, name=name)
        try:
            print(f"[bench] tier={engine.name} ntok={args.ntok} warmup={args.warmup}",
                  flush=True)
            if args.submission:
                report = run_bench(engine, args.ntok, args.warmup, submission=True)
            else:
                report = run_bench(engine, args.ntok, args.warmup)
            tier = engine.name
        finally:
            engine.close()
    if args.submission:
        block = _submission_block(tier, report, args)
        payload = json.dumps(block, ensure_ascii=False, allow_nan=False)
        if args.json:
            print(payload)
        else:
            print()
            for key, value in block.items():
                if value is None:
                    text = ""
                elif isinstance(value, float):
                    text = f"{value:.2f}"
                else:
                    text = str(value)
                print(f"{key + ':':<18} {text}".rstrip())
    return 0


if __name__ == "__main__":
    raise SystemExit(main())