Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
"""Minimal edge0 API walkthrough — the same path ``edge0 demo`` runs.
Install:
python3.12 -m venv .venv && .venv/bin/pip install -e '.[dev]'
Run (checkpoint path or registered tier name):
.venv/bin/python examples/demo.py --model-dir /path/to/qwen35/model
.venv/bin/python examples/demo.py --model edge0-8b
Three lines do everything: build the engine (prerouter + LoRA + SSD
offload installed automatically from the tier config), template the
prompt, generate.
The demo runs each tier's showcase configuration: ``edge0-8b`` defaults
to the gate-routed exact path (``prerouter=None``). Use ``edge0 chat``
(or ``AutoEngine.from_pretrained(..., prerouter=...)``) for the
prediction path.
"""
from __future__ import annotations
import argparse
import sys
from edge0 import AutoEngine, demo_kwargs
from edge0.server.chat import ChatMessage, ChatRequest, ChatSession
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--model", default=None,
help="registered tier name, e.g. edge0-35b")
ap.add_argument("--model-dir", default=None,
help="checkpoint directory (tier auto-detected)")
ap.add_argument("--prompt",
default="Hello! Write one short sentence about the seaside.")
ap.add_argument("--max-new", type=int, default=24)
ap.add_argument("--no-prerouter", action="store_true",
help="route with the layers' own gates (edge0-8b demos "
"already run that path)")
args = ap.parse_args()
# ``--model <tier>`` resolves through the tier env var (same contract
# as ``edge0 demo``); ``--model-dir`` always wins.
if not args.model_dir and args.model:
import os
from edge0.cli import TIER_ENV
env = TIER_ENV.get(args.model)
args.model_dir = os.environ.get(env, "") if env else ""
if not args.model_dir:
ap.error(f"--model {args.model} needs ${env or '?'} set to the "
f"checkpoint directory (or pass --model-dir)")
kw = ({"prerouter": None} if args.no_prerouter
else demo_kwargs(args.model_dir, args.model))
engine = AutoEngine.from_pretrained(args.model_dir, name=args.model, **kw)
req = ChatRequest(
model=engine.name,
messages=[ChatMessage(role="user", content=args.prompt)],
max_tokens=args.max_new,
)
tokens, meta = ChatSession(engine, req).run()
print(f"user : {args.prompt}")
print(f"edge0: {engine._tok.decode(tokens)}")
print(f"# {len(tokens)} tokens in {meta['wall_s']}s", file=sys.stderr)
engine.close()
return 0
if __name__ == "__main__":
sys.exit(main())