Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
2.6 KB
Raw Blame
"""Minimal edge0 API walkthrough — the same path ``edge0 demo`` runs.

Install:
    python3.12 -m venv .venv && .venv/bin/pip install -e '.[dev]'

Run (checkpoint path or registered tier name):
    .venv/bin/python examples/demo.py --model-dir /path/to/qwen35/model
    .venv/bin/python examples/demo.py --model edge0-8b

Three lines do everything: build the engine (prerouter + LoRA + SSD
offload installed automatically from the tier config), template the
prompt, generate.

The demo runs each tier's showcase configuration: ``edge0-8b`` defaults
to the gate-routed exact path (``prerouter=None``).  Use ``edge0 chat``
(or ``AutoEngine.from_pretrained(..., prerouter=...)``) for the
prediction path.
"""

from __future__ import annotations

import argparse
import sys

from edge0 import AutoEngine, demo_kwargs
from edge0.server.chat import ChatMessage, ChatRequest, ChatSession


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--model", default=None,
                    help="registered tier name, e.g. edge0-35b")
    ap.add_argument("--model-dir", default=None,
                    help="checkpoint directory (tier auto-detected)")
    ap.add_argument("--prompt",
                    default="Hello! Write one short sentence about the seaside.")
    ap.add_argument("--max-new", type=int, default=24)
    ap.add_argument("--no-prerouter", action="store_true",
                    help="route with the layers' own gates (edge0-8b demos "
                         "already run that path)")
    args = ap.parse_args()

    # ``--model <tier>`` resolves through the tier env var (same contract
    # as ``edge0 demo``); ``--model-dir`` always wins.
    if not args.model_dir and args.model:
        import os

        from edge0.cli import TIER_ENV
        env = TIER_ENV.get(args.model)
        args.model_dir = os.environ.get(env, "") if env else ""
        if not args.model_dir:
            ap.error(f"--model {args.model} needs ${env or '?'} set to the "
                     f"checkpoint directory (or pass --model-dir)")

    kw = ({"prerouter": None} if args.no_prerouter
          else demo_kwargs(args.model_dir, args.model))
    engine = AutoEngine.from_pretrained(args.model_dir, name=args.model, **kw)
    req = ChatRequest(
        model=engine.name,
        messages=[ChatMessage(role="user", content=args.prompt)],
        max_tokens=args.max_new,
    )
    tokens, meta = ChatSession(engine, req).run()
    print(f"user : {args.prompt}")
    print(f"edge0: {engine._tok.decode(tokens)}")
    print(f"# {len(tokens)} tokens in {meta['wall_s']}s", file=sys.stderr)
    engine.close()
    return 0


if __name__ == "__main__":
    sys.exit(main())