Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
"""Real-checkpoint smoke tests for the shipped model tiers.
Set ``EDGE0_8B_MODEL`` and/or ``EDGE0_35B_MODEL`` to checkpoint directories
before running ``pytest -m slow``. A missing tier is an explicit skip so a
machine without every large checkpoint can still exercise the available one.
"""
from __future__ import annotations
import os
import time
from pathlib import Path
import pytest
from edge0 import AutoEngine
from edge0.config import GenerationConfig
pytestmark = pytest.mark.slow
MAX_NEW_TOKENS = 8
def _checkpoint(env_name: str, tier: str) -> str:
model_dir = os.environ.get(env_name)
if not model_dir:
pytest.skip(f"{env_name} is not set; skipping {tier} checkpoint")
path = Path(model_dir).expanduser()
if not path.is_dir():
pytest.skip(f"{env_name} checkpoint not found: {path}")
return str(path)
def _chat_ids(engine, messages: list[dict[str, str]]) -> list[int]:
if hasattr(engine, "encode_chat"):
return list(engine.encode_chat(messages, think=False))
tok = engine._tok
rendered = tok.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True)
return list(tok(rendered)["input_ids"])
def _generate(env_name: str, tier: str,
messages: list[dict[str, str]]) -> list[int]:
model_dir = _checkpoint(env_name, tier)
engine = AutoEngine.from_pretrained(model_dir, name=tier)
try:
prompt_ids = _chat_ids(engine, messages)
started = time.perf_counter()
output = engine.generate(
prompt_ids,
GenerationConfig(
temperature=0.0,
top_k=1,
top_p=1.0,
max_new_tokens=MAX_NEW_TOKENS,
),
max_new_tokens=MAX_NEW_TOKENS,
)
elapsed = time.perf_counter() - started
tok_s = len(output) / elapsed if elapsed else 0.0
print(f"[slow] {tier}: {len(output)} tokens in {elapsed:.2f}s "
f"({tok_s:.1f} tok/s)", flush=True)
finally:
engine.close()
return output
def test_edge0_8b_real_checkpoint():
output = _generate(
"EDGE0_8B_MODEL",
"edge0-8b",
[{"role": "user", "content": "你好,请用一句话介绍海滨城市。"}],
)
assert output
assert len(output) <= MAX_NEW_TOKENS
def test_edge0_35b_real_checkpoint():
output = _generate(
"EDGE0_35B_MODEL",
"edge0-35b",
[{"role": "user", "content":
"Hello! Write one short sentence about the seaside."}],
)
assert output
assert len(output) <= MAX_NEW_TOKENS
def test_edge0_8b_prefill_ondemand_switch_skips_whole_layers(monkeypatch):
"""The `--prefill-ondemand` path (config switch -> engine) really takes
the on-demand prefill, and leaves the next token identical."""
from edge0.backends import core
from edge0.streaming.layer import StreamingSwitchGLU
model_dir = _checkpoint("EDGE0_8B_MODEL", "edge0-8b")
messages = [{"role": "user", "content": "你好,请用一句话介绍海滨城市。"}]
engine = AutoEngine.from_pretrained(model_dir, name="edge0-8b")
try:
ids = _chat_ids(engine, messages)
assert len(ids) > 1, "need a multi-token prefill"
engine.reset()
engine.prefill(ids)
baseline = int(core.argmax(engine.next_logits(), axis=-1).item())
finally:
engine.close()
loads: list[int] = []
original = StreamingSwitchGLU.load_full_layer
def counted(self, *args, **kwargs):
loads.append(1)
return original(self, *args, **kwargs)
monkeypatch.setattr(StreamingSwitchGLU, "load_full_layer", counted)
engine = AutoEngine.from_pretrained(model_dir, name="edge0-8b",
prefill_ondemand=True)
try:
engine.reset()
engine.prefill(ids)
assert loads == [], (
f"--prefill-ondemand still loaded {len(loads)} whole layers")
assert int(core.argmax(engine.next_logits(), axis=-1).item()) == baseline
finally:
engine.close()
def test_edge0_8b_ondemand_prefill_is_honored(monkeypatch):
"""``full_layer_prefill=False`` must really skip the whole-layer prefill.
Regression guard: the flag was a no-op on this tier. The prefill hook
was installed for every multi-token prefill regardless of the flag, and
with ``full_n=0`` ("every layer") it always fell through to
``load_full_layer()`` -- so disabling it still read all 128 experts of
all 23 layers (~4.1 GiB for a 27-token prompt) instead of the routed
experts only (~0.42 GiB, measured on an M4 Pro).
"""
from dataclasses import replace
from edge0.backends import core
from edge0.streaming.layer import StreamingSwitchGLU
from edge0.streaming.options import LayerOptions
model_dir = _checkpoint("EDGE0_8B_MODEL", "edge0-8b")
messages = [{"role": "user", "content": "你好,请用一句话介绍海滨城市。"}]
# 1) tier default (whole-layer load-drop prefill)
engine = AutoEngine.from_pretrained(model_dir, name="edge0-8b")
try:
prompt_ids = _chat_ids(engine, messages)
assert len(prompt_ids) > 1, "need a multi-token prefill"
engine.reset()
engine.prefill(prompt_ids)
baseline = int(core.argmax(engine.next_logits(), axis=-1).item())
finally:
engine.close()
# 2) on-demand prefill: no whole-layer load may fire, same next token
loads: list[int] = []
original = StreamingSwitchGLU.load_full_layer
def counted(self, *args, **kwargs):
loads.append(1)
return original(self, *args, **kwargs)
monkeypatch.setattr(StreamingSwitchGLU, "load_full_layer", counted)
engine = AutoEngine.from_pretrained(
model_dir, name="edge0-8b",
options=replace(LayerOptions.prod_k8(), full_layer_prefill=False))
try:
engine.reset()
engine.prefill(prompt_ids)
assert loads == [], (
f"on-demand prefill still loaded {len(loads)} whole layers")
assert int(core.argmax(engine.next_logits(), axis=-1).item()) == baseline
finally:
engine.close()