Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
6.0 KB
Raw Blame
"""Real-checkpoint smoke tests for the shipped model tiers.

Set ``EDGE0_8B_MODEL`` and/or ``EDGE0_35B_MODEL`` to checkpoint directories
before running ``pytest -m slow``.  A missing tier is an explicit skip so a
machine without every large checkpoint can still exercise the available one.
"""

from __future__ import annotations

import os
import time
from pathlib import Path

import pytest

from edge0 import AutoEngine
from edge0.config import GenerationConfig


pytestmark = pytest.mark.slow

MAX_NEW_TOKENS = 8


def _checkpoint(env_name: str, tier: str) -> str:
    model_dir = os.environ.get(env_name)
    if not model_dir:
        pytest.skip(f"{env_name} is not set; skipping {tier} checkpoint")
    path = Path(model_dir).expanduser()
    if not path.is_dir():
        pytest.skip(f"{env_name} checkpoint not found: {path}")
    return str(path)


def _chat_ids(engine, messages: list[dict[str, str]]) -> list[int]:
    if hasattr(engine, "encode_chat"):
        return list(engine.encode_chat(messages, think=False))

    tok = engine._tok
    rendered = tok.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True)
    return list(tok(rendered)["input_ids"])


def _generate(env_name: str, tier: str,
              messages: list[dict[str, str]]) -> list[int]:
    model_dir = _checkpoint(env_name, tier)
    engine = AutoEngine.from_pretrained(model_dir, name=tier)
    try:
        prompt_ids = _chat_ids(engine, messages)
        started = time.perf_counter()
        output = engine.generate(
            prompt_ids,
            GenerationConfig(
                temperature=0.0,
                top_k=1,
                top_p=1.0,
                max_new_tokens=MAX_NEW_TOKENS,
            ),
            max_new_tokens=MAX_NEW_TOKENS,
        )
        elapsed = time.perf_counter() - started
        tok_s = len(output) / elapsed if elapsed else 0.0
        print(f"[slow] {tier}: {len(output)} tokens in {elapsed:.2f}s "
              f"({tok_s:.1f} tok/s)", flush=True)
    finally:
        engine.close()
    return output


def test_edge0_8b_real_checkpoint():
    output = _generate(
        "EDGE0_8B_MODEL",
        "edge0-8b",
        [{"role": "user", "content": "你好,请用一句话介绍海滨城市。"}],
    )
    assert output
    assert len(output) <= MAX_NEW_TOKENS


def test_edge0_35b_real_checkpoint():
    output = _generate(
        "EDGE0_35B_MODEL",
        "edge0-35b",
        [{"role": "user", "content":
          "Hello! Write one short sentence about the seaside."}],
    )
    assert output
    assert len(output) <= MAX_NEW_TOKENS


def test_edge0_8b_prefill_ondemand_switch_skips_whole_layers(monkeypatch):
    """The `--prefill-ondemand` path (config switch -> engine) really takes
    the on-demand prefill, and leaves the next token identical."""
    from edge0.backends import core
    from edge0.streaming.layer import StreamingSwitchGLU

    model_dir = _checkpoint("EDGE0_8B_MODEL", "edge0-8b")
    messages = [{"role": "user", "content": "你好,请用一句话介绍海滨城市。"}]

    engine = AutoEngine.from_pretrained(model_dir, name="edge0-8b")
    try:
        ids = _chat_ids(engine, messages)
        assert len(ids) > 1, "need a multi-token prefill"
        engine.reset()
        engine.prefill(ids)
        baseline = int(core.argmax(engine.next_logits(), axis=-1).item())
    finally:
        engine.close()

    loads: list[int] = []
    original = StreamingSwitchGLU.load_full_layer

    def counted(self, *args, **kwargs):
        loads.append(1)
        return original(self, *args, **kwargs)

    monkeypatch.setattr(StreamingSwitchGLU, "load_full_layer", counted)
    engine = AutoEngine.from_pretrained(model_dir, name="edge0-8b",
                                        prefill_ondemand=True)
    try:
        engine.reset()
        engine.prefill(ids)
        assert loads == [], (
            f"--prefill-ondemand still loaded {len(loads)} whole layers")
        assert int(core.argmax(engine.next_logits(), axis=-1).item()) == baseline
    finally:
        engine.close()


def test_edge0_8b_ondemand_prefill_is_honored(monkeypatch):
    """``full_layer_prefill=False`` must really skip the whole-layer prefill.

    Regression guard: the flag was a no-op on this tier.  The prefill hook
    was installed for every multi-token prefill regardless of the flag, and
    with ``full_n=0`` ("every layer") it always fell through to
    ``load_full_layer()`` -- so disabling it still read all 128 experts of
    all 23 layers (~4.1 GiB for a 27-token prompt) instead of the routed
    experts only (~0.42 GiB, measured on an M4 Pro).
    """
    from dataclasses import replace

    from edge0.backends import core
    from edge0.streaming.layer import StreamingSwitchGLU
    from edge0.streaming.options import LayerOptions

    model_dir = _checkpoint("EDGE0_8B_MODEL", "edge0-8b")
    messages = [{"role": "user", "content": "你好,请用一句话介绍海滨城市。"}]

    # 1) tier default (whole-layer load-drop prefill)
    engine = AutoEngine.from_pretrained(model_dir, name="edge0-8b")
    try:
        prompt_ids = _chat_ids(engine, messages)
        assert len(prompt_ids) > 1, "need a multi-token prefill"
        engine.reset()
        engine.prefill(prompt_ids)
        baseline = int(core.argmax(engine.next_logits(), axis=-1).item())
    finally:
        engine.close()

    # 2) on-demand prefill: no whole-layer load may fire, same next token
    loads: list[int] = []
    original = StreamingSwitchGLU.load_full_layer

    def counted(self, *args, **kwargs):
        loads.append(1)
        return original(self, *args, **kwargs)

    monkeypatch.setattr(StreamingSwitchGLU, "load_full_layer", counted)
    engine = AutoEngine.from_pretrained(
        model_dir, name="edge0-8b",
        options=replace(LayerOptions.prod_k8(), full_layer_prefill=False))
    try:
        engine.reset()
        engine.prefill(prompt_ids)
        assert loads == [], (
            f"on-demand prefill still loaded {len(loads)} whole layers")
        assert int(core.argmax(engine.next_logits(), axis=-1).item()) == baseline
    finally:
        engine.close()