Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
test(streaming): cover layer discovery without tensor calculations
Demetrios Agourakisa6ad384550aa
diff --git a/python/tests/test_moe_spec.py b/python/tests/test_moe_spec.pyindex b3dd18f..34a0f13 100644--- a/python/tests/test_moe_spec.py+++ b/python/tests/test_moe_spec.py@@ -2,6 +2,8 @@from __future__ import annotations+from types import SimpleNamespace+import pytestfrom edge0.moe.spec import (MoESpec, QuantSpec, RouterKind, WeightLayout)@@ -59,6 +61,41 @@ def test_block_of_digit_segments():assert s.block_of(m, 0) is m.language_model.model.layers[0].mlp.switch_mlp+def test_layer_exists_stops_at_end_of_list():+ spec = _spec()+ model = _FakeModel(n=2)+ assert spec.layer_exists(model, 0) is True+ assert spec.layer_exists(model, 1) is True+ assert spec.layer_exists(model, 2) is False+ assert spec.layer_exists(_FakeModel(n=0), 0) is False+++def test_layer_exists_stops_on_attribute_error():+ spec = _spec(block_path="layer_{layer}")+ model = SimpleNamespace(layer_0=object(), layer_1=object())+ assert spec.layer_exists(model, 0) is True+ assert spec.layer_exists(model, 1) is True+ assert spec.layer_exists(model, 2) is False+++def test_layer_exists_reraises_unrelated_index_error():+ spec = _spec(block_path="layers.{layer}.experts.9")+ model = SimpleNamespace(layers=[SimpleNamespace(experts=[object()])])+ with pytest.raises(IndexError):+ spec.layer_exists(model, 0)++[email protected]("block_path", [+ "missing.layers.{layer}.mlp",+ "layers.{layer}.missing",+])+def test_layer_exists_reraises_unrelated_attribute_error(block_path):+ spec = _spec(block_path=block_path)+ model = SimpleNamespace(layers=[SimpleNamespace(mlp=object())])+ with pytest.raises(AttributeError):+ spec.layer_exists(model, 0)++def test_layer_of_defaults_from_block_path():s = _spec()m = _FakeModel()diff --git a/python/tests/test_streaming_install.py b/python/tests/test_streaming_install.pynew file mode 100644index 0000000..da88de3--- /dev/null+++ b/python/tests/test_streaming_install.py@@ -0,0 +1,46 @@+"""Layer discovery and installation without tensor calculations."""++from types import SimpleNamespace++import pytest++from edge0.moe.spec import MoESpec+from edge0.streaming import install+++def _spec():+ return MoESpec(+ num_experts=8, top_k=2, intermediate_size=64,+ block_path="layers.{layer}.mlp",+ )+++def test_install_discovers_mixed_layers(monkeypatch):+ twin = object()+ resident = object()+ moe = SimpleNamespace(switch_mlp=resident)+ dense = SimpleNamespace()+ model = SimpleNamespace(layers=[+ SimpleNamespace(mlp=moe), SimpleNamespace(mlp=dense),+ ])+ calls = []++ def make_twin(shards, layer, spec, **kwargs):+ calls.append((shards, layer, spec))+ return twin++ monkeypatch.setattr(install, "StreamingSwitchGLU", make_twin)+ spec = _spec()+ twins = install.install_streaming_experts(model, [], spec)++ assert twins == [twin, None]+ assert calls == [([], 0, spec)]+ assert moe.switch_mlp is twin+ assert moe._edge0_resident_switch is resident+ assert not hasattr(dense, "switch_mlp")+++def test_install_rejects_empty_layer_list():+ model = SimpleNamespace(layers=[])+ with pytest.raises(ValueError, match="resolves no layer 0"):+ install.install_streaming_experts(model, [], _spec())diff --git a/python/tests/test_streaming_math.py b/python/tests/test_streaming_math.pyindex b8aae01..660d289 100644--- a/python/tests/test_streaming_math.py+++ b/python/tests/test_streaming_math.py@@ -374,41 +374,3 @@ def test_install_discovers_layer_count(tmp_path):assert twins[1] is None and twins[2] is None # dense layersassert moe.switch_mlp is twins[0]twins[0].close()---def test_layer_exists_stops_on_attribute_error():- """The AttributeError half of ``layer_exists``'s except clause is- reachable independently of IndexError: a family whose layer container- is attribute-based (no list, no ``__getitem__``) runs off the end via- a plain missing attribute, not an out-of-range index."""- from types import SimpleNamespace-- spec = MoESpec(- num_experts=N_EXPERTS, top_k=4, intermediate_size=INTER,- key_template="layer_{layer}.mlp.switch_mlp",- block_path="layer_{layer}",- )- model = SimpleNamespace(layer_0=object(), layer_1=object())- assert spec.layer_exists(model, 0) is True- assert spec.layer_exists(model, 1) is True- assert spec.layer_exists(model, 2) is False # no `layer_2` attribute---def test_layer_exists_reraises_unrelated_index_error():- """An IndexError from a segment *other* than the ``{layer}`` slot is a- real bug (a malformed block_path or a broken model), not end-of-list,- and must propagate instead of being read as "past the last layer" --- otherwise install_streaming_experts(num_layers=None) would silently- under-count layers instead of surfacing the break."""- from types import SimpleNamespace-- spec = MoESpec(- num_experts=N_EXPERTS, top_k=4, intermediate_size=INTER,- key_template="layers.{layer}.experts.9.switch_mlp",- block_path="layers.{layer}.experts.9",- )- # `layer=0` is in range for `layers`, but the fixed trailing index `9`- # is out of range for `experts` -- unrelated to layer-count discovery.- model = SimpleNamespace(layers=[SimpleNamespace(experts=[object()])])- with pytest.raises(IndexError):- spec.layer_exists(model, 0)