Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
perf(streaming): WILLNEED readahead + in-place staged stack for edge0-35b
bupalinyu62dcab22479b
diff --git a/src/edge0/streaming/layer.py b/src/edge0/streaming/layer.pyindex e794842..ef1320b 100644--- a/src/edge0/streaming/layer.py+++ b/src/edge0/streaming/layer.py@@ -294,6 +294,10 @@ class StreamingSwitchGLU:# machinery; it enters the LRU only when evicted from the stack.self._incr_writeback = (self._incr_mode and o.incr_writeback)+ # Async readahead hint before a prefetch/staged build (see+ # warm_experts_willneed): turns per-page demand faults into bulk+ # kernel readahead.+ self._warm_willneed = bool(getattr(o, "warm_willneed", False))self._incr_tensors = None # {(proj,part): core.array [n+1, ...]}self._incr_slot_table = None # core.array [num_experts] int32self._incr_occupancy = None # list[slot] -> expert id | None@@ -385,6 +389,30 @@ class StreamingSwitchGLU:sl = raw[e * per:(e + 1) * per]_ = sl.sum()+ def warm_experts_willneed(self, experts):+ """Async readahead hint over the byte ranges of ``experts``.++ One ``madvise(MADV_WILLNEED)`` per (expert, tensor) range: the+ kernel issues bulk readahead and returns immediately, so the+ subsequent build (or exact-path load) finds the pages resident and+ only takes MINOR faults. Unlike :meth:`warm_pages` this does not+ touch the pages itself, so the work does not serialize on the VM+ map lock and no MLX arrays are retained.+ """+ for e in experts:+ for proj in ("gate_proj", "up_proj", "down_proj"):+ for part in ("weight", "scales", "biases"):+ name = f"{self._prefix}.{proj}.{part}"+ try:+ sh = self._shard_for(name)+ except Exception: # noqa: BLE001 — advisory only+ continue+ ent = getattr(sh, "entries", {}).get(name)+ if ent is None:+ continue+ per = ent["size"] // self.num_experts+ sh.advise_willneed_range(ent["offset"] + e * per, per)+def _get_bundles(self, experts):"""Resolve bundles: pinned/LRU hits synchronously, misses via pool."""bundles = {}@@ -471,6 +499,8 @@ class StreamingSwitchGLU:):continuemissing.append(e)+ if missing and self._warm_willneed:+ self.warm_experts_willneed(missing)for e in missing:fut = self._prefetch_pool.submit(self._build, e)self._inflight[e] = fut@@ -690,6 +720,8 @@ class StreamingSwitchGLU:unique = sorted({int(e) for e in experts})if not unique:return+ if self._warm_willneed:+ self.warm_experts_willneed(unique)self._drain_rearm()with self._staging_lock:if self._staging_future is not None:diff --git a/src/edge0/streaming/mmap.py b/src/edge0/streaming/mmap.pyindex d8d5bdb..beebacb 100644--- a/src/edge0/streaming/mmap.py+++ b/src/edge0/streaming/mmap.py@@ -49,6 +49,33 @@ class SafetensorsMmap:except Exception: # noqa: BLE001 — advisory onlypass+ def advise_willneed_range(self, offset: int, length: int) -> None:+ """Async readahead hint over ONE byte range of the shard.++ ``madvise(MADV_WILLNEED)`` returns immediately and lets the kernel+ issue bulk readahead, instead of the caller demand-faulting page by+ page. Demand faults serialize on the VM map lock, so on a+ memory-starved host a step can degrade into "number of cold pages x+ per-fault latency"; a range hint moves that work to the kernel's+ readahead path in one syscall.++ The range is page-aligned outward because madvise requires it.+ """+ if length <= 0:+ return+ try:+ mmap_madv = getattr(mmap, "MADV_WILLNEED", None)+ if mmap_madv is None:+ return+ page = mmap.PAGESIZE+ start = (offset // page) * page+ end = (offset + length + page - 1) // page * page+ end = min(end, len(self._mm))+ if end > start:+ self._mm.madvise(mmap_madv, start, end - start)+ except Exception: # noqa: BLE001 — advisory only+ pass+def seq_read(self, chunk: int = 1 << 24):"""Force every page resident: one sequential pass over the shard.diff --git a/src/edge0/streaming/options.py b/src/edge0/streaming/options.pyindex ff71e1a..8a293f8 100644--- a/src/edge0/streaming/options.py+++ b/src/edge0/streaming/options.py@@ -83,6 +83,14 @@ class LayerOptions:full_layer_prefill: bool = Falseprefill_full_layers: int = 0prefill_hot: int = 0+ #: Issue madvise(MADV_WILLNEED) over the byte ranges of the experts a+ #: prefetch/stage is about to touch, before building bundles. Lets the+ #: kernel do bulk ASYNC readahead (one syscall per tensor range) instead+ #: of the loader DEMAND-faulting page by page — on a memory-starved host+ #: a step otherwise degrades into "cold pages x per-fault latency", with+ #: those faults serializing on the VM map lock. Retains no MLX arrays,+ #: so it does not displace the page cache.+ warm_willneed: bool = False# ---- presets ----------------------------------------------------------@@ -94,10 +102,31 @@ class LayerOptions:``staged_replace`` stays False: the trained prerouter head suppliesthe routing (the MoE block routes via the prerouter logits), so thestaged set == the routing set exactly — the slot table maps without- drops."""+ drops.++ Memory-starved hosts (measured on a 16 GiB M2, 19.7 GB checkpoint,+ 60-step replay, 5 rounds, same-run pairing):++ * ``staged_sync=True`` + ``incr_stack=True`` ++ ``incr_writeback=True`` — the incremental stack replaces the+ per-step ``core.stack`` with persistent in-place tensors and keeps+ the staged bundles out of the shared LRU. Peak MLX 2.92 -> 2.60+ GiB, file-backed page cache 4.25 -> 4.72 GiB, minor faults/step+ 5162 -> 1925, CPU 143 -> 109 ms/step. (Requires sync staged.)+ * ``warm_willneed=True`` — issue ``madvise(MADV_WILLNEED)`` over the+ predicted experts' byte ranges before building. Staging wall+ 162 -> 27 ms/step, because the kernel does bulk async readahead+ instead of the loader demand-faulting page by page (faults+ serialize on the VM map lock).++ Together: step 244.25 -> 187.58 ms, 4.09 -> 5.33 tok/s (+31.0%,+ paired 4/5), and the prerouter's own gain over ``prerouter=None``+ rises from +8.0% to +37.8% (paired 5/5) with bit-identical output.+ """return cls(staged=True, staged_replace=False, staged_n=4,staged_trigger=4, staged_sync=True, history_prefetch=True,+ incr_stack=True, incr_writeback=True, warm_willneed=True,hot_per_layer=0, hot_update_interval=4, hot_decay=0.75,cache_slots=64, prefetch_cap=48, load_threads=8,prefetch_threads=4, use_compile=True, top_k=4,