Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

perf(streaming): WILLNEED readahead + in-place staged stack for edge0-35b

bupalinyu62dcab22479b
diff --git a/src/edge0/streaming/layer.py b/src/edge0/streaming/layer.py
index e794842..ef1320b 100644
--- a/src/edge0/streaming/layer.py
+++ b/src/edge0/streaming/layer.py
@@ -294,6 +294,10 @@ class StreamingSwitchGLU:
# machinery; it enters the LRU only when evicted from the stack.
self._incr_writeback = (
self._incr_mode and o.incr_writeback)
+ # Async readahead hint before a prefetch/staged build (see
+ # warm_experts_willneed): turns per-page demand faults into bulk
+ # kernel readahead.
+ self._warm_willneed = bool(getattr(o, "warm_willneed", False))
self._incr_tensors = None # {(proj,part): core.array [n+1, ...]}
self._incr_slot_table = None # core.array [num_experts] int32
self._incr_occupancy = None # list[slot] -> expert id | None
@@ -385,6 +389,30 @@ class StreamingSwitchGLU:
sl = raw[e * per:(e + 1) * per]
_ = sl.sum()
+ def warm_experts_willneed(self, experts):
+ """Async readahead hint over the byte ranges of ``experts``.
+
+ One ``madvise(MADV_WILLNEED)`` per (expert, tensor) range: the
+ kernel issues bulk readahead and returns immediately, so the
+ subsequent build (or exact-path load) finds the pages resident and
+ only takes MINOR faults. Unlike :meth:`warm_pages` this does not
+ touch the pages itself, so the work does not serialize on the VM
+ map lock and no MLX arrays are retained.
+ """
+ for e in experts:
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ for part in ("weight", "scales", "biases"):
+ name = f"{self._prefix}.{proj}.{part}"
+ try:
+ sh = self._shard_for(name)
+ except Exception: # noqa: BLE001 — advisory only
+ continue
+ ent = getattr(sh, "entries", {}).get(name)
+ if ent is None:
+ continue
+ per = ent["size"] // self.num_experts
+ sh.advise_willneed_range(ent["offset"] + e * per, per)
+
def _get_bundles(self, experts):
"""Resolve bundles: pinned/LRU hits synchronously, misses via pool."""
bundles = {}
@@ -471,6 +499,8 @@ class StreamingSwitchGLU:
):
continue
missing.append(e)
+ if missing and self._warm_willneed:
+ self.warm_experts_willneed(missing)
for e in missing:
fut = self._prefetch_pool.submit(self._build, e)
self._inflight[e] = fut
@@ -690,6 +720,8 @@ class StreamingSwitchGLU:
unique = sorted({int(e) for e in experts})
if not unique:
return
+ if self._warm_willneed:
+ self.warm_experts_willneed(unique)
self._drain_rearm()
with self._staging_lock:
if self._staging_future is not None:
diff --git a/src/edge0/streaming/mmap.py b/src/edge0/streaming/mmap.py
index d8d5bdb..beebacb 100644
--- a/src/edge0/streaming/mmap.py
+++ b/src/edge0/streaming/mmap.py
@@ -49,6 +49,33 @@ class SafetensorsMmap:
except Exception: # noqa: BLE001 — advisory only
pass
+ def advise_willneed_range(self, offset: int, length: int) -> None:
+ """Async readahead hint over ONE byte range of the shard.
+
+ ``madvise(MADV_WILLNEED)`` returns immediately and lets the kernel
+ issue bulk readahead, instead of the caller demand-faulting page by
+ page. Demand faults serialize on the VM map lock, so on a
+ memory-starved host a step can degrade into "number of cold pages x
+ per-fault latency"; a range hint moves that work to the kernel's
+ readahead path in one syscall.
+
+ The range is page-aligned outward because madvise requires it.
+ """
+ if length <= 0:
+ return
+ try:
+ mmap_madv = getattr(mmap, "MADV_WILLNEED", None)
+ if mmap_madv is None:
+ return
+ page = mmap.PAGESIZE
+ start = (offset // page) * page
+ end = (offset + length + page - 1) // page * page
+ end = min(end, len(self._mm))
+ if end > start:
+ self._mm.madvise(mmap_madv, start, end - start)
+ except Exception: # noqa: BLE001 — advisory only
+ pass
+
def seq_read(self, chunk: int = 1 << 24):
"""Force every page resident: one sequential pass over the shard.
diff --git a/src/edge0/streaming/options.py b/src/edge0/streaming/options.py
index ff71e1a..8a293f8 100644
--- a/src/edge0/streaming/options.py
+++ b/src/edge0/streaming/options.py
@@ -83,6 +83,14 @@ class LayerOptions:
full_layer_prefill: bool = False
prefill_full_layers: int = 0
prefill_hot: int = 0
+ #: Issue madvise(MADV_WILLNEED) over the byte ranges of the experts a
+ #: prefetch/stage is about to touch, before building bundles. Lets the
+ #: kernel do bulk ASYNC readahead (one syscall per tensor range) instead
+ #: of the loader DEMAND-faulting page by page — on a memory-starved host
+ #: a step otherwise degrades into "cold pages x per-fault latency", with
+ #: those faults serializing on the VM map lock. Retains no MLX arrays,
+ #: so it does not displace the page cache.
+ warm_willneed: bool = False
# ---- presets ----------------------------------------------------------
@@ -94,10 +102,31 @@ class LayerOptions:
``staged_replace`` stays False: the trained prerouter head supplies
the routing (the MoE block routes via the prerouter logits), so the
staged set == the routing set exactly — the slot table maps without
- drops."""
+ drops.
+
+ Memory-starved hosts (measured on a 16 GiB M2, 19.7 GB checkpoint,
+ 60-step replay, 5 rounds, same-run pairing):
+
+ * ``staged_sync=True`` + ``incr_stack=True`` +
+ ``incr_writeback=True`` — the incremental stack replaces the
+ per-step ``core.stack`` with persistent in-place tensors and keeps
+ the staged bundles out of the shared LRU. Peak MLX 2.92 -> 2.60
+ GiB, file-backed page cache 4.25 -> 4.72 GiB, minor faults/step
+ 5162 -> 1925, CPU 143 -> 109 ms/step. (Requires sync staged.)
+ * ``warm_willneed=True`` — issue ``madvise(MADV_WILLNEED)`` over the
+ predicted experts' byte ranges before building. Staging wall
+ 162 -> 27 ms/step, because the kernel does bulk async readahead
+ instead of the loader demand-faulting page by page (faults
+ serialize on the VM map lock).
+
+ Together: step 244.25 -> 187.58 ms, 4.09 -> 5.33 tok/s (+31.0%,
+ paired 4/5), and the prerouter's own gain over ``prerouter=None``
+ rises from +8.0% to +37.8% (paired 5/5) with bit-identical output.
+ """
return cls(
staged=True, staged_replace=False, staged_n=4,
staged_trigger=4, staged_sync=True, history_prefetch=True,
+ incr_stack=True, incr_writeback=True, warm_willneed=True,
hot_per_layer=0, hot_update_interval=4, hot_decay=0.75,
cache_slots=64, prefetch_cap=48, load_threads=8,
prefetch_threads=4, use_compile=True, top_k=4,