Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

fix(35b): clip per-layer hidden states so one fp16 overflow cannot NaN-poison the net

linyubupa51b869722582
diff --git a/src/edge0/backends/mlx/_impl/qwen3_5.py b/src/edge0/backends/mlx/_impl/qwen3_5.py
index c5cf965..3472fee 100644
--- a/src/edge0/backends/mlx/_impl/qwen3_5.py
+++ b/src/edge0/backends/mlx/_impl/qwen3_5.py
@@ -6,8 +6,10 @@
# mlx_lm package, sibling model files point at this vendored package so
# the prerouter block patch lands on OUR ``Qwen3NextSparseMoeBlock``;
# (2) ``Qwen3_5TextModel.__call__`` gained the edge0 engine callback
-# hooks (before/after_layer_cb + async_eval_per_layer).
+# hooks (before/after_layer_cb + async_eval_per_layer) plus the
+# hidden-state clip / NaN scrub (``QWEN_HIDDEN_CLIP``).
+import os
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union
@@ -272,11 +274,23 @@ class Qwen3_5TextModel(nn.Module):
fa_mask = create_attention_mask(hidden_states, cache[self.fa_idx])
ssm_mask = create_ssm_mask(hidden_states, cache[self.ssm_idx])
+ # Hidden-state clip (QWEN_HIDDEN_CLIP, e.g. 1000): clamps each
+ # layer's output to [-v, +v] and zeroes NaN entries so a single
+ # fp16 overflow inside one layer cannot poison the whole network
+ # (edge0-35b: prefill overflow at layer 10 -> layers 11+ all-NaN ->
+ # all-NaN logits -> argmax falls back to token 0 ('!') collapse).
+ # Off by default, enabled by the engine (parity with the ling
+ # LING_HIDDEN_CLIP fix).
+ _clip_v = float(os.environ.get("QWEN_HIDDEN_CLIP", "0"))
for li, (layer, c) in enumerate(zip(self.layers, cache)):
if before_layer_cb is not None:
before_layer_cb(li)
mask = ssm_mask if layer.is_linear else fa_mask
hidden_states = layer(hidden_states, mask=mask, cache=c)
+ if _clip_v > 0:
+ hidden_states = mx.where(
+ mx.isnan(hidden_states), mx.zeros_like(hidden_states),
+ mx.clip(hidden_states, -_clip_v, _clip_v))
if after_layer_cb is not None:
after_layer_cb(li, hidden_states)
if async_eval_per_layer:
diff --git a/src/edge0/engine/qwen.py b/src/edge0/engine/qwen.py
index dbd3d54..12e0953 100644
--- a/src/edge0/engine/qwen.py
+++ b/src/edge0/engine/qwen.py
@@ -14,6 +14,8 @@ Port of the deployment's ``engine_qwen.py`` trained-prerouter path:
from __future__ import annotations
+import os
+
from edge0.backends import core
from edge0.backends.mlx._impl.qwen3_5_moe import Model as Qwen35Model
@@ -86,6 +88,16 @@ class Qwen35Engine(Edge0Engine):
name = "edge0-35b"
+ def __init__(self, model_dir: str, cfg, tokenizer=None):
+ # NAN_BANG_COLLAPSE_FIX parity (engine/ling.py): hidden clip default
+ # 1000 unless the deployer overrides QWEN_HIDDEN_CLIP explicitly.
+ # One fp16 overflow inside a layer otherwise poisons the whole net
+ # into all-NaN logits -> argmax fallback token 0 ('!') collapse,
+ # which does not recover until the process restarts.
+ if "QWEN_HIDDEN_CLIP" not in os.environ:
+ os.environ["QWEN_HIDDEN_CLIP"] = "1000"
+ super().__init__(model_dir, cfg, tokenizer=tokenizer)
+
def _build(self):
cfg = self.cfg
(self.model, self.model_config, self.shards,