Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

edge0 initial commit: MLX streaming MoE inference (prerouter + LoRA + SSD offload)

linyubupa17021c587146
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..e1bc0b7
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,20 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.pytest_cache/
+.coverage
+
+# Virtualenvs
+.venv/
+
+# Trained adapter artifacts (never committed; see README)
+artifacts/
+
+# macOS
+.DS_Store
+
+# Bench / runtime logs
+*.log
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..ddee8a9
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,21 @@
+edge0
+=====
+
+This project bundles or adapts the following third-party works:
+
+mlx-lm (MIT License)
+ - `src/edge0/backends/mlx/_impl/qwen3_next.py` is vendored from mlx-lm
+ 0.31.0 (https://github.com/ml-explore/mlx-lm), with additive hooks for
+ prerouter routing and layer callbacks. Non-prerouter paths are kept
+ byte-identical to the upstream file.
+ - mlx-lm is used at runtime for checkpoint / tokenizer loading.
+ Copyright (c) 2023-2024 Apple Inc. and the mlx-lm contributors.
+
+bailing_hybrid (Apache License 2.0)
+ - `src/edge0/backends/mlx/_impl/bailing_hybrid.py` is vendored from the
+ Ling 3.0 MLX implementation (rapid-mlx / Ling-3.0-tiny, Apache-2.0), with
+ prerouter hook identifiers renamed for consistency with the framework.
+ Copyright (c) the bailing / Ling MLX contributors.
+
+All other files are Copyright (c) 2025 edge0 contributors, licensed under
+the Apache License, Version 2.0 (see LICENSE).
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..c74f2e9
--- /dev/null
+++ b/README.md
@@ -0,0 +1,72 @@
+# edge0
+
+**edge0** 是一个面向 Apple Silicon 的开源流式 MoE 推理框架:把「SSD 专家 offload +
+并行 LoRA + prerouter 路由预判」方案抽象成可扩展的通用框架,开箱支持两个模型:
+
+| 模型 | 说明 | 推理档 |
+|---|---|---|
+| `edge0-35b` | Qwen3.6-35B-A3B 4bit(40 层,256 专家) | prerouter K=4 |
+| `edge0-10b` | Ling-10B bailing_hybrid(24 层,128 专家) | prerouter K=8 |
+
+设计目标:
+
+- **像 transformers 一样使用**:`AutoModel` / `AutoConfig` / `AutoEngine` 按模型名自动选类;
+- **后端隔离**:全部 MLX 代码收在 `edge0/backends/mlx/`,核心逻辑只依赖后端门面,
+ 为未来 CUDA 后端预留平级插槽(`backends/cuda/`);
+- **适配器统一为 safetensors**:LoRA 与 prerouter 权重均为带元数据的 `.safetensors`;
+- **旧术语零残留**:预路由头在代码与文档中一律称 `prerouter`。
+
+## 快速开始
+
+```bash
+# 1) 安装(Python ≥3.10,Apple Silicon + mlx)
+python3.12 -m venv .venv && .venv/bin/pip install -e '.[dev]'
+
+# 2) 一键试玩:自动探测开发机上的 checkpoint;也可以显式传路径或 tier 名
+edge0 demo
+edge0 demo /path/to/qwen35/model
+edge0 demo edge0-10b
+
+# 3) vLLM 风格起服务:模型是位置参数,checkpoint 类型自动识别
+edge0 serve /path/to/qwen35/model
+edge0 serve edge0-35b # tier 名:EDGE0_35B_MODEL 环境变量可覆盖默认路径
+
+curl http://127.0.0.1:8000/v1/chat/completions \
+ -H 'Content-Type: application/json' \
+ -d '{"messages":[{"role":"user","content":"Hello!"}],"max_tokens":32}'
+
+# 4) 命令行一问一答(同样接受路径或 tier 名)
+edge0 chat /path/to/model --prompt "你好,用一句话介绍珠海。"
+```
+
+`python -m edge0 ...` 与 `edge0 ...` 完全等价。
+
+### 模型与适配器
+
+- **checkpoint**:qwen35 / ling v7 的原始模型目录(含 `config.json`、`model.safetensors`、
+ `tokenizer.json`)。`edge0 serve <dir>` 按 `config.json` 自动识别 tier。
+- **适配器**(LoRA + prerouter,`artifacts/` 目录内自动使用,无需手动指定):
+ - `artifacts/lora_edge0_35b_k4.safetensors` + `artifacts/prerouter_edge0_35b_k4.safetensors`
+ - `artifacts/lora_edge0_10b.safetensors` + `artifacts/prerouter_edge0_10b.safetensors`
+ - 旧版 npz 迁移:`edge0 convert-adapters`
+
+### 验证
+
+```bash
+pytest # 单元测试(不含真实权重)
+pytest -m slow # 真实 checkpoint 端到端(qwen/ling 生成 + HTTP)
+scripts/e2e_smoke.py # 两档模型 staged vs exact 数值一致性冒烟
+scripts/zhuhai_example.py # 完整 API 上手例子(--thinking 开关演示)
+examples/demo.py # 最小 API walkthrough(edge0 demo 的等价代码)
+```
+
+## 文档
+
+- [架构总览](docs/architecture.md)
+- [注意力抽象](docs/attention.md) / [MoE 抽象](docs/moe.md) / [SSD 流式](docs/streaming.md) / [prerouter](docs/prerouter.md)
+- [如何接入新模型](docs/adding-a-model.md)
+- [edge0-35b](docs/models/edge0-35b.md) / [edge0-10b](docs/models/edge0-10b.md)
+
+## License
+
+Apache-2.0,含 vendored 第三方代码(详见 [NOTICE](NOTICE.md))。
diff --git a/docs/adding-a-model.md b/docs/adding-a-model.md
new file mode 100644
index 0000000..5e784ec
--- /dev/null
+++ b/docs/adding-a-model.md
@@ -0,0 +1,207 @@
+# 接入一个新模型
+
+本文档给出向 edge0 接入一个新模型族的分步指南,用 `edge0-35b`(`src/edge0/models/
+edge0_35b/__init__.py`)作为逐步示例,逐处标注真实代码行号。`edge0-10b`
+(`src/edge0/models/edge0_10b/__init__.py`)是第二个活例子,路径解析与契约完全一致。
+
+接入目标:让 `AutoConfig` / `AutoModel` / `AutoEngine` 能按模型名(或 checkpoint 的
+`config.json` 的 `model_type`)解析你的模型,且流式 MoE 层能由一个通用的
+`StreamingSwitchGLU` 驱动。
+
+## 接入总览(契约)
+
+`edge0.registry` 的契约是:**一个适配器模块暴露三件套**:
+
+- `Config` —— 一个 `ModelConfig` 子类(含 `from_pretrained`);
+- `build_model(model_dir, **overrides)` —— 装配好的模型骨架;
+- `build_engine(model_dir, **overrides)` —— 可生成引擎。
+
+`register_model(name, adapter)` 注册;`TYPE_ALIASES` 把 checkpoint 的 `model_type`
+映射到注册名。见 `registry.py` 的 `_resolve_name` 与 `AutoConfig.from_pretrained`
+(registry.py:73–101)。
+
+## 第 1 步:创建适配器模块与目录
+
+在 `src/edge0/models/` 下建一个包,名取你的档位,例如 `edge0_35b/`。目录内至少一个
+`__init__.py` 作为适配器。模型目录布局要求见「模型目录布局」一节。
+
+## 第 2 步:定义 `Config` 子类与 `_defaults`
+
+`Config` 是 `ModelConfig`(`models/base.py`)的子类。`ModelConfig` 的字段即「跑起一个
+模型档位需要的一切」:
+
+| 字段 | 类型 | 语义 |
+| --- | --- | --- |
+| `name` | `str` | 注册名 |
+| `model_dir` | `str` | checkpoint 目录 |
+| `moe_spec` | `MoESpec` | MoE 规格(见 moe.md) |
+| `options` | `LayerOptions` | 流式层选项 |
+| `prerouter` | `PrerouterSpec \| None` | 预路由头规格 |
+| `prerouter_top_k` | `int` | 预路由宽度(0 → 用 `options.top_k`) |
+| `lora` / `lora_r` / `lora_alpha` | `str` / `int` / `float` | LoRA 权重路径与超参(`""` 禁用) |
+| `gen` | `GenerationConfig` | 采样默认(温度 / top-p / top-k / eos 等) |
+| `prefill_chunk` / `hot_window` / `intra_staging` / `prefetch_history` | — | 流式与预取行为 |
+| `port` | `int` | 服务端口 |
+| `target_tok_s` / `peak_active_mem_mb` | `float` | 验收指标(生产机实测) |
+
+子类只需实现类方法 `_defaults(model_dir) -> Config`,返回族默认配置。edge0-35b 的
+实现见 `edge0_35b/__init__.py:31–69`:`_defaults` 用一份完整的 `MoESpec`、`LayerOptions`
+预设 `staged_k4()`、`PrerouterSpec`、采样默认与验收指标构造 `Qwen35Config`。
+
+`from_pretrained(model_dir=None, **overrides)` 是 `ModelConfig` 提供的模板方法
+(`models/base.py:60–73`):它调用 `_defaults` 得到基配置,再对每个 override 校验后
+`replace` 覆盖;未知字段抛 `TypeError` 并列出已知字段。因此**每个 public 属性都能被
+用户 override**。
+
+要点:
+
+- 仓库根目录有 `scripts/convert_adapters_legacy.py` 一次性把训练 npz 导出转成
+ safetensors 产物到 `artifacts/`;`ModelConfig.artifact(name)`(`models/base.py:28–30`)
+ 返回这些产物的绝对路径,适配器用它填 LoRA / prerouter 权重路径。
+- LoRA override 走 `resolve_lora`(`models/base.py:88–95`):裸模型名解析为该档产物,
+ `"model_dir"` 表示「保留训练权重在原位」,空串禁用。
+
+## 第 3 步:`build_model` / `build_engine`
+
+两个模块级函数,复用引擎自己的构建路径(DRY)。
+
+`edge0_35b/__init__.py:72–79`:
+
+```python
+def build_model(model_dir=None, **overrides):
+ from edge0.engine.qwen import load_installed
+ cfg = Qwen35Config.from_pretrained(model_dir, **overrides)
+ model, _mcfg, _shards, _installs = load_installed(cfg.model_dir, cfg)
+ return model
+```
+
+`edge0_35b/__init__.py:82–87`:
+
+```python
+def build_engine(model_dir=None, **overrides):
+ from edge0.engine.qwen import Qwen35Engine
+ cfg = Qwen35Config.from_pretrained(model_dir, **overrides)
+ return Qwen35Engine(cfg.model_dir, cfg)
+```
+
+你的模型若族数学不同(如 edge0-10b 的 `SIGMOID_GROUP` + 混合 MLA),在
+`src/edge0/engine/` 写一个对应引擎(参照 `engine/qwen.py` / `engine/ling.py`),
+`build_model` / `build_engine` 引用它。`load_installed` 负责装配流式孪生 / LoRA /
+prerouter,是框架给引擎提供的构建路径。
+
+## 第 4 步:注册到注册表
+
+文件末尾调用 `register_model`,并把 `Config` 暴露为模块属性(契约要求):
+
+```python
+# edge0_35b/__init__.py:90–93
+register_model("edge0-35b", sys.modules[__name__])
+Config = Qwen35Config # registry contract: adapter.Config
+```
+
+- `register_model`(`registry.py:25–28`):重名抛 `ValueError`。
+- 适配器模块必须被 import 才会注册。`edge0.models.__init__`(models/__init__.py:15)
+ 显式 import 各档包来 populate 注册表,`AutoConfig.from_pretrained` 里也会
+ `from edge0 import models` 兜底。
+- 你的模型包应加进 `edge0/models/__init__.py` 的 import 列表。
+
+## 第 5 步:`TYPE_ALIASES`
+
+`TYPE_ALIASES`(`registry.py:15–22`)把 checkpoint `config.json` 的 `model_type`
+映射到注册名,使 `AutoEngine.from_pretrained(model_dir=...)` 能**直接从目录解析**而无需
+显式传 name。例:
+
+```python
+TYPE_ALIASES = {
+ "qwen3_5_moe_text": "edge0-35b",
+ "qwen3_5_moe": "edge0-35b",
+ "bailing_hybrid": "edge0-10b",
+ "bailing_moe_linear": "edge0-10b",
+}
+```
+
+`_resolve_name`(registry.py:42–56)的查找顺序:显式 `name` → 目录的 `model_type`
+(`_model_type_from_dir` 读 `config.json`,失败则退回目录 basename)→ 匹配
+`MODEL_REGISTRY` → 匹配 `TYPE_ALIASES` → 否则 `KeyError` 列出已注册模型。
+未知名会因 `test_unknown_name_lists_registry` 之类的断言被测试钉住。
+
+## 模型目录布局要求
+
+`AutoConfig.from_pretrained(model_dir=...)` 会读 `model_dir/config.json` 的
+`model_type` 字段(`registry.py:58–70`)。因此 checkpoint 目录至少要满足:
+
+- `config.json` 存在,且 `model_type` 已登记(或你的 `TYPE_ALIASES` 覆盖该值);
+- 权重为 safetensors 格式,key 前缀与 `moe_spec.key_template` 一致(如
+ `language_model.model.layers.N.mlp.switch_mlp`);
+- LoRA / prerouter 权重为带元数据的 `.safetensors`(`models/base.py` 顶部注释:由
+ `convert_adapters_legacy.py` 从训练 npz 一次性转换)。
+
+目录 basename 是最后兜底的解析手段(`_model_type_from_dir`),所以目录名最好与
+注册名对齐,但不是必须。
+
+## 第 6 步:测试建议
+
+参考 `tests/test_moe_spec.py` 与 `tests/test_registry.py`(纯逻辑单测,无需真实
+checkpoint)。
+
+- **注册与别名**(test_registry.py:12–20):断言 `MODEL_REGISTRY` 含你的注册名、
+ `TYPE_ALIASES` 解析正确。
+- **Config 默认值**(test_registry.py:23–33):`AutoConfig.from_pretrained(name=...)`
+ 的 `moe_spec` / `options` / `prerouter` / `prerouter_top_k` 等字段符合预期。
+- **profile 细化**(test_registry.py:35–71):逐字段断言你的档位数值(专家数、top_k、
+ quant、端口、验收指标等)。
+- **override 与拒绝**(test_registry.py:74–81):`from_pretrained(..., prerouter=None,
+ lora="")` 生效;未知字段抛 `TypeError`。
+- **适配器契约**(test_registry.py:89–92):断言 `MODEL_REGISTRY` 每个条目都暴露
+ `Config` / `build_model` / `build_engine`。
+- **路径解析**(test_moe_spec.py):`keys` 模板、`block_of` 数字段下标、`layer_of`
+ 从 `block_path` 推出的默认、`bundle_projs` 随布局切换。
+- **数学 parity**:路由数学必须与 vendored 模型 bit 级一致(`moe/routing.py` 用 parity
+ 测试钉死),若你的模型引入新路由种类需补对照。
+
+## 完整最小骨架(对照 edge0-35b)
+
+```python
+# src/edge0/models/my_model/__init__.py
+import sys
+from edge0.models.base import ModelConfig
+from edge0.moe.spec import MoESpec, QuantSpec, RouterKind, WeightLayout
+from edge0.registry import register_model
+
+class MyConfig(ModelConfig):
+ @classmethod
+ def _defaults(cls, model_dir):
+ return cls(
+ name="my-model",
+ model_dir=model_dir,
+ moe_spec=MoESpec(
+ num_experts=..., top_k=..., intermediate_size=...,
+ router=RouterKind.SOFTMAX_TOPK,
+ quant=QuantSpec(bits=4, group_size=64, mode="affine"),
+ layout=WeightLayout.SEPARATE,
+ key_template="model.layers.{layer}.mlp.experts",
+ block_path="model.layers.{layer}.mlp",
+ layer_path="model.layers.{layer}",
+ ),
+ options=LayerOptions.staged_k4(), # 你的档位用 staged_k4 / staged_k8 等预设
+ prerouter=...,
+ )
+
+def build_model(model_dir=None, **overrides):
+ cfg = MyConfig.from_pretrained(model_dir, **overrides)
+ return ... # load_installed 或你的装配路径
+
+def build_engine(model_dir=None, **overrides):
+ cfg = MyConfig.from_pretrained(model_dir, **overrides)
+ return ... # 你的引擎
+
+register_model("my-model", sys.modules[__name__])
+Config = MyConfig
+```
+
+## 参考
+
+- `src/edge0/registry.py` —— 注册表 + Auto 三件套
+- `src/edge0/models/base.py` —— `ModelConfig` / `from_pretrained` / artifact 解析
+- `src/edge0/models/edge0_35b/__init__.py`、`edge0_10b/__init__.py` —— 两个活例子
+- `tests/test_registry.py`、`tests/test_moe_spec.py` —— 行为示例
diff --git a/docs/architecture.md b/docs/architecture.md
new file mode 100644
index 0000000..1eb557f
--- /dev/null
+++ b/docs/architecture.md
@@ -0,0 +1,136 @@
+# 架构总览
+
+**edge0** 是一个面向 Apple Silicon 的开源流式 MoE 推理框架,把生产部署中验证过的
+「SSD 专家 offload + 并行 LoRA + prerouter 路由预判」方案抽象成可扩展的通用框架。
+开箱支持两个模型:`edge0-35b`(Qwen3.5-MoE,K=4)与 `edge0-10b`(Ling 3.0
+hybrid,K=8)。
+
+## 核心设计目标
+
+1. **像 transformers 一样使用**:`AutoConfig` / `AutoModel` / `AutoEngine` 按模型
+ 名(或 checkpoint 的 `model_type`)自动解析适配器;
+2. **后端隔离**:全部 MLX 代码收在 `edge0/backends/mlx/`,框架代码只通过
+ `edge0.backends` 暴露的 `core` / `nn` / `io` / `quant` 命名空间接触后端,
+ 为未来 CUDA 后端预留平级插槽(`backends/cuda/`,由 `EDGE0_BACKEND` 环境变量
+ 选择);
+3. **适配器统一为 safetensors**:LoRA 与 prerouter 权重均为带元数据的
+ `.safetensors`,旧 npz 训练导出只经一次性迁移脚本转换后即弃用;
+4. **术语统一**:预路由头一律称 **prerouter**,代码与文档零旧术语残留。
+
+## 分层结构
+
+```
+src/edge0/
+├── backends/ # 后端抽象(唯一允许接触 MLX 的边界)
+│ ├── base.py # TensorStore 协议 + open_tensor_store()
+│ ├── __init__.py # 按 EDGE0_BACKEND 装配 core/nn/io/quant
+│ └── mlx/ # MLX 参考实现(_impl/ 为 vendored 模型)
+├── config.py # GenerationConfig(采样参数)
+├── sampling.py # 采样器(温度/top-k/top-p/重复惩罚,经 backends.core)
+├── registry.py # 模型注册表 + AutoConfig/AutoModel/AutoEngine
+├── attention/spec.py # 注意力规格(MHA/MLA 差异的抽象层)
+├── moe/ # MoE 抽象
+│ ├── spec.py # MoESpec/QuantSpec/RouterKind/WeightLayout
+│ └── routing.py # 路由数学(与 vendored 模型 bit 级一致)
+├── streaming/ # SSD 流式专家层
+│ ├── options.py # LayerOptions(typed 替换部署期 env 旋钮)
+│ ├── layer.py # StreamingSwitchGLU(每层状态机,全部执行路径)
+│ ├── mmap.py # SafetensorsMmap(byte-range mmap)
+│ └── cache.py # SharedExpertCache(跨层 LRU)
+├── prerouter/ # 路由预判
+│ ├── spec.py # PrerouterSpec(纯配置)
+│ ├── install.py # 头权重安装
+│ ├── heads.py # 头数学(fc1 -> erf gelu -> fc2 + linear_init)
+│ ├── stager.py # 跨 token 预测保存/提交(每家族子类)
+│ └── state.py # 跨 token 状态
+├── engine/ # 推理编排(prefill/decode 循环,按模型子类化)
+│ ├── base.py # 共享循环(经 backends.core)
+│ ├── qwen.py # Qwen35Engine(K=4 档)
+│ └── ling.py # Ling10BEngine(K=8 档)
+├── adapters/lora.py # 并行 LoRA 安装(backends.nn)
+├── models/ # 模型适配层(transformers 风格)
+│ ├── base.py # ModelConfig 基类 + artifact 路径解析
+│ ├── edge0_35b/ # edge0-35b 档(注册名 "edge0-35b")
+│ └── edge0_10b/ # edge0-10b 档(注册名 "edge0-10b")
+└── server/ # OpenAI 兼容 HTTP 服务(纯 stdlib)
+ ├── app.py # ThreadingHTTPServer + 路由
+ └── chat.py # /v1/chat/completions 会话状态
+```
+
+### 依赖方向
+
+```
+server / cli
+ └─ engine ──> streaming / prerouter / adapters ──> moe / attention spec
+ └──────────────> backends.{core,nn,io,quant}
+ └─> backends/mlx/(MLX 唯一入口)
+```
+
+纯配置层(`config.py`、`moe/spec.py`、`prerouter/spec.py`、`streaming/options.py`、
+`registry.py`、`models/*`、`server/`)不 import 任何后端包;运行时逻辑一律通过
+`edge0.backends` 命名空间。CI 用 grep 强制:MLX 包不得在 `backends/mlx/` 之外被
+import。
+
+## 关键数据流
+
+### Prefill(整层装载)
+
+对前 `prefill_full_layers` 个层:`load_full_layer()` 把整层 9 个张量(gate/up/down
+× weight/scales/biases)按 bundle 布局装入,`_gather_sort` + 排序 gather + 反排序
+一次前向算完,随后 `clear_full_layer()` 释放,页缓存承担热数据(E3b 策略)。
+其余层走 hot-stack(LRU 常驻 top-N 专家)或 on-demand exact 路径。
+
+### Decode(staged 双缓冲)
+
+1. prerouter 头在 token t-1 预测 token t 的专家集;
+2. `stager` 把预测写入双缓冲槽;step 边界同步/异步填充槽位;
+3. `StreamingSwitchGLU` 用槽位表 + 增量栈(`incr_stack`)把 9 个
+ `mx.stack` 图节点换成本地 `put_along_axis` 行写入;
+4. 专家权重经 `quant.gather_qmm` 按槽位 gather,路由索引全程不离开 GPU。
+
+### 采样与生成
+
+`engine/base.py` 的共享循环驱动 prefill → decode,逐 token 调
+`sampling.sample()`(温度/top-k/top-p/重复惩罚,向量化历史惩罚,单次
+host-sync categorical 抽样)。
+
+## 注册表与模型接入
+
+适配器模块暴露三件套(契约):
+
+- `Config` —— `ModelConfig` 子类(`from_pretrained` 合并 override);
+- `build_model(model_dir, **overrides)` —— 装配好的模型骨架;
+- `build_engine(model_dir, **overrides)` —— 可生成引擎。
+
+`register_model(name, module)` 注册;`TYPE_ALIASES` 把 checkpoint
+`config.json` 的 `model_type`(如 `qwen3_5_moe`、`bailing_hybrid`)映射到注册名,
+因此 `AutoEngine.from_pretrained(model_dir=...)` 可以直接从目录解析。
+详见 [adding-a-model.md](adding-a-model.md)。
+
+## 后端隔离的落法
+
+`edge0.backends` 是框架唯一允许出现的后端 import 面:
+
+- `core` —— 数组与算子命名空间(matmul/softmax/topk/take/eval/compile/random 等);
+- `nn` —— 模块工厂(Module/Linear/RMSNorm/silu/gelu);
+- `io` —— 模型/分词器/张量库加载;
+- `quant` —— 量化 gather 内核(`gather_qmm`)。
+
+MLX 后端是参考实现;未来 CUDA 后端实现同一表面即可复用全部框架代码。
+注意:**每个模型的引擎胶水**(`engine/qwen.py`、`engine/ling.py`)因绑定 vendored
+后端模型而引用 `edge0.backends.mlx._impl`,这是有意的例外——模型本身就是后端
+资产,编排循环(`engine/base.py`)才是跨后端共享的。
+
+## 测试策略
+
+- `tests/test_streaming_math.py` —— 数学契约:exact/staged/hot/full 各路径与
+ dequant 参考逐元素对比(相对 L2 < 1%),防 gate/up 交换、行错位类回归;
+- `tests/test_moe_spec.py`、`tests/test_registry.py`、`tests/test_sampling.py` ——
+ 纯逻辑层单测,无需真实 checkpoint;
+- 验收指标:`target_tok_s` 与 `peak_active_mem_mb` 记录在生产机器上实测的
+ 吞吐与峰值活跃内存,作为回归标尺。
+
+## License
+
+Apache-2.0,含 vendored 第三方模型代码(mlx-lm 的 Qwen3.5-MoE、生产部署的
+ling backbone),详见根目录 [NOTICE](../NOTICE)。
diff --git a/docs/attention.md b/docs/attention.md
new file mode 100644
index 0000000..84fa02e
--- /dev/null
+++ b/docs/attention.md
@@ -0,0 +1,126 @@
+# 注意力规格(AttentionSpec)
+
+`edge0` 不重新实现注意力内核:具体的序列混合计算由 **vendored 基础模型**携带
+(`edge0.backends.mlx._impl` 中的 GQA / GatedDeltaNet 对应 edge0-35b,
+MLA / DeltaNet 对应 edge0-10b)。注意力内核属于**后端模型资产**,不属于框架。
+
+那为什么还要一个 `AttentionSpec`?因为框架需要在**不依赖具体实现**的情况下
+**审视(introspect)**一个模型:缓存尺寸、层角色、文档、未来的内核适配。
+`AttentionSpec` 就是这层「注意力分类学」的载体。本文档说明它是什么、字段语义、
+引擎如何使用,以及为什么单独抽象(MLA/MHA 的差异)。
+
+源码:`src/edge0/attention/spec.py`。
+
+## 设计动机
+
+模块头部的 docstring 明确了边界:
+
+```
+edge0 does not re-implement attention kernels: the vendored base model
+implementations (edge0.backends.mlx._impl) carry the kernels (GQA /
+GatedDeltaNet for edge0-35b, MLA / DeltaNet for edge0-10b). This module
+describes attention so the framework can introspect a model — cache
+sizing, layer roles, documentation, and future kernels — without knowing
+the concrete implementation.
+```
+
+新增一种注意力类型 = 一个新的 `AttentionKind` 成员 + 一份 `AttentionSpec` 描述 +
+一个内核实现(在 vendored 基础模型或某个后端的 `_impl` 模块里)。
+
+## 注意力种类:`AttentionKind`
+
+`AttentionKind` 是 `str` 枚举,标识 edge0 支持的注意力 / 序列混合家族。
+
+| 成员 | 值 | 含义 |
+| --- | --- | --- |
+| `GQA` | `"gqa"` | 分组查询注意力,softmax + KV cache |
+| `MLA` | `"mla"` | 多头潜在注意力(压缩 KV) |
+| `DELTANET` | `"deltanet"` | DeltaNet 线性注意力(门控 delta 规则,无 KV cache) |
+| `GATED_DELTANET` | `"gated_deltanet"` | GatedDeltaNet(带衰减的门控 delta / 线性注意力) |
+| `DENSE_MLP` | `"dense_mlp"` | 无序列混合(纯 MLP 层) |
+
+`DENSE_MLP` 的存在说明:一个模型的某些层可能根本没有注意力块(例如 edge0-10b 的
+layer 0 是 dense 层),分类学要能显式表达「这一层不做序列混合」。
+
+## 规格:`AttentionSpec`
+
+`AttentionSpec` 是一个 `frozen=True` 的 dataclass,描述**一个层**的序列混合块。
+
+| 字段 | 类型 | 默认值 | 语义 |
+| --- | --- | --- | --- |
+| `kind` | `AttentionKind` | (必填) | 注意力家族 |
+| `layer_indices` | `tuple[int, ...]` | (必填) | 该规格覆盖的 0 基层号 |
+| `num_heads` | `int \| None` | `None` | 头数(仅 GQA/MLA;否则为 `None`) |
+| `num_kv_heads` | `int \| None` | `None` | KV 头数(仅 GQA/MLA) |
+| `head_dim` | `int \| None` | `None` | 每头维度(仅 GQA/MLA) |
+| `cache` | `bool` | `True` | 该块是否维护 KV 式缓存(线性注意力层为 `False`) |
+| `notes` | `str` | `""` | 自由说明(例如混合调度计划) |
+
+要点:
+
+- `cache` 字段区分了「softmax 注意力(要 KV cache)」与「线性注意力(无 KV cache)」,
+ 这是缓存尺寸计算的关键分叉。
+- `layer_indices` 是一个元组而非单层,因为**一个规格可以覆盖多个层**(例如一个
+ GatedDeltaNet 规格可以描述连续的若干层),也便于表达混合调度。
+- `__repr__` 被设计为**紧凑单行**,便于打日志 / 文档,例如:
+ `AttentionSpec(gated_deltanet, layers=0..19, cache=False)`。
+
+### 汇总:`summarize()`
+
+```python
+def summarize(specs: list[AttentionSpec]) -> str
+```
+
+把一份规格列表压缩成一行人类可读摘要,例如用于 CLI 启动横幅:
+
+```
+gated_deltanetx20, dense_mlpx1
+```
+
+即 `{kind.value}x{len(layer_indices)}` 的逗号拼接。文档标注它用于 CLI banner。
+
+## 引擎如何使用
+
+当前源码里,`AttentionSpec` 及其辅助函数**尚未被任何引擎 / 模型代码引用**——它是一份
+「先定义、后接线」的分类学模块(`grep` 全仓仅命中 `attention/spec.py` 自身)。
+这一点需要诚实标注:`AttentionKind` / `AttentionSpec` / `summarize` 目前的消费方只有
+本模块,属于面向未来的框架设施,其设计意图(缓存尺寸、层角色、文档、未来内核)由
+docstring 与字段定义承载。
+
+即便如此,抽象方向已经明确:
+
+- **缓存尺寸**:`cache` 字段让框架无需理解内核细节即可判断该层要不要分配 KV cache、
+ 以及(对 GQA/MLA)按 `num_heads` / `num_kv_heads` / `head_dim` 计算尺寸。
+- **层角色**:`kind` 把「softmax 注意力 / 线性注意力 / 无混合」区分开,供调度、文档与
+ 性能剖析参考。
+- **未来内核**:新增注意力类型无需改动框架其余部分,只要新加枚举成员 + 规格 + 内核。
+
+这与后端抽象(`edge0.backends`)是一致的思想:框架代码只依赖抽象的规格 / 命名空间,
+具体数学由后端或 vendored 模型实现。
+
+## 为什么单独抽象(MLA / MHA 差异)
+
+与其说「引擎在跑 attention」,不如说引擎在编排「一层序列混合」。把注意力显式抽象出来,
+是为了容纳差异巨大的实现而不改框架:
+
+| 维度 | MHA / GQA(softmax) | MLA / 线性注意力 |
+| --- | --- | --- |
+| KV 表示 | 完整缓存,随层增长 | 压缩潜在 KV |
+| 缓存需求 | 需要 KV cache(`cache=True`) | 无 / 极简缓存(`cache=False`) |
+| 头几何 | `num_heads` / `num_kv_heads` / `head_dim` 有意义 | 通常不适用(`None`) |
+| 混合层级 | 几乎每层 | 可能只有部分层(含 dense 层) |
+
+`AttentionSpec` 用统一的字段集表达这两类差异:`kind` 表达家族,`cache` 表达缓存有无,
+头几何字段仅在 GQA/MLA 下填充,`layer_indices` 表达混合层级与调度。这样框架对
+edge0-35b 的 GatedDeltaNet 与 edge0-10b 的 MLA 可以走同一套审视逻辑,而不必各自
+硬编码。
+
+## 新增一种注意力的步骤
+
+按模块 docstring,新增 `Xxx` 注意力:
+
+1. 在 `AttentionKind` 加一个成员;
+2. 写一份 `AttentionSpec` 描述;
+3. 提供内核实现(vendored 基础模型或某后端 `_impl` 模块)。
+
+以上即 `src/edge0/attention/spec.py` 的全部内容与意图。
diff --git a/docs/models/edge0-10b.md b/docs/models/edge0-10b.md
new file mode 100644
index 0000000..ec37560
--- /dev/null
+++ b/docs/models/edge0-10b.md
@@ -0,0 +1,163 @@
+# edge0-10b
+
+edge0 平台的高吞吐轻量档:基于 Ling 3.0 混合架构(MLA + MoE)的 10B 级稀疏混合专家模型。相比 35b 档牺牲一定的参数量,换来更低的峰值内存与更高的生成速率,适合对延迟与显存敏感的场景。
+
+生产档案基于 v7 checkpoint 实测,默认由 `Ling10BConfig`(`src/edge0/models/edge0_10b/__init__.py`)固定。
+
+## 模型档案
+
+| 项目 | 值 |
+| --- | --- |
+| 参数量级 | 10B 级 |
+| 层数 | 24(第 0 层为 dense) |
+| 专家数 | 128(路由专家)+ 1 常驻共享专家 |
+| top_k(K) | 8(原生路由宽度) |
+| 路由方式 | `SIGMOID_GROUP`(sigmoid + group 限制 top-k:`n_group=8, topk_group=4, routed_scaling=2.5, norm_topk_prob=True`) |
+| 专家量化 | 4-bit affine,group 64 |
+| 权重布局 | `WeightLayout.SEPARATE`(gate/up/down 分离张量堆叠) |
+| 专家权重路径 | `model.layers.N.mlp.experts` |
+| 前置路由(prerouter) | 22 个头(显式 owners 1..22),start_layer 1,hidden 512,fp16 |
+| 解码调用方式 | `patch_call=False`(头内建在 `BailingSparseMoE` 中,从 `prerouter_cache` logits 消费) |
+| LoRA | `r=16, alpha=32.0` |
+| 预填分块 | 2048 |
+| 热窗 | 1 |
+| 流式预取历史 | 开(`prefetch_history=True`) |
+| 服务端口 | 8083 |
+| 验收吞吐 | ≈ 33 tok/s |
+| 验收峰值激活内存 | ≈ 1.4 GB |
+
+> 说明:数值全部取自 `Ling10BConfig._defaults()` 与 `LayerOptions.staged_k8()`。头部数量来自显式 `owners=range(1, 23)`,共 22 个头;`feature_topk="executed"`。
+
+## 分阶段解码(staged decode)
+
+该档使用 `LayerOptions.staged_k8()` 预设:
+
+- 固定槽分阶段解码(`staged=True`,`staged_n=8`,`staged_trigger=8`,`staged_sync=True`),逐层无 host 同步。
+- `staged_replace=False`:路由由内建的混合 prerouter 提供,分阶段集合与路由集合一致,槽表映射零丢弃。
+- 不使用热专家钉住(`hot_per_layer=0`),`prefill_hot=0`。
+- 整层 E3b prefill(`full_layer_prefill=True`)。
+
+## 使用方式
+
+### CLI 起服务
+
+```bash
+edge0 serve --name edge0-10b --model-dir /path/to/checkpoint --host 127.0.0.1 --port 8083
+```
+
+可选参数:
+
+- `--no-prerouter`:禁用前置路由(`prerouter=None`)。
+- `--no-lora`:禁用 LoRA(`lora=""`)。
+- `--flask`:改用 Flask 传输(需要安装 flask,支持 SSE 流式)。
+
+单轮对话(终端)可改用 `chat`:
+
+```bash
+edge0 chat --name edge0-10b --model-dir /path/to/checkpoint --prompt "你好"
+```
+
+查看该档默认档案:
+
+```bash
+edge0 models
+```
+
+### Python API
+
+```python
+from edge0 import AutoEngine, AutoModel, AutoConfig
+
+# 直接可生成的引擎
+engine = AutoEngine.from_pretrained(
+ "/path/to/checkpoint", name="edge0-10b",
+)
+ids = engine.generate([156895]) # 内部使用配置里的默认采样
+text = engine._tok.decode(ids)
+engine.close()
+
+# 仅加载权重(流式专家 + prerouter + LoRA 已装配)
+model = AutoModel.from_pretrained("/path/to/checkpoint", name="edge0-10b")
+
+# 仅拿配置
+cfg = AutoConfig.from_pretrained("/path/to/checkpoint", name="edge0-10b")
+```
+
+`AutoEngine` / `AutoModel` / `AutoConfig` 三者也可省略 `name`,从 checkpoint 的 `config.json` 的 `model_type` 或目录 basename 自动解析(见 `src/edge0/registry.py`)。
+
+## HTTP API
+
+`edge0 serve` 暴露一个 OpenAI 兼容的单模型端点。引擎一次独占一个请求,生成以 FIFO 队列串行执行。
+
+| 方法 | 路径 | 说明 |
+| --- | --- | --- |
+| `GET` | `/healthz` | 健康检查 |
+| `GET` | `/v1/models` | 列出已加载模型 |
+| `POST` | `/v1/chat/completions` | 对话补全(支持 `stream`) |
+| `POST` | `/v1/completions` | 不支持,返回 400 |
+
+### 非流式对话
+
+```bash
+curl -s http://127.0.0.1:8083/v1/chat/completions \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "edge0-10b",
+ "messages": [{"role": "user", "content": "介绍一下 Ling"}],
+ "temperature": 0.7,
+ "max_tokens": 256
+ }'
+```
+
+响应字段(非流式):
+
+```json
+{
+ "id": "chatcmpl-...",
+ "object": "chat.completion",
+ "created": 1750000000,
+ "model": "edge0-10b",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "..."},
+ "finish_reason": "stop"
+ }],
+ "usage": {"prompt_tokens": 12, "completion_tokens": 30, "total_tokens": 42}
+}
+```
+
+请求可选字段:`model`、`messages`(含 `role`/`content`,content 支持多段文本自动拼接)、`temperature`、`top_p`、`top_k`、`max_tokens`、`seed`、`stream`。
+
+### 流式对话(需 Flask)
+
+```bash
+curl -N http://127.0.0.1:8083/v1/chat/completions \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "edge0-10b",
+ "messages": [{"role": "user", "content": "数到五"}],
+ "stream": true
+ }'
+```
+
+每个 token 输出一段 `data: {"object":"chat.completion.chunk", ...}` SSE 事件,结束以 `data: [DONE]` 收尾。
+
+## 配置覆盖
+
+`from_pretrained` 支持对任意公开字段做覆盖(未知字段会抛 `TypeError`)。
+
+```python
+from edge0 import AutoEngine
+
+engine = AutoEngine.from_pretrained(
+ "/path/to/checkpoint",
+ name="edge0-10b",
+ port=9083, # 覆盖默认端口 8083
+ target_tok_s=35.0, # 覆盖验收吞吐目标
+ prerouter=None, # 关闭前置路由
+ lora="", # 关闭 LoRA
+ prefill_chunk=1024, # 缩小预填分块
+)
+```
+
+CLI 里对应的覆盖是 `--no-prerouter` / `--no-lora`(见 `src/edge0/cli.py` 的 `_engine_kwargs`)。引擎参数覆盖请直接走 `Ling10BConfig.from_pretrained(model_dir, **overrides)`。
diff --git a/docs/models/edge0-35b.md b/docs/models/edge0-35b.md
new file mode 100644
index 0000000..c9b8a14
--- /dev/null
+++ b/docs/models/edge0-35b.md
@@ -0,0 +1,163 @@
+# edge0-35b
+
+edge0 平台的第一档主力模型:基于 Qwen3.5-MoE(K=4 档)的 35B 级稀疏混合专家模型。通过流式专家加载(streaming experts)、训练前置路由(prerouter)与 LoRA 适配,把整份权重量化后驻留在磁盘、按需装载,在一块消费级设备上即可服务。
+
+生产档案基于 round-7 checkpoint 实测,默认由 `Qwen35Config`(`src/edge0/models/edge0_35b/__init__.py`)固定。
+
+## 模型档案
+
+| 项目 | 值 |
+| --- | --- |
+| 参数量级 | 35B 级 |
+| 层数 | 40 |
+| 专家数 | 256(路由专家)+ 1 常驻共享专家 |
+| top_k(K) | 4 |
+| 路由方式 | `SOFTMAX_TOPK`(softmax → top-k → renormalize,`norm_topk_prob=True`) |
+| 专家量化 | 4-bit affine,group 64 |
+| 权重布局 | `WeightLayout.SEPARATE`(gate/up/down 分离张量堆叠) |
+| 专家权重路径 | `language_model.model.layers.N.mlp.switch_mlp` |
+| 前置路由(prerouter) | 33 个头(owners 6..38),start_layer 7,hidden 512,fp16 |
+| 解码调用方式 | `patch_call=True`,跨 token 分阶段解码,K=4 |
+| LoRA | `r=16, alpha=32.0` |
+| 预填分块 | 2048 |
+| 热窗 | 4 |
+| 流式预取历史 | 开(`prefetch_history=True`) |
+| 服务端口 | 8085 |
+| 验收吞吐 | ≈ 13 tok/s |
+| 验收峰值激活内存 | ≈ 3.3 GB |
+
+> 说明:数值全部取自 `Qwen35Config._defaults()` 与 `LayerOptions.staged_k4()`。头部数量来自显式 `owners` 列表(6 到 38,共 33 个头);`feature_topk="executed"` 表示喂入头的 top-k 特征即解码时实际路由的集合。
+
+## 分阶段解码(staged decode)
+
+该档使用 `LayerOptions.staged_k4()` 预设:
+
+- 固定槽分阶段解码(`staged=True`,`staged_n=4`,`staged_sync=True`),逐层无 host 同步。
+- `staged_replace=False`:路由由训练好的 prerouter 头提供(MoE 块经由 prerouter logits 路由),因此分阶段集合与路由集合完全一致,槽表映射零丢弃。
+- 常驻热专家钉住(`hot_per_layer=32`),预热钉每 4 次 forward 刷新。
+- 整层 prefill(`full_layer_prefill=True`),前 12 层整层装载(E3b),prefill 热栈 32。
+
+## 使用方式
+
+### CLI 起服务
+
+```bash
+edge0 serve --name edge0-35b --model-dir /path/to/checkpoint --host 127.0.0.1 --port 8085
+```
+
+可选参数:
+
+- `--no-prerouter`:禁用前置路由(`prerouter=None`)。
+- `--no-lora`:禁用 LoRA(`lora=""`)。
+- `--flask`:改用 Flask 传输(需要安装 flask,支持 SSE 流式)。
+
+单轮对话(终端)可改用 `chat`:
+
+```bash
+edge0 chat --name edge0-35b --model-dir /path/to/checkpoint --prompt "你好"
+```
+
+查看该档默认档案:
+
+```bash
+edge0 models
+```
+
+### Python API
+
+```python
+from edge0 import AutoEngine, AutoModel, AutoConfig
+
+# 直接可生成的引擎
+engine = AutoEngine.from_pretrained(
+ "/path/to/checkpoint", name="edge0-35b",
+)
+ids = engine.generate([248044]) # 内部使用配置里的默认采样
+text = engine._tok.decode(ids)
+engine.close()
+
+# 仅加载权重(流式专家 + prerouter + LoRA 已装配)
+model = AutoModel.from_pretrained("/path/to/checkpoint", name="edge0-35b")
+
+# 仅拿配置
+cfg = AutoConfig.from_pretrained("/path/to/checkpoint", name="edge0-35b")
+```
+
+`AutoEngine` / `AutoModel` / `AutoConfig` 三者也可省略 `name`,从 checkpoint 的 `config.json` 的 `model_type` 或目录 basename 自动解析(见 `src/edge0/registry.py`)。
+
+## HTTP API
+
+`edge0 serve` 暴露一个 OpenAI 兼容的单模型端点。引擎一次独占一个请求,生成以 FIFO 队列串行执行。
+
+| 方法 | 路径 | 说明 |
+| --- | --- | --- |
+| `GET` | `/healthz` | 健康检查 |
+| `GET` | `/v1/models` | 列出已加载模型 |
+| `POST` | `/v1/chat/completions` | 对话补全(支持 `stream`) |
+| `POST` | `/v1/completions` | 不支持,返回 400 |
+
+### 非流式对话
+
+```bash
+curl -s http://127.0.0.1:8085/v1/chat/completions \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "edge0-35b",
+ "messages": [{"role": "user", "content": "介绍你自己"}],
+ "temperature": 0.6,
+ "max_tokens": 256
+ }'
+```
+
+响应字段(非流式):
+
+```json
+{
+ "id": "chatcmpl-...",
+ "object": "chat.completion",
+ "created": 1750000000,
+ "model": "edge0-35b",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "..."},
+ "finish_reason": "stop"
+ }],
+ "usage": {"prompt_tokens": 12, "completion_tokens": 30, "total_tokens": 42}
+}
+```
+
+请求可选字段:`model`、`messages`(含 `role`/`content`,content 支持多段文本自动拼接)、`temperature`、`top_p`、`top_k`、`max_tokens`、`seed`、`stream`。
+
+### 流式对话(需 Flask)
+
+```bash
+curl -N http://127.0.0.1:8085/v1/chat/completions \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "edge0-35b",
+ "messages": [{"role": "user", "content": "数到五"}],
+ "stream": true
+ }'
+```
+
+每个 token 输出一段 `data: {"object":"chat.completion.chunk", ...}` SSE 事件,结束以 `data: [DONE]` 收尾。
+
+## 配置覆盖
+
+`from_pretrained` 支持对任意公开字段做覆盖(未知字段会抛 `TypeError`)。
+
+```python
+from edge0 import AutoEngine
+
+engine = AutoEngine.from_pretrained(
+ "/path/to/checkpoint",
+ name="edge0-35b",
+ port=9090, # 覆盖默认端口 8085
+ target_tok_s=14.0, # 覆盖验收吞吐目标
+ prerouter=None, # 关闭前置路由
+ lora="", # 关闭 LoRA
+ prefill_chunk=1024, # 缩小预填分块
+)
+```
+
+CLI 里对应的覆盖是 `--no-prerouter` / `--no-lora`(见 `src/edge0/cli.py` 的 `_engine_kwargs`)。引擎参数覆盖请直接走 `Qwen35Config.from_pretrained(model_dir, **overrides)`。
diff --git a/docs/moe.md b/docs/moe.md
new file mode 100644
index 0000000..8c74172
--- /dev/null
+++ b/docs/moe.md
@@ -0,0 +1,188 @@
+# MoE 规格与路由(MoESpec / Routing)
+
+MoE 抽象位于 `src/edge0/moe/`,两个文件:
+
+- `spec.py` —— 路由种类(`RouterKind`)、量化规格(`QuantSpec`)、权重布局
+ (`WeightLayout`),以及驱动**驻留数学与流式层**的 `MoESpec`;
+- `routing.py` —— 两种路由数学,与 vendored 模型 **bit 级一致**(从它们逐字抽取)。
+
+设计核心:**每个 MoE 块由一份 `MoESpec` 描述**,流式子系统消费这份规格(布局 +
+key 模板),因此**一个通用的 `StreamingSwitchGLU` 就能服务所有模型**。
+
+## 路由种类:`RouterKind`
+
+`RouterKind` 是 `str` 枚举,edge0 支持两类路由数学家族。
+
+| 成员 | 值 | 语义 |
+| --- | --- | --- |
+| `SOFTMAX_TOPK` | `"softmax_topk"` | 精确 softmax → top-k → 重新归一化(Qwen3.5-MoE 风格,`norm_topk_prob=True`) |
+| `SIGMOID_GROUP` | `"sigmoid_group"` | sigmoid 分数 + 分组限 top-k + routed 缩放(DeepSeek-V3 / Bailing 风格) |
+
+两个家族对应两份模型:edge0-35b 用 `SOFTMAX_TOPK`,edge0-10b 用 `SIGMOID_GROUP`。
+对应数学实现见「路由函数」一节。
+
+## 量化规格:`QuantSpec`
+
+`QuantSpec` 是 `frozen` dataclass,描述**逐张量的默认**权重量化。
+
+| 字段 | 类型 | 默认值 | 语义 |
+| --- | --- | --- | --- |
+| `bits` | `int` | `4` | 量化位宽 |
+| `group_size` | `int` | `64` | 量化组大小 |
+| `mode` | `str` | `"affine"` | 量化模式(仿射) |
+
+两个生产模型都使用 `bits=4, group_size=64, mode="affine"`。流式层的量化 gather
+(`quant.gather_qmm`)消费这三个字段。
+
+## 权重布局:`WeightLayout`
+
+`WeightLayout` 枚举描述专家投影在 checkpoint 里的存储方式。
+
+| 成员 | 值 | 语义 |
+| --- | --- | --- |
+| `SEPARATE` | `"separate"` | `gate_proj` / `up_proj` / `down_proj` 三份堆叠张量 |
+| `FUSED_GATE_UP` | `"fused_gate_up"` | `gate_proj + up_proj` 融合成一份堆叠张量(out 轴行数加倍);`down_proj` 独立 |
+
+当前两个模型都用 `SEPARATE`;`FUSED_GATE_UP` 为支持融合 checkpoint 的模型预留。
+
+## 规格:`MoESpec`
+
+`MoESpec` 是 `frozen` dataclass,描述一个模型所有 MoE 块需要的一切。
+
+| 字段 | 类型 | 默认值 | 语义 |
+| --- | --- | --- | --- |
+| `num_experts` | `int` | (必填) | 每层被路由的专家数 |
+| `top_k` | `int` | (必填) | 每 token 选择的专家数(路由宽度) |
+| `intermediate_size` | `int` | (必填) | 专家 FFN 隐层尺寸(**仅文档**;数学从权重张量推导尺寸) |
+| `router` | `RouterKind` | `SOFTMAX_TOPK` | 路由数学家族 |
+| `norm_topk_prob` | `bool` | `True` | 是否对选择权重重新归一化 |
+| `routed_scaling` | `float \| None` | `None` | 施加在选择权重上的缩放系数(sigmoid-group 路由);softmax-topk 为 `None` |
+| `n_group` | `int \| None` | `None` | 分组路由的组数(仅 sigmoid-group) |
+| `topk_group` | `int \| None` | `None` | 存活的分组数(仅 sigmoid-group) |
+| `shared_experts` | `int` | `0` | 常驻共享专家数(**从不被流式**) |
+| `quant` | `QuantSpec` | `QuantSpec()` | 权重量化 |
+| `layout` | `WeightLayout` | `SEPARATE` | 权重布局 |
+| `key_template` | `str` | `""` | safetensors key 前缀,含 `{layer}`;`{proj}` / `{part}` 占位由消费方追加 |
+| `block_path` | `str` | `""` | 从加载的模型对象到 MoE 块的点分属性路径,含 `{layer}` |
+| `layer_path` | `str` | `""` | 到 **decoder 层对象**(块的宿主)的点分路径,含 `{layer}`;prerouter stager 用它读每层缓存;默认由 `block_path` 去掉末尾属性段推出 |
+| `expert_row_axis` | `int` | `0` | 堆叠 `[num_experts, ...]` 张量中,一个专家是连续切片的轴(受支持布局恒为 0,仅为文档保留) |
+
+关键差异标注:
+
+- `intermediate_size` 仅供文档——**数学不依赖它**,尺寸一律从权重张量推导。
+- `shared_experts` 的语义是「常驻、永不流式」:共享专家留在基础模型里,只有路由专家被
+ `StreamingSwitchGLU` 流式处理。
+
+### Bundle 布局与 up-first 顺序(重点)
+
+`bundle_projs` 属性返回**缓存专家 bundle 所含投影,按堆叠顺序**:
+
+```python
+if self.layout is WeightLayout.FUSED_GATE_UP:
+ return ("gate_up_proj", "down_proj")
+return ("up_proj", "gate_proj", "down_proj")
+```
+
+**unfused 顺序是 `("up_proj", "gate_proj", "down_proj")`——up 在前。**
+
+这个顺序不是随意的,它**直接对齐数学参数的顺序**。GLU 激活的数学是:
+
+```python
+def _swiglu(up, gate):
+ return nn.silu(gate) * up
+```
+
+即 `_swiglu(up, gate)`。因此权重栈把 `up` 放在 `gate` 之前,正好对应函数签名
+`(up, gate)`——消费方只需按顺序喂参数,**不需要做任何重排**。`StreamingSwitchGLU`
+的 bundle 构建与数学路径正是按 `self._bundle_projs` 的顺序展开的。
+
+**fused 布局则把 gate 行叠在 up 行之上**(out 轴方向),所以
+
+```python
+x_gate, x_up = core.split(x_gu, 2, axis=-1)
+```
+
+一次 `split(x_gu, 2)` 恰好得到 `(gate, up)`。行序是「gate 在上、up 在下」。构建 fused
+bundle 时,shard 里 gate/up 是两份独立张量,必须**按专家逐个交错**拼接(gate 切片在前、
+up 切片在后),而非整体拼接,否则会行错位(见 `streaming/layer.py` 的 `_build`)。
+
+`fuse_gu` 属性是 `layout is FUSED_GATE_UP` 的快捷判断,供流式层分支使用。
+
+### 路径解析:`keys()` / `block_of()` / `layer_of()`
+
+`MoESpec` 用三条路径字符串把「层号」翻译成真实对象 / 张量 key。
+
+**`keys(layer, proj, part)`** —— 解析某层某投影某部分的一个 safetensors key:
+
+```python
+prefix = self.key_template.format(layer=layer)
+return f"{prefix}.{proj}.{part}"
+```
+
+例(edge0-35b):`keys(3, "gate_proj", "weight")` 得到
+`language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight`。
+
+**`block_of(model, layer)`** —— 解析 MoE 块对象。沿 `block_path` 逐段 `getattr`,
+遇到纯数字段则按下标取(多数家族里 layer 是普通 list)。
+
+**`layer_of(model, layer)`** —— 解析 decoder 层对象(块的宿主)。若给了 `layer_path`
+用它的模板;否则按约定回退:块位于 `<layer>.<mlp>.<block>`,层就是去掉末尾**两段**
+属性后的路径(`split(".")[:-2]`)。prerouter stager 用 `layer_of` 读取每层缓存。
+
+测试(`tests/test_moe_spec.py`)用假模型验证了这些路径解析:数字段走下标、`layer_path`
+缺失时从 `block_path` 正确推出层路径、`bundle_projs` 随布局切换、`QuantSpec` 默认值、
+两种 `RouterKind` 的值。
+
+## 路由函数:`routing.py`
+
+两个路由函数共享**驻留**与**流式**两条 MoE 路径。模块 docstring 强调:
+
+> These functions must stay bit-identical to the vendored base
+> implementations (they are extracted from them verbatim); parity tests pin
+> this.
+
+即它们是从 vendored 基础实现**逐字抽取**的,必须与之一致,由 parity 测试钉死。
+
+### `select_from_logits(logits, top_k, norm=True)`
+
+Softmax-topk 路由(Qwen3.5-MoE / `norm_topk_prob=True` 语义):
+
+```python
+gates = core.softmax(logits, axis=-1, precise=True)
+inds = core.argpartition(gates, kth=-top_k, axis=-1)[..., -top_k:]
+scores = core.take_along_axis(gates, inds, axis=-1)
+if norm:
+ scores = scores / scores.sum(axis=-1, keepdims=True)
+return inds, scores
+```
+
+流程:**精确 softmax → top-k → (可选)重新归一化**。返回 `(inds [..., k], scores [..., k])`。
+
+### `group_select_from_logits(logits, top_k, n_group, topk_group, routed_scaling, norm=True, expert_bias=None)`
+
+Sigmoid + 分组限 top-k 路由(DeepSeek-V3 / Bailing 规则):
+
+```python
+scores = core.sigmoid(logits.astype(core.float32))
+select = scores + expert_bias if expert_bias is not None else scores
+```
+
+- **选择分数**是 `sigmoid(logits) + expert_bias`;
+- 按每组**前二**选择分数之和排序,保留 `topk_group` 个最优组,其余组分数置 `-inf`
+ 排除(`k_drop = n_group - topk_group`;`topk_group == n_group` 时无组被丢);
+- 在存活组内按选择分数取 top-k 专家;
+- **权重**是所选专家的**原始 sigmoid 分数**(不含 bias),可选归一化
+ (`w / (w.sum + 1e-20)`)后乘以 `routed_scaling`。
+
+返回 `(inds [..., k], scores [..., k])`。注意「选谁」用选择分数(含 bias),「权重」用
+raw sigmoid(不含 bias)——这是 DeepSeek 式路由的关键区分。
+
+两份模型各取其一:edge0-35b 用 `select_from_logits`,edge0-10b 用
+`group_select_from_logits`;prerouter stager 也复用这两个函数做跨 token 的教师路由。
+
+## 参考
+
+- `src/edge0/moe/spec.py` —— `MoESpec` / `QuantSpec` / `RouterKind` / `WeightLayout`
+- `src/edge0/moe/routing.py` —— 两个路由函数
+- `src/edge0/moe/__init__.py` —— 重新导出以上符号
+- `tests/test_moe_spec.py` —— 路径解析与布局契约测试
diff --git a/docs/prerouter.md b/docs/prerouter.md
new file mode 100644
index 0000000..8a39e0a
--- /dev/null
+++ b/docs/prerouter.md
@@ -0,0 +1,71 @@
+# prerouter:跨 token 路由预判
+
+## 动机
+
+MoE decode 每步都需要「本层输出 → 路由 → 下层的专家选择」,而路由依赖前一层
+输出——SSD 流式下,等路由结果出来再加载专家意味着每步都要等权重装载。解法:
+**用一层小网络提前一个 token 预测路由**,让专家装载与生成并行,staged 槽位
+永远先于需求就绪。
+
+## 双移位:prev-layer + prev-token
+
+- prerouter 头 **OWNED by 层 N**:在 token t 用层 N 的 MoE 输入(attention 后
+ norm 输出)预测**层 N+1** 的路由(层移位);
+- 层 N+1 消费的是层 N 的头在 **token t-1** 的预测(token 移位)。
+
+所以一个 decode step 的 staged 专家集就是 prerouter 的预测本身——**按构造零
+drop**(`staged_replace` 语义)。
+
+## 头结构(`prerouter/heads.py`)
+
+特征向量:`concat[hidden, 本 token top-k one-hot, 上 token top-k one-hot]`
+(`feature_topk` 决定 one-hot 来源:"executed" = 块实际路由的 top-k,qwen 训练
+语义;"teacher" = 原 gate 重算的 top-k,ling v7 训练语义)。
+
+```
+head: fc1 -> exact(erf) gelu -> fc2 + linear_init (与训练导出 bit 级一致)
+```
+
+## 安装与接线(`prerouter/install.py`)
+
+`install_prerouter(model, spec, store)`:
+
+- 从 safetensors 读每个 owner 的头权重(键形如 `layers.<N>.fc1.weight`);
+- 替换/注入模型中的 prerouter 模块;
+- **patch_call**:qwen 的 vendored MoE 块不是 prerouter-aware,安装类级
+ `__call__` 补丁让 decode 路由走 `pred_inds`;ling 的 `bailing_hybrid` 内置了
+ 消费钩子,`patch_call=False`。
+
+## 跨 token 分派(`prerouter/stager.py`)
+
+`CrossTokenStager`(qwen)/ `LingPrerouterStager`(ling)在 step 边界把预测
+**提交**给下一层:先保存本轮 logits,step 后按 `start_layer`/`owners` 把每个
+owner 的预测写入对应 consumer 的 staged 槽位。家族差异通过三个 hook 表达:
+`_features`(特征组装)、`_select`(logits → 专家选择)、`_store`(提交)。
+
+## 与 streaming 的配合
+
+- `LayerOptions.staged_replace=True`:路由集 == staged 集(prerouter 模式,
+ 无 drop);
+- 生产 profile(`staged_k4`/`staged_k8`)使用**显式索引**路由:prerouter 预测
+ 直接作为 staged 填充来源,`staged_replace=False` 但槽表精确映射,同样零 drop;
+- `pin_bonus`:prerouter 预测的专家在 hot 驻留选择时额外加分,让「即将用到」
+ 的专家优先常驻。
+
+## 模型档
+
+| | edge0-35b | edge0-10b |
+|---|---|---|
+| 路由家族 | SOFTMAX_TOPK(precise softmax → top-k → 归一化) | SIGMOID_GROUP(sigmoid + 组限制 top-k,n_group 8 / topk_group 4,routed_scaling 2.5) |
+| start_layer | 7(首个消费层) | 1 |
+| heads(owners) | 33 个(6..38) | 22 个(1..22) |
+| hidden | 512(fp16) | 512(fp16) |
+| feature_topk | executed | executed |
+| patch_call | True(qwen3_next 需补丁) | False(模型内置钩子) |
+| 权重文件 | `artifacts/prerouter_edge0_35b_k4.safetensors` | `artifacts/prerouter_edge0_10b.safetensors` |
+
+## 测试与回归
+
+`tests/test_streaming_math.py` 的 staged 路径在「无 prerouter」下用显式索引
+模拟槽位填充,保证 staged 数学与 exact 等价;prerouter 头本身的数值由
+`heads.py` 的 bit 级移植保证(源自训练实现),引擎 e2e 冒烟验证真实生成链路。
diff --git a/docs/streaming.md b/docs/streaming.md
new file mode 100644
index 0000000..7779b0a
--- /dev/null
+++ b/docs/streaming.md
@@ -0,0 +1,109 @@
+# SSD 流式专家层(streaming)
+
+## 问题
+
+Qwen3.5-MoE 有 40 层 × 256 专家,每层 MoE 权重约 310MB(4-bit 量化后)——全量
+常驻远超 Apple Silicon 统一内存预算。edge0 的方案:**权重留在 SSD,按需 mmap
+加载进 GPU,用 LRU + 预取 + 固定槽把「每步只动几十 MB」变成「每步几乎不动」**。
+
+核心部件(`edge0/streaming/`):
+
+| 模块 | 职责 |
+|---|---|
+| `mmap.py` | `SafetensorsMmap`:单文件 byte-range mmap,按张量名惰性读原始字节 |
+| `cache.py` | `SharedExpertCache`:跨层共享的 LRU,容量 `cache_slots` |
+| `layer.py` | `StreamingSwitchGLU`:每层一个实例,持有全部执行路径与状态 |
+| `options.py` | `LayerOptions`:typed 配置(部署期 env 旋钮的替代品) |
+
+## 数学契约(必须与 vendored 模型 bit 级一致)
+
+MoE 块数学为 `down(silu(gate(x)) * up(x))`,即
+`_swiglu(up, gate) = nn.silu(gate) * up`。**bundle 顺序直接对应数学参数顺序**,
+权重栈不需要任何重排:
+
+- **separate(未融合)**:bundle 顺序为
+ `("up_proj", "gate_proj", "down_proj")` —— up 在前!堆栈后的 wargs 顺序与
+ `_swiglu(up, gate)` 签名逐位对应:`(w_u, s_u, b_u, w_g, s_g, b_g, w_d, s_d, b_d)`。
+- **fused gate+up**:bundle 顺序为 `("gate_up_proj", "down_proj")`;gate_up 行内
+ 布局为 **gate 行在上、up 行在下**(沿输出特征轴拼接),`mx.split(x_gu, 2, -1)`
+ 得到 `(gate, up)`。
+
+任何一处重排(如历史上把 bundle 写成 gate-first)都会在
+`tests/test_streaming_math.py` 的参考对比中暴露(相对 L2 从 ≈0.2% 跳到 ≈100%)。
+
+### 量化布局
+
+- `switch_mlp.<proj>.weight`:打包 u32 `[E, out, in/8]`(4-bit affine,组 64);
+- `scales` / `biases`:bf16 位型,按组 `[E, out, in/64]`;
+- 内核:`backends.quant.gather_qmm`(MLX 量化 gather matmul),dequant 后 bf16
+ 内部精度(相对 L2 ≈0.24%,测试容差按此标定)。
+
+## 执行路径(由快到全)
+
+### 1. exact(on-demand bundle)
+
+路由索引去重 → `_get_bundles(unique)` 逐专家构建(从 LRU 或 mmap 读)→ 堆栈 →
+`gather_qmm`。最慢但最通用,是其他路径的正确性基准。
+
+### 2. staged(固定槽双缓冲,decode 主力)
+
+- `staged_n` 个固定槽位 + 溢出零槽;
+- 路由索引 → `mx.take` 槽位表,**索引不离开 GPU**(每层每步零 host 同步);
+- `staged_sync`:step 边界同步填充;`asm_cache`:按专家集缓存槽表与堆栈图节点,
+ 重复集不重建;`incr_stack`:把 9 个 `mx.stack` 节点换成增量栈的
+ `put_along_axis` 行写入(staged 期间从 LRU 去重,`incr_writeback` 归还);
+- 缺专家映射到溢出零槽、贡献丢弃;槽未就绪时回退 exact 路径。
+
+### 3. hot(LRU 常驻 top-N)
+
+- 每层按衰减计数 `_hot_counts` 选 top-N(`hot_per_layer`),LRU 命中的专家以
+ 常驻堆栈形式存在(`load_hot_layer` / `materialize_hot`,滑动窗口
+ `hot_window` 控制同时驻留的层数);
+- 命中走堆栈 gather,未命中走 exact 并 scatter-add。
+
+### 4. full-layer(E3b 整层装载,prefill 主力)
+
+- `load_full_layer()`:整层 9 个张量直接装入(checkpoint 本来就是逐层堆叠的
+ 单张量,mmap 位型转换 ≈9ms/层,CPU 装载隐藏在上一层的 GPU 执行下);
+- `_gather_sort` 把 batch 并入 token 维 → 排序 gather → `_scatter_unsort`
+ 反排序并恢复 batch 维;
+- `prefill_full_layers` 只对前导层整层装载(qwen 为 12 层),其余层走 hot/exact;
+- 用后 `clear_full_layer()` 释放 GPU 副本,页缓存承担热数据。
+
+## 排序与编译
+
+`use_compile` 时,staged/exact 路径包进 `mx.compile`:
+
+- 集合规模 ≥64 时用排序变体 `_moe_math_sorted`(`_gather_sort` 预处理 +
+ 排序 gather + `_scatter_unsort` 还原);
+- 否则用直接 gather 变体 `_moe_math`;
+- 两者数学等价,测试对每条路径都做了 exact 对比。
+
+## 关键选项(`LayerOptions`)
+
+| 字段 | 含义 | 默认 |
+|---|---|---|
+| `staged` / `staged_n` / `staged_trigger` | 固定槽 decode 开关/槽数/触发 top-k | False / 8 / 8 |
+| `staged_replace` | staged 集**取代**路由(配合 prerouter,零 drop) | False |
+| `staged_sync` / `asm_cache` / `incr_stack` / `incr_writeback` | 填充同步 / 图节点缓存 / 增量栈 / 归还 LRU | True / True / False / False |
+| `hot_per_layer` / `hot_update_interval` / `hot_decay` / `pin_bonus` | 热专家驻留数与刷新 | 0 / 4 / 0.75 / 2.0 |
+| `cache_slots` / `prefetch_cap` | LRU 容量 / 预取缓冲 | 64 / 48 |
+| `load_threads` / `prefetch_threads` | 构建 / 预取线程数 | 8 / 4 |
+| `full_layer_prefill` / `prefill_full_layers` | 整层装载 prefill / 前导层数 | False / 0 |
+| `prefill_hot` | prefill 期 hot 栈规模 | 0 |
+| `use_compile` / `top_k` | 编译包装 / 路由 top-k 覆盖 | True / None |
+
+预设:`staged_k4()`(edge0-35b:staged 4 槽 + hot 32 + 整层 prefill 12 层)、
+`staged_k8()`(edge0-10b:staged 8 槽,无 hot 驻留)。两档共用
+`cache_slots=64`。
+
+## 为什么「整层」加载也快
+
+checkpoint 的 MoE 权重本来就是逐层堆叠的单张量(`[256, 512, 256]` 这类),
+整层装载是 9 次直接读 + 位型 view,不是 256×9 次逐专家构建。CPU 装载与 GPU
+执行通过 `before_layer_cb` + `async_eval_per_layer` 流水线重叠。
+
+> 注意 fused gate_up 的整层装载必须**逐专家交错** gate/up 行(与 `_build`
+> 的逐专家拼接一致);把两个 `[E, ...]` 张量整体拼接再 reshape 会产生行错位
+> (早期版本的真实 bug,已由 `test_full_layer_prefill_matches_exact[fused]`
+> 钉死)。
diff --git a/examples/demo.py b/examples/demo.py
new file mode 100644
index 0000000..5fc3b0a
--- /dev/null
+++ b/examples/demo.py
@@ -0,0 +1,50 @@
+"""Minimal edge0 API walkthrough — the same path ``edge0 demo`` runs.
+
+Install:
+ python3.12 -m venv .venv && .venv/bin/pip install -e '.[dev]'
+
+Run (checkpoint path or registered tier name):
+ .venv/bin/python examples/demo.py --model-dir /path/to/qwen35/model
+ .venv/bin/python examples/demo.py --model edge0-10b
+
+Three lines do everything: build the engine (prerouter + LoRA + SSD
+offload installed automatically from the tier config), template the
+prompt, generate.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from edge0 import AutoEngine
+from edge0.server.chat import ChatMessage, ChatRequest, ChatSession
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--model", default=None,
+ help="registered tier name, e.g. edge0-35b")
+ ap.add_argument("--model-dir", default=None,
+ help="checkpoint directory (tier auto-detected)")
+ ap.add_argument("--prompt",
+ default="Hello! Write one short sentence about Zhuhai.")
+ ap.add_argument("--max-new", type=int, default=24)
+ args = ap.parse_args()
+
+ engine = AutoEngine.from_pretrained(args.model_dir, name=args.model)
+ req = ChatRequest(
+ model=engine.name,
+ messages=[ChatMessage(role="user", content=args.prompt)],
+ max_tokens=args.max_new,
+ )
+ tokens, meta = ChatSession(engine, req).run()
+ print(f"user : {args.prompt}")
+ print(f"edge0: {engine._tok.decode(tokens)}")
+ print(f"# {len(tokens)} tokens in {meta['wall_s']}s", file=sys.stderr)
+ engine.close()
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..3fe8d0b
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,53 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "edge0"
+version = "0.1.0"
+description = "edge0: streaming MoE inference for Apple Silicon — SSD expert offload, parallel LoRA, prerouter routing prediction. Backend-isolated (MLX now, CUDA-ready)."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "Apache-2.0" }
+authors = [{ name = "edge0 contributors" }]
+keywords = ["mlx", "moe", "inference", "lora", "ssd-offload", "streaming"]
+classifiers = [
+ "Development Status :: 3 - Alpha",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: Apache Software License",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+ # NOTE: mlx-lm is pinned to 0.31.0 on purpose. Newer releases crash in
+ # decode with a `tolist()` on lazily-evaluated arrays. Do not upgrade
+ # without re-running the full test suite.
+ "mlx==0.30.4",
+ "mlx-metal==0.30.4; platform_system == 'Darwin'",
+ "mlx-lm==0.31.0",
+ "numpy>=1.24",
+ "safetensors>=0.4",
+ "tokenizers>=0.15",
+ "psutil>=5.9",
+]
+
+[project.optional-dependencies]
+dev = [
+ "pytest>=8.0",
+]
+
+[project.scripts]
+edge0 = "edge0.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+ "slow: real-weight e2e tests (skip with -m 'not slow')",
+]
+addopts = "-m 'not slow'"
diff --git a/scripts/alignment_results.json b/scripts/alignment_results.json
new file mode 100644
index 0000000..19c58b0
--- /dev/null
+++ b/scripts/alignment_results.json
@@ -0,0 +1,182 @@
+{
+ "demo_qwen": {
+ "prompt_ids": [
+ 248045,
+ 846,
+ 198,
+ 9419,
+ 0,
+ 9357,
+ 799,
+ 2716,
+ 11316,
+ 883,
+ 32246,
+ 12180,
+ 2075,
+ 13,
+ 248046,
+ 198,
+ 248045,
+ 74455,
+ 198,
+ 248068,
+ 198
+ ],
+ "out": [
+ 90700,
+ 8340,
+ 25,
+ 271,
+ 16,
+ 13,
+ 220,
+ 2972,
+ 2014,
+ 53983,
+ 279,
+ 5952,
+ 64700,
+ 198
+ ],
+ "text": "Thinking Process:\n\n1. **Analyze the Request:**\n"
+ },
+ "edge0_qwen": {
+ "prompt_ids": [
+ 248045,
+ 846,
+ 198,
+ 9419,
+ 0,
+ 9357,
+ 799,
+ 2716,
+ 11316,
+ 883,
+ 32246,
+ 12180,
+ 2075,
+ 13,
+ 248046,
+ 198,
+ 248045,
+ 74455,
+ 198,
+ 248068,
+ 198
+ ],
+ "out": [
+ 90700,
+ 8340,
+ 25,
+ 271,
+ 16,
+ 13,
+ 220,
+ 2972,
+ 2014,
+ 53983,
+ 279,
+ 5952,
+ 64700,
+ 198
+ ],
+ "text": "Thinking Process:\n\n1. **Analyze the Request:**\n"
+ },
+ "demo_ling": {
+ "prompt_ids": [
+ 157151,
+ 90827,
+ 157152,
+ 14136,
+ 5381,
+ 6350,
+ 928,
+ 156895,
+ 157151,
+ 39,
+ 116171,
+ 157152,
+ 34355,
+ 7011,
+ 26227,
+ 5286,
+ 40904,
+ 311,
+ 156895,
+ 157151,
+ 8469,
+ 7342,
+ 5468,
+ 157152,
+ 198,
+ 156903,
+ 156904
+ ],
+ "out": [
+ 40904,
+ 364,
+ 6939,
+ 1782,
+ 24330,
+ 77010,
+ 1288,
+ 1462,
+ 9376,
+ 290,
+ 39738,
+ 4334,
+ 2589,
+ 22703
+ ],
+ "text": "珠海是位于中国广东省珠江口西岸的沿海城市,以美丽的"
+ },
+ "edge0_ling": {
+ "prompt_ids": [
+ 157151,
+ 90827,
+ 157152,
+ 14136,
+ 5381,
+ 6350,
+ 928,
+ 156895,
+ 157151,
+ 39,
+ 116171,
+ 157152,
+ 34355,
+ 7011,
+ 26227,
+ 5286,
+ 40904,
+ 311,
+ 156895,
+ 157151,
+ 8469,
+ 7342,
+ 5468,
+ 157152,
+ 198,
+ 156903,
+ 156904
+ ],
+ "out": [
+ 40904,
+ 26312,
+ 39765,
+ 290,
+ 18565,
+ 4334,
+ 2589,
+ 424,
+ 22573,
+ 84624,
+ 447,
+ 45661,
+ 23183,
+ 3878
+ ],
+ "text": "珠海是中国改革开放的窗口城市,以“浪漫之都”闻名,拥有博"
+ }
+}
\ No newline at end of file
diff --git a/scripts/convert_adapters_legacy.py b/scripts/convert_adapters_legacy.py
new file mode 100644
index 0000000..e8a01de
--- /dev/null
+++ b/scripts/convert_adapters_legacy.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""One-shot legacy adapter converter: npz -> safetensors.
+
+The training pipelines exported adapters as ``.npz`` (qwen round-7 and
+ling v7). edge0 consumes safetensors ONLY — this script converts each
+legacy pair once into ``artifacts/`` (gitignored), writing the
+conversion provenance (model, kind, K, r/alpha, source md5, format
+version) into the safetensors ``__metadata__``. The npz sources are
+NOT deleted (see the conversion note in README); they are only needed
+to regenerate the artifacts.
+
+Usage:
+ python scripts/convert_adapters_legacy.py [--force]
+
+Idempotent: existing artifacts are skipped unless ``--force``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import numpy as np
+
+ROOT = Path(__file__).resolve().parents[1]
+ARTIFACTS = ROOT / "artifacts"
+
+# (npz path, tier, kind) — every legacy source this script understands.
+SOURCES = [
+ ("/Users/linyu/Documents/qwen35-v7-deploy/pregate_qwen35_v7_round7.npz",
+ "edge0-35b", "prerouter"),
+ ("/Users/linyu/Documents/qwen35-v7-deploy/lora_qwen35_v7_round7.npz",
+ "edge0-35b", "lora"),
+ ("/Users/linyu/Documents/ling-mlx-server/v7-deploy/pregate_v7_fp16.npz",
+ "edge0-10b", "prerouter"),
+ ("/Users/linyu/Documents/ling-mlx-server/v7-deploy/lora_v7_fp16.npz",
+ "edge0-10b", "lora"),
+]
+
+
+def md5(path: Path) -> str:
+ h = hashlib.md5()
+ with open(path, "rb") as f:
+ for chunk in iter(lambda: f.read(1 << 20), b""):
+ h.update(chunk)
+ return h.hexdigest()
+
+
+def rewrite_key(key: str, tier: str, kind: str) -> str:
+ """Map a legacy npz key to the edge0 artifact key.
+
+ prerouter heads are normalized to ``layers.<N>.<part>.weight``;
+ qwen lora gains the ``language_model.`` prefix (the checkpoint
+ namespace the adapter is applied into).
+ """
+ if kind == "prerouter":
+ # qwen round-7 npz already uses layers.N.*; ling v7 uses pregate.N.*
+ parts = key.split(".")
+ if parts[0] == "pregate":
+ return f"layers.{parts[1]}.{parts[2]}.{parts[3]}"
+ return key
+ if kind == "lora" and tier == "edge0-35b" and key.startswith("model.layers."):
+ return "language_model." + key
+ return key
+
+
+def convert(source: Path, tier: str, kind: str, force: bool = False) -> Path:
+ name = {
+ ("edge0-35b", "prerouter"): "prerouter_edge0_35b_k4.safetensors",
+ ("edge0-35b", "lora"): "lora_edge0_35b_k4.safetensors",
+ ("edge0-10b", "prerouter"): "prerouter_edge0_10b.safetensors",
+ ("edge0-10b", "lora"): "lora_edge0_10b.safetensors",
+ }[(tier, kind)]
+ out = ARTIFACTS / name
+ if out.exists() and not force:
+ print(f"skip {out.name} (exists)")
+ return out
+
+ data = np.load(source)
+ tensors = {}
+ for key in data.files:
+ tensors[rewrite_key(key, tier, kind)] = data[key]
+
+ # provenance: rank from the A matrix, alpha from the training run
+ # (32 for both shipped pairs), K from the tier profile.
+ meta = {
+ "model": tier,
+ "kind": kind,
+ "K": "4" if tier == "edge0-35b" else "8",
+ "r": "16", "alpha": "32",
+ "source": str(source),
+ "source_md5": md5(source),
+ "converted": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+ "format_version": "1",
+ }
+ if kind == "prerouter":
+ owners = sorted({int(k.split(".")[1])
+ for k in tensors if k.startswith("layers.")})
+ meta["owners"] = json.dumps(owners)
+
+ from safetensors.numpy import save_file as save_np
+ save_np(tensors, str(out), metadata={"__metadata__": json.dumps(meta)})
+ print(f"write {out.name} ({len(tensors)} tensors, "
+ f"{out.stat().st_size >> 20} MiB)")
+ return out
+
+
+def main(argv: list[str] | None = None) -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--force", action="store_true",
+ help="reconvert even if the artifact exists")
+ args = ap.parse_args(argv)
+
+ ARTIFACTS.mkdir(parents=True, exist_ok=True)
+ missing = [s[0] for s in SOURCES if not Path(s[0]).is_file()]
+ if missing:
+ print("missing legacy sources (skip their conversion):")
+ for m in missing:
+ print(" -", m)
+ for path, tier, kind in SOURCES:
+ if not Path(path).is_file():
+ continue
+ try:
+ convert(Path(path), tier, kind, force=args.force)
+ except Exception as exc: # noqa: BLE001 — report and continue
+ print(f"FAIL {Path(path).name}: {exc}", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/e2e_smoke.py b/scripts/e2e_smoke.py
new file mode 100644
index 0000000..216abe2
--- /dev/null
+++ b/scripts/e2e_smoke.py
@@ -0,0 +1,83 @@
+"""End-to-end smoke: real checkpoints, real adapters.
+
+Runs both shipped tiers through the full prerouter + lora + staged-decode
+path and cross-checks staged vs exact math by comparing GREEDY token
+sequences (deterministic, so any staged/exact divergence shows up as a
+different sequence).
+
+Usage:
+ .venv/bin/python scripts/e2e_smoke.py [--qwen-dir DIR] [--ling-dir DIR]
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+
+from edge0 import AutoEngine
+from edge0.backends import io
+from edge0.config import GenerationConfig
+from edge0.streaming.options import LayerOptions
+
+QWEN_DEFAULT = "/Users/linyu/Documents/qwen35-v7-deploy/model"
+LING_DEFAULT = "/Users/linyu/Documents/ling-mlx-server/v7-deploy/model"
+
+
+def _prompt_ids(model_dir: str, text: str) -> list[int]:
+ tok = io.load_tokenizer(model_dir)
+ return list(tok(text)["input_ids"])
+
+
+def run(name: str, model_dir: str, prompt_ids: list[int], max_new: int,
+ options: LayerOptions | None = None, label: str = ""):
+ t0 = time.time()
+ kwargs = {"name": name, "model_dir": model_dir}
+ if options is not None:
+ kwargs["options"] = options # None would override the tier default
+ eng = AutoEngine.from_pretrained(**kwargs)
+ t_load = time.time() - t0
+ t0 = time.time()
+ out = eng.generate(
+ prompt_ids,
+ GenerationConfig(temperature=0.0, top_k=None, top_p=None,
+ max_new_tokens=max_new),
+ max_new_tokens=max_new)
+ t_gen = time.time() - t0
+ eng.close()
+ print(f"[{name}{label}] load={t_load:.1f}s gen={t_gen:.1f}s "
+ f"({max_new / t_gen:.1f} tok/s) tokens={out}")
+ return out
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--qwen-dir", default=QWEN_DEFAULT)
+ ap.add_argument("--ling-dir", default=LING_DEFAULT)
+ ap.add_argument("--max-new", type=int, default=16)
+ args = ap.parse_args()
+
+ qwen_prompt = _prompt_ids(args.qwen_dir, "Hello!")
+ ling_prompt = _prompt_ids(args.ling_dir, "你好")
+
+ staged = run("edge0-35b", args.qwen_dir, qwen_prompt, args.max_new,
+ label="[staged]")
+ exact = run("edge0-35b", args.qwen_dir, qwen_prompt, args.max_new,
+ options=LayerOptions(staged=False, staged_replace=False,
+ full_layer_prefill=False,
+ hot_per_layer=0, use_compile=False,
+ history_prefetch=False),
+ label="[exact]")
+ if staged != exact:
+ print("MISMATCH: staged vs exact greedy sequences differ "
+ f"(staged={staged} exact={exact})")
+ return 1
+ print("qwen staged == exact greedy: OK")
+
+ run("edge0-10b", args.ling_dir, ling_prompt, args.max_new, label="[staged]")
+ print("edge0 e2e smoke OK")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/verify_alignment.py b/scripts/verify_alignment.py
new file mode 100644
index 0000000..27f5355
--- /dev/null
+++ b/scripts/verify_alignment.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+"""Adapter alignment check: correct demos (round8 / r3traces) vs edge0 artifacts.
+
+Runs the SAME prompt through the correct demo engine (production env from
+start_server.sh, with the NEW adapters: qwen round8, ling r3traces) and
+through edge0 (artifacts/*.safetensors), both greedy, and writes both token
+sequences + decoded text to one JSON result file. Identical out-ids =>
+adapters mount identically.
+
+Usage:
+ python verify_alignment.py demo qwen
+ python verify_alignment.py edge0 qwen
+ python verify_alignment.py demo ling
+ python verify_alignment.py edge0 ling
+
+Result file: scripts/alignment_results.json (each run merges its entry).
+"""
+import argparse
+import json
+import os
+import sys
+
+N = 14 # max new tokens (greedy)
+PROMPTS = {
+ "qwen": "Hello! Write one short sentence about Zhuhai.",
+ "ling": "你好,用一句话介绍珠海。",
+}
+QMODEL = "/Users/linyu/Documents/qwen35-v7-deploy/model"
+LMODEL = "/Users/linyu/Documents/ling-mlx-server/v7-deploy/model"
+QDIR = "/Users/linyu/Documents/qwen35-v7-deploy"
+LDIR = "/Users/linyu/Documents/ling-mlx-server/v7-deploy"
+RESULT = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ "alignment_results.json")
+
+
+def save(tag, payload):
+ data = {}
+ if os.path.exists(RESULT):
+ with open(RESULT) as f:
+ data = json.load(f)
+ data[tag] = payload
+ with open(RESULT, "w") as f:
+ json.dump(data, f, ensure_ascii=False, indent=2)
+ print("saved", tag)
+
+
+def qwen_demo():
+ os.environ.update(
+ MODEL_DIR=QMODEL,
+ LORA_ADAPTERS=QDIR + "/lora_qwen35_v7_round8.npz",
+ LORA_R="16", LORA_ALPHA="32",
+ PREGATE_NPZ=QDIR + "/pregate_qwen35_v7_round8.npz",
+ QWEN_PG_START="7", QWEN_PG_HIDDEN="512",
+ QWEN_STAGED="1", QWEN_STAGED_TOP_K="4", QWEN_STAGED_SLOTS="4",
+ QWEN_STAGED_REPLACE="0", QWEN_PREGATE_V7="0",
+ QWEN_HISTORY_PREFETCH="0", QWEN_INTRA_STAGE="0", QWEN_ASM_CACHE="0",
+ )
+ sys.path.insert(0, QDIR)
+ import serve # noqa: E402 (ENGINE built in main(); build it here)
+ from mlx_lm.utils import load_tokenizer # noqa: E402
+ tok = load_tokenizer(QMODEL)
+ serve.ENGINE = serve.QwenStreamingEngine(
+ QMODEL, cache_slots=900, hot_per_layer=0)
+ serve.ENGINE.load_tokenizer(tok)
+ eng = serve.ENGINE
+ tok = eng.tokenizer()
+ messages = [{"role": "user", "content": PROMPTS["qwen"]}]
+ ids = serve.encode_chat(tok, messages)
+ eng.prefill(ids)
+ logits = eng.next_logits()
+ cur = int(serve.mx.argmax(logits).item())
+ out = [cur]
+ for _ in range(N - 1):
+ logits = eng.step(out[-1])
+ serve.mx.eval(logits)
+ cur = int(serve.sample(logits, temperature=0.0, top_k=1, top_p=1.0,
+ repetition_penalty=1.0, history=ids + out))
+ out.append(cur)
+ save("demo_qwen", {"prompt_ids": ids, "out": out, "text": tok.decode(out)})
+
+
+def edge0_qwen():
+ from edge0 import AutoEngine # noqa: E402
+ from edge0.config import GenerationConfig # noqa: E402
+ eng = AutoEngine.from_pretrained(QMODEL)
+ tok = eng._tok
+ messages = [{"role": "user", "content": PROMPTS["qwen"]}]
+ text = tok.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=True)
+ ids = list(tok(text)["input_ids"])
+ out = eng.generate(ids, GenerationConfig(temperature=0.0, top_k=1,
+ top_p=1.0,
+ repetition_penalty=1.0),
+ max_new_tokens=N)
+ save("edge0_qwen", {"prompt_ids": ids, "out": out, "text": tok.decode(out)})
+ eng.close()
+
+
+def ling_demo():
+ os.environ.update(
+ CHECKPOINT_DIR=LMODEL,
+ PREGATE_HYBRID="1", PREGATE_INTRA="0", PREGATE_START_LAYER="1",
+ PREGATE_FEATURE_TOPK="executed", PREGATE_DTYPE="fp16",
+ PREGATE_NPZ=LDIR + "/pregate_r3traces.npz",
+ LORA_ADAPTERS=LDIR + "/lora_r3traces.npz",
+ THINK_MODE="0", REPETITION_PENALTY="1.0",
+ STAGED_DECODE="0", STAGED_SYNC="0",
+ STAGED_SLOTS="8", STAGED_LAYERS="1-23", EXPERT_CACHE_SLOTS="64",
+ PREFETCH_ENABLE="0", PREFILL_FULL_LAYER="1", PREFILL_PIN="1",
+ LING_PREWARM="1", MLX_COMPILE_MOE="1", LING_PREFILL_CHUNK="2048",
+ )
+ sys.path.insert(0, LDIR)
+ import server # noqa: E402 (engine built in main(); build it here)
+ server.engine = server.LingStreamingEngine(
+ LMODEL, think=False, prefill_chunk=512)
+ eng = server.engine
+ msgs = [{"role": "user", "content": PROMPTS["ling"]}]
+ ids = eng.encode_chat(msgs, think=False)
+ eng.prefill_prompt(ids, chunk_size=512)
+ out = []
+ cur = int(server.mx.argmax(eng.next_logits(), axis=-1).item())
+ out.append(cur)
+ for _ in range(N - 1):
+ logits = eng.step(out[-1])
+ server.mx.eval(logits)
+ cur = int(server.mx.argmax(logits, axis=-1).item())
+ out.append(cur)
+ if cur == 156895:
+ break
+ save("demo_ling", {"prompt_ids": ids, "out": out,
+ "text": eng.decode(out)})
+
+
+def edge0_ling():
+ from edge0 import AutoEngine # noqa: E402
+ from edge0.config import GenerationConfig # noqa: E402
+ eng = AutoEngine.from_pretrained(LMODEL)
+ tok = eng._tok
+ messages = [{"role": "user", "content": PROMPTS["ling"]}]
+ text = tok.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=True,
+ enable_thinking=False)
+ ids = list(tok(text)["input_ids"])
+ eng.prefill(ids, chunk_size=512)
+ out = []
+ cur = int(eng.next_logits().argmax().item())
+ out.append(cur)
+ for _ in range(N - 1):
+ logits = eng.step(out[-1])
+ cur = int(logits.argmax().item())
+ out.append(cur)
+ if cur == 156895:
+ break
+ save("edge0_ling", {"prompt_ids": ids, "out": out,
+ "text": tok.decode(out)})
+ eng.close()
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("side", choices=["demo", "edge0"])
+ ap.add_argument("model", choices=["qwen", "ling"])
+ args = ap.parse_args()
+ fn = {"demo": {"qwen": qwen_demo, "ling": ling_demo},
+ "edge0": {"qwen": edge0_qwen, "ling": edge0_ling}}[args.side][args.model]
+ fn()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/zhuhai_example.py b/scripts/zhuhai_example.py
new file mode 100644
index 0000000..77f0ebd
--- /dev/null
+++ b/scripts/zhuhai_example.py
@@ -0,0 +1,90 @@
+"""Complete end-to-end example: generate an English introduction to
+Zhuhai with edge0-35b (prerouter + LoRA + SSD offload, default tier
+profile — staged decode on, K=4).
+
+Usage:
+ .venv/bin/python scripts/zhuhai_example.py [--max-new N] [--temp T]
+
+The example walks through the full pipeline a user of the framework
+would hit:
+
+1. load the tokenizer and the engine (real checkpoint + adapters),
+2. tokenize a prompt,
+3. prefill (whole-layer load-drop, hot pins),
+4. greedy preview of the first token,
+5. sampled generation with the tier's default GenerationConfig,
+6. decode back to text.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+
+from edge0 import AutoEngine
+from edge0.backends import io
+from edge0.config import GenerationConfig
+
+
+PROMPT = (
+ "Write a short introduction to Zhuhai, a city in southern China, "
+ "in English. Keep it to 3-4 sentences."
+)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--max-new", type=int, default=64)
+ ap.add_argument("--temp", type=float, default=0.7)
+ ap.add_argument("--thinking", action="store_true",
+ help="enable the <think> block (enable_thinking=True)")
+ args = ap.parse_args()
+
+ model_dir = "/Users/linyu/Documents/qwen35-v7-deploy/model"
+ print(f"[1] loading tokenizer + edge0-35b from {model_dir}", flush=True)
+ tok = io.load_tokenizer(model_dir)
+ t0 = time.time()
+ eng = AutoEngine.from_pretrained(name="edge0-35b", model_dir=model_dir)
+ print(f"[2] engine built in {time.time() - t0:.1f}s", flush=True)
+
+ ids = list(tok(PROMPT)["input_ids"])
+ print(f"[3] prompt: {len(ids)} tokens: {ids[:12]} ...", flush=True)
+
+ # Deployment parity (bench_v7.py / serve.py): build the prompt with
+ # the chat template. enable_thinking=False closes an empty <think>
+ # block so the model answers directly; --thinking leaves the block
+ # open so the model emits reasoning first, then </think> + the answer.
+ chat = tok.apply_chat_template(
+ [{"role": "user", "content": PROMPT}],
+ add_generation_prompt=True, tokenize=True,
+ enable_thinking=args.thinking)
+ chat = list(chat.input_ids)
+ print(f"[3b] chat template: {len(chat)} tokens", flush=True)
+
+ # Greedy preview of the first token (deterministic).
+ t0 = time.time()
+ eng.prefill(chat)
+ l0 = eng.next_logits()
+ first = int(l0.argmax().item())
+ print(f"[4] prefill in {time.time() - t0:.1f}s; first greedy token "
+ f"{first} = {tok.decode([first])!r}", flush=True)
+ eng.reset()
+
+ # Full generation with the tier default sampler (temp 0.7, top_p 0.95).
+ gc = GenerationConfig(temperature=args.temp, max_new_tokens=args.max_new)
+ t0 = time.time()
+ out = eng.generate(chat, gc, max_new_tokens=args.max_new)
+ dt = time.time() - t0
+ text = tok.decode(out)
+ print(f"[5] generated {len(out)} tokens in {dt:.1f}s "
+ f"({len(out) / dt:.1f} tok/s)", flush=True)
+ print("=" * 60, flush=True)
+ print(text, flush=True)
+ print("=" * 60, flush=True)
+ eng.close()
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/src/edge0/__init__.py b/src/edge0/__init__.py
new file mode 100644
index 0000000..432e6ca
--- /dev/null
+++ b/src/edge0/__init__.py
@@ -0,0 +1,35 @@
+"""edge0: streaming MoE inference for Apple Silicon.
+
+SSD expert offload + parallel LoRA + prerouter routing prediction, with a
+transformers-style AutoModel registry and a backend-isolated design (all
+MLX code lives under ``edge0.backends.mlx``; a CUDA backend can be added
+as a sibling package implementing the same interface).
+
+Public API::
+
+ from edge0 import AutoConfig, AutoModel, AutoEngine
+
+ cfg = AutoConfig.from_pretrained(model_dir)
+ model = AutoModel.from_pretrained(model_dir, name="edge0-35b", ...)
+ engine = AutoEngine.from_pretrained(model_dir, ...)
+"""
+
+from edge0 import registry # noqa: F401 (populates the model registry)
+
+__version__ = "0.1.0"
+
+__all__ = [
+ "__version__",
+ "AutoConfig",
+ "AutoModel",
+ "AutoEngine",
+]
+
+
+def __getattr__(name: str):
+ # Deferred imports so `import edge0` stays light and backend selection
+ # (EDGE0_BACKEND) happens on first use.
+ if name in ("AutoConfig", "AutoModel", "AutoEngine"):
+ from edge0 import registry as _reg
+ return getattr(_reg, name)
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/src/edge0/__main__.py b/src/edge0/__main__.py
new file mode 100644
index 0000000..24576b1
--- /dev/null
+++ b/src/edge0/__main__.py
@@ -0,0 +1,7 @@
+"""``python -m edge0`` entry point (same surface as the ``edge0``
+console script)."""
+
+from edge0.cli import main
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/src/edge0/adapters/__init__.py b/src/edge0/adapters/__init__.py
new file mode 100644
index 0000000..0cd99f4
--- /dev/null
+++ b/src/edge0/adapters/__init__.py
@@ -0,0 +1,5 @@
+"""Adapter installation (parallel LoRA + prerouter heads)."""
+
+from edge0.adapters.lora import LoraLinear, install_lora
+
+__all__ = ["LoraLinear", "install_lora"]
diff --git a/src/edge0/adapters/lora.py b/src/edge0/adapters/lora.py
new file mode 100644
index 0000000..93b5beb
--- /dev/null
+++ b/src/edge0/adapters/lora.py
@@ -0,0 +1,125 @@
+"""Parallel (unmerged) LoRA adapters, safetensors-native.
+
+Port of the deployment's ``lora_apply.py`` scheme (both qwen and ling
+tiers): each adapter target is wrapped as a *parallel delta path*::
+
+ y = base(x) + scale * ((x @ A.T) @ B.T) # scale = alpha / r
+
+The base weights (including 4bit quantized linears) stay byte-identical —
+no dequant / merge / requant — and the LoRA lives only in tiny fp16 A/B
+tensors. Mathematically identical to a merged weight.
+
+Safetensors key naming (framework contract)::
+
+ <target>.lora_A [r, in]
+ <target>.lora_B [out, r]
+
+where ``<target>`` is the module path inside the loaded model (e.g.
+``model.layers.3.attention.q_proj`` or
+``language_model.model.layers.0.linear_attn.in_proj_qkv``). The legacy
+npz files used the same A/B naming, so conversion is a pure format
+migration (see ``scripts/convert_adapters_legacy.py``).
+"""
+
+from __future__ import annotations
+
+from edge0.backends import core
+from edge0.backends import nn
+
+from edge0.backends.mlx.io import load_safetensors
+
+
+class LoraLinear(nn.Module):
+ """Wraps a Linear/QuantizedLinear and adds the low-rank delta."""
+
+ def __init__(self, base: nn.Module, lora_a: core.array, lora_b: core.array,
+ scale: float):
+ super().__init__()
+ self.base = base
+ self.lora_A = lora_a # [r, in]
+ self.lora_B = lora_b # [out, r]
+ self.lora_scale = scale
+
+ def __call__(self, x: core.array) -> core.array:
+ y = self.base(x)
+ ad = self.lora_A.dtype
+ xd = x if x.dtype == ad else x.astype(ad)
+ # delta = (B @ A) applied on x: ((x @ A.T) @ B.T), shape [..., out]
+ d = (xd @ self.lora_A.T) @ self.lora_B.T
+ return y + (self.lora_scale * d).astype(y.dtype)
+
+
+def _resolve(model: nn.Module, key: str):
+ """Resolve ``model.layers.N.<...>`` or a bare dotted path to the
+ module inside the loaded model."""
+ parts = key.split(".")
+ if len(parts) >= 3 and parts[0] == "model" and parts[1] == "layers":
+ node = model.model.layers[int(parts[2])]
+ rest = parts[3:]
+ elif len(parts) >= 3 and parts[0] == "language_model" and parts[1] == "model":
+ node = model.language_model.model.layers[int(parts[3])]
+ rest = parts[4:]
+ else:
+ node = model
+ rest = parts
+ for p in rest:
+ node = getattr(node, p)
+ return node
+
+
+def install_lora(model: nn.Module, adapters_path: str,
+ r: int = 16, alpha: float = 32.0,
+ dtype=core.float16, strict: bool = True) -> dict:
+ """Attach every LoRA pair in ``adapters_path`` (safetensors) as a
+ parallel delta path.
+
+ Returns ``{applied: [...], not_found: [...], skipped: [...],
+ scale: float, dtype: str}``.
+ """
+ lora = load_safetensors(adapters_path)
+ if not lora:
+ raise ValueError("adapters has no lora tensors")
+
+ # group by target key
+ targets: dict[str, dict] = {}
+ for k, v in lora.items():
+ if k.endswith(".lora_A"):
+ targets.setdefault(k[: -len(".lora_A")], {})["A"] = v
+ elif k.endswith(".lora_B"):
+ targets.setdefault(k[: -len(".lora_B")], {})["B"] = v
+ else:
+ raise ValueError(f"unexpected lora key: {k}")
+
+ scale = alpha / r
+ applied, not_found, skipped = [], [], []
+
+ for key, d in sorted(targets.items()):
+ if "A" not in d or "B" not in d:
+ skipped.append((key, "missing A or B"))
+ continue
+ mod = _resolve(model, key)
+ if mod is None:
+ not_found.append(key)
+ if strict:
+ raise KeyError(f"lora target not found in MLX model: {key}")
+ continue
+ if not hasattr(mod, "weight") and not hasattr(mod, "base"):
+ # not a linear layer we can wrap (e.g. RMSNorm / gate)
+ skipped.append((key, type(mod).__name__))
+ if strict:
+ raise TypeError(
+ f"lora target {key} is {type(mod).__name__}, not Linear")
+ continue
+ a = d["A"].astype(dtype)
+ b = d["B"].astype(dtype)
+ # re-bind the module attribute (e.g. attention.q_proj = LoraLinear)
+ parts = key.split(".")
+ owner = _resolve(model, ".".join(parts[:-1]))
+ attr = parts[-1]
+ setattr(owner, attr, LoraLinear(mod, a, b, scale))
+ applied.append(key)
+
+ print(f"[lora] applied={len(applied)} not_found={len(not_found)} "
+ f"skipped={len(skipped)} scale={scale}", flush=True)
+ return {"applied": applied, "not_found": not_found, "skipped": skipped,
+ "scale": scale, "dtype": str(dtype)}
diff --git a/src/edge0/attention/spec.py b/src/edge0/attention/spec.py
new file mode 100644
index 0000000..88928e3
--- /dev/null
+++ b/src/edge0/attention/spec.py
@@ -0,0 +1,71 @@
+"""Attention taxonomy.
+
+edge0 does not re-implement attention kernels: the vendored base model
+implementations (``edge0.backends.mlx._impl``) carry the kernels (GQA /
+GatedDeltaNet for edge0-35b, MLA / DeltaNet for edge0-10b). This module
+describes attention so the framework can introspect a model — cache
+sizing, layer roles, documentation, and future kernels — without knowing
+the concrete implementation.
+
+A new attention kind = a new ``AttentionKind`` member + an
+``AttentionSpec`` description + a kernel implementation (in a vendored
+base or a backend ``_impl`` module). See docs/attention.md.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class AttentionKind(Enum):
+ """Attention / sequence-mixing families supported by edge0 models."""
+
+ GQA = "gqa"
+ """Grouped-query attention with softmax and KV cache."""
+
+ MLA = "mla"
+ """Multi-head latent attention (compressed KV)."""
+
+ DELTANET = "deltanet"
+ """DeltaNet linear attention (gated delta rule, no KV cache)."""
+
+ GATED_DELTANET = "gated_deltanet"
+ """GatedDeltaNet (gated delta rule with decay / linear attention)."""
+
+ DENSE_MLP = "dense_mlp"
+ """No sequence mixing (mlp-only layer)."""
+
+
+@dataclass(frozen=True)
+class AttentionSpec:
+ """Description of one layer's sequence-mixing block.
+
+ Attributes:
+ kind: attention family.
+ layer_indices: 0-based layer indices covered by this spec.
+ num_heads / num_kv_heads / head_dim: head geometry
+ (GQA/MLA only; None otherwise).
+ cache: whether the block maintains a KV-style cache
+ (False for linear-attention layers).
+ notes: free-form (e.g. hybrid schedules).
+ """
+
+ kind: AttentionKind
+ layer_indices: tuple[int, ...]
+ num_heads: int | None = None
+ num_kv_heads: int | None = None
+ head_dim: int | None = None
+ cache: bool = True
+ notes: str = ""
+
+ def __repr__(self) -> str: # compact one-liner for logs/docs
+ return (f"AttentionSpec({self.kind.value}, layers="
+ f"{self.layer_indices[0]}..{self.layer_indices[-1]}, "
+ f"cache={self.cache})")
+
+
+def summarize(specs: list[AttentionSpec]) -> str:
+ """Human-readable one-line summary, e.g. for the CLI banner."""
+ return ", ".join(
+ f"{s.kind.value}x{len(s.layer_indices)}" for s in specs)
diff --git a/src/edge0/backends/__init__.py b/src/edge0/backends/__init__.py
new file mode 100644
index 0000000..71e662c
--- /dev/null
+++ b/src/edge0/backends/__init__.py
@@ -0,0 +1,61 @@
+"""Backend abstraction.
+
+The framework core (engine, streaming, prerouter, adapters, model
+adapters, server) imports ONLY the backend namespaces re-exported here::
+
+ from edge0.backends import core, nn, io, quant
+
+A backend is a directory under ``edge0/backends/<name>/`` exposing:
+
+``backend.py``
+ ``core`` — array ops namespace (matmul, softmax, argpartition, take,
+ put_along_axis, eval, compile, ...)
+ ``nn`` — module factory namespace (Module, Linear, RMSNorm, silu, ...)
+ ``io`` — model/tokenizer/tensor-store loading
+ ``quant`` — quantized gather kernels (``gather_qmm``)
+ ``BackendImpl`` — small object with ``name`` and metadata
+
+The MLX backend is the reference implementation. A future CUDA backend
+implements the same surface with eager semantics; engine / prerouter /
+server / model-adapter code is shared verbatim. The op list below is the
+contract the framework currently exercises (grow it as models are added).
+
+Contract (documented, enforced by tests/grep in CI):
+ * The MLX package (``mlx`` / ``mlx_lm``) must never be imported
+ outside ``edge0/backends/mlx/`` — framework code reaches the backend
+ only through the namespaces re-exported here.
+ * ``core`` must provide (for the MLX backend these are mlx.core names):
+ array, zeros, eye, arange, full, expand_dims, squeeze, reshape,
+ transpose, concatenate, stack, split, matmul, softmax, silu (via nn),
+ sigmoid, erf, where, sum, cumsum, sort, topk, argpartition, take,
+ take_along_axis, put_along_axis, astype, item, tolist, eval, compile,
+ random.seed, random.categorical, and the dtypes float16 / float32 /
+ bfloat16 / int32 / uint32 / int64.
+ * ``nn`` must provide Module, Linear, RMSNorm, silu, gelu.
+ * ``io`` must provide load_model, load_tokenizer, open_tensor_store.
+ * ``quant`` must provide gather_qmm.
+"""
+
+import os
+
+_BACKEND = os.environ.get("EDGE0_BACKEND", "mlx")
+
+if _BACKEND == "mlx":
+ from edge0.backends.mlx.backend import ( # noqa: F401
+ BackendImpl,
+ core,
+ io,
+ nn,
+ quant,
+ )
+elif _BACKEND == "cuda":
+ raise ImportError(
+ "the CUDA backend is not implemented yet; set EDGE0_BACKEND=mlx "
+ "(or unset it)")
+else:
+ raise ImportError(
+ f"unknown backend {_BACKEND!r}; available: mlx")
+
+backend = BackendImpl()
+
+__all__ = ["backend", "core", "io", "nn", "quant"]
diff --git a/src/edge0/backends/base.py b/src/edge0/backends/base.py
new file mode 100644
index 0000000..0b209c2
--- /dev/null
+++ b/src/edge0/backends/base.py
@@ -0,0 +1,48 @@
+"""Tensor-store protocol: where adapter weights (LoRA, prerouter heads)
+are loaded from.
+
+The canonical format is safetensors (mmap-backed, lazy, with provenance
+metadata). No other format is supported.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+
+class TensorStore(ABC):
+ """Read-only view over a weights file (adapter artifacts).
+
+ Implementations must be lazily opened and must not materialize the
+ whole file into memory.
+ """
+
+ @abstractmethod
+ def keys(self) -> list[str]:
+ """Sorted tensor names in the store."""
+
+ @abstractmethod
+ def get(self, name: str):
+ """Return one tensor as an array (backend-native).
+
+ Raises KeyError for unknown names.
+ """
+
+ @abstractmethod
+ def metadata(self) -> dict:
+ """File-level provenance metadata (safetensors __metadata__)."""
+
+ @property
+ @abstractmethod
+ def path(self) -> str:
+ """Source file path (for error messages / logging)."""
+
+
+def open_tensor_store(path: str) -> TensorStore:
+ """Open a tensor store by extension (safetensors)."""
+ if path.endswith(".safetensors"):
+ from edge0.backends import io # deferred: backend selection
+ return io.SafeTensorsStore(path)
+ raise ValueError(
+ f"unsupported adapter format: {path!r} (only .safetensors is "
+ f"supported)")
diff --git a/src/edge0/backends/mlx/__init__.py b/src/edge0/backends/mlx/__init__.py
new file mode 100644
index 0000000..e092433
--- /dev/null
+++ b/src/edge0/backends/mlx/__init__.py
@@ -0,0 +1,9 @@
+"""MLX backend — the reference backend implementation.
+
+Everything that touches ``mlx`` lives under this package. Nothing else in
+the framework may import mlx directly.
+"""
+
+from edge0.backends.mlx.backend import BackendImpl, core, io, nn, quant
+
+__all__ = ["BackendImpl", "core", "io", "nn", "quant"]
diff --git a/src/edge0/backends/mlx/_impl/__init__.py b/src/edge0/backends/mlx/_impl/__init__.py
new file mode 100644
index 0000000..b9c0548
--- /dev/null
+++ b/src/edge0/backends/mlx/_impl/__init__.py
@@ -0,0 +1,11 @@
+"""Vendored base model implementations (licensing in NOTICE).
+
+* ``qwen3_next.py`` — mlx-lm 0.31.0's Qwen3.5-MoE model, byte-identical
+ to the upstream file except the import block, which was rewritten to
+ absolute ``mlx_lm.models`` paths so the file can live outside the
+ mlx_lm package. Pinned to mlx-lm==0.31.0 by pyproject.
+* ``bailing_hybrid.py`` — the ling 3.0 backbone vendored from the
+ production deployment, with the prerouter machinery renamed from the
+ legacy identifiers (``*_gate*`` -> ``prerouter*``) and no other
+ changes.
+"""
diff --git a/src/edge0/backends/mlx/_impl/bailing_hybrid.py b/src/edge0/backends/mlx/_impl/bailing_hybrid.py
new file mode 100644
index 0000000..2f83282
--- /dev/null
+++ b/src/edge0/backends/mlx/_impl/bailing_hybrid.py
@@ -0,0 +1,915 @@
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import os
+
+"""Vendored inclusionAI Ling 3.0 backbone (``model_type: bailing_hybrid``).
+
+**Why vendor:** Ling-3.0-tiny (7.9B total / 1.3B active, MIT) landed with
+day-0 SGLang/vLLM support and no MLX story beyond an unmerged mlx-lm PR:
+
+- mlx-lm: no ``bailing_hybrid`` (checked 0.31.3 + upstream main,
+ 2026-08-10); PR ml-explore/mlx-lm#1227 (Ling-2.6-era shape) open and
+ unreviewed since 2026-07-06 — the same situation ``hy_v3`` was vendored
+ out of.
+- transformers: no native port; the HF repo ships remote code that
+ imports ``fla`` (triton) at module level, so it cannot even run on
+ macOS.
+
+One module serves the whole family: Ling-3.0-tiny, Ling-3.0-flash and
+Ling-2.6-flash all declare ``bailing_hybrid``.
+
+**References (math verified against ALL of):**
+
+- HF remote code ``modeling_bailing_moe_v3.py`` — authoritative
+- fla kernels (``fla/ops/kda``) — exact ``safe_gate``/``lower_bound``
+ semantics (see ``_kda_gate``)
+- mlx-lm ``kimi_linear.py`` — MLX KDA idiom (KDA is Kimi's design;
+ Ling 3.0 reuses it with a safe-gate variant); ``ShortConv1d`` and the
+ group expert select are adapted from it
+- mlx-lm PR #1227 — Ling-2.6 MLX port (cross-check)
+
+Architecture (Ling-3.0-tiny values):
+
+- 24 layers in ``[KDA ×3, MLA]`` groups (``layer_group_size`` 4; the
+ trailing remainder layers are MLA); layer 0 is a dense MLP
+ (``first_k_dense_replace`` 1), all later layers are sparse MoE
+- KDA (Kimi Delta Attention, linear): q/k/v projections through causal
+ short convolutions (kernel 4, silu), per-head decay gate with the
+ SAFE-GATE law ``g = lower_bound * sigmoid(exp(A_log) * (f(x) +
+ dt_bias))`` (fla ``USE_LOWER_BOUND`` branch — NOT the softplus law
+ mlx-lm's ``compute_g`` implements), delta-rule state update, gated
+ RMSNorm output
+- MLA (DeepSeek-style): q LoRA 256 / kv LoRA 512, 128 nope + 64 rope
+ dims, interleaved RoPE theta 6e6, plus a V3-only HEAD-WISE output
+ gate (``g_proj``: hidden → n_heads, output scaled by sigmoid); the
+ output projection is named ``dense`` in the checkpoint
+- MoE: 128 routed experts top-8 + 1 shared, sigmoid scores with
+ ``noaux_tc`` expert-bias selection, 8 groups pick 4
+ (``topk_group``), normalized top-k probs scaled by 2.5
+- untied ``lm_head``; embeddings under ``model.word_embeddings``
+
+**Registration:** installed as ``sys.modules["mlx_lm.models.bailing_hybrid"]``
+by ``vllm_mlx.utils.tokenizer._register_vendored_archs`` (same trick as
+``deepseek_v4`` / ``hy_v3`` / ``muse_glimmer``), with a ``find_spec``
+probe that defers to native mlx-lm support the moment upstream ships it.
+
+**Sync policy:** when mlx-lm merges #1227's successor with V3 support,
+diff and delete this file.
+"""
+
+from dataclasses import dataclass
+from typing import Any
+
+import mlx.core as mx
+import mlx.nn as nn
+
+# Install the MLX hardware compatibility shim before importing any mlx-lm
+# module. mlx-lm captures its default stream during package import, which is
+# unsafe on the M5 single-stream path; every vendored model follows this
+# ordering contract.
+from mlx_lm.models.base import (
+ BaseModelArgs,
+ create_attention_mask,
+ scaled_dot_product_attention,
+)
+from mlx_lm.models.cache import ArraysCache, KVCache
+from mlx_lm.models.switch_layers import SwitchGLU
+
+
+def _rope_interleave_torch(
+ x: mx.array, positions: mx.array, theta: float
+) -> mx.array:
+ """Upstream ``apply_rotary_pos_emb_interleave``, verbatim from
+ ``modeling_bailing_moe_v3`` (transformers 4.x ``default`` RoPE, which is
+ what ``rope_scaling: null`` means for Ling-3.0-tiny):
+
+ freqs = 1 / theta ** (arange(0, d, 2) / d)
+ emb = cat((pos @ freqs, pos @ freqs))
+ qi = q.view(d//2, 2).transpose(-1, -2).reshape(d)
+ out = qi * cos + rotate_half(qi) * sin
+
+ ``positions`` are absolute token ids (arange(L) + cache offset), matching
+ torch's ``position_ids``/``cache_position``. We cannot use
+ ``mx.fast.rope`` for this: its ``offset`` is a single absolute position
+ (vector must have 1 element), so a prefill with offset=0 silently skips
+ rotation. Verified against a float64 numpy replica of the upstream
+ formula (rel ~1e-7, float32 rounding only).
+ """
+ B, H, L, D = x.shape
+ freqs = 1.0 / (theta ** (mx.arange(0, D, 2, dtype=mx.float32) / D))
+ angles = mx.outer(positions.astype(mx.float32), freqs)
+ emb = mx.concatenate([angles, angles], axis=-1)
+ cos = mx.cos(emb)[None, None, :, :]
+ sin = mx.sin(emb)[None, None, :, :]
+ xi = x.reshape(B, H, L, D // 2, 2).transpose(0, 1, 2, 4, 3).reshape(
+ B, H, L, D
+ )
+ h = D // 2
+ rh = mx.concatenate([-xi[..., h:], xi[..., :h]], axis=-1)
+ return xi * cos + rh * sin
+
+
+@dataclass
+class ModelArgs(BaseModelArgs):
+ model_type: str = "bailing_hybrid"
+ hidden_size: int = 1536
+ num_hidden_layers: int = 24
+ intermediate_size: int = 4608
+ num_attention_heads: int = 16
+ num_key_value_heads: int = 16
+ head_dim: int = 128
+ rms_norm_eps: float = 1e-6
+ vocab_size: int = 157184
+ max_position_embeddings: int = 131072
+ tie_word_embeddings: bool = False
+ # ---- layer schedule -------------------------------------------------
+ layer_group_size: int = 4
+ first_k_dense_replace: int = 1
+ # ---- KDA (linear attention) ----------------------------------------
+ short_conv_kernel_size: int = 4
+ no_kda_lora: bool = True
+ kda_safe_gate: bool = True
+ kda_lower_bound: float = -5.0
+ # ---- MLA -----------------------------------------------------------
+ q_lora_rank: int | None = 256
+ kv_lora_rank: int = 512
+ qk_nope_head_dim: int = 128
+ qk_rope_head_dim: int = 64
+ v_head_dim: int = 128
+ rope_theta: float = 6000000.0
+ rope_interleave: bool = True
+ rope_scaling: dict | None = None
+ use_qkv_bias: bool = False
+ gated_attention_proj_granularity_type: str | None = "head_wise"
+ # ---- MoE -----------------------------------------------------------
+ num_experts: int = 128
+ num_experts_per_tok: int = 8
+ num_shared_experts: int = 1
+ moe_intermediate_size: int = 512
+ moe_shared_expert_intermediate_size: int = 512
+ n_group: int = 8
+ topk_group: int = 4
+ norm_topk_prob: bool = True
+ routed_scaling_factor: float = 2.5
+ moe_router_enable_expert_bias: bool = True
+ # ---- pre-gated MoE (hybrid experiment) -----------------------------
+ prerouter_enabled: bool = False
+ prerouter_start_layer: int = 7
+ prerouter_hidden: int = 512
+
+ @property
+ def qk_head_dim(self) -> int:
+ return self.qk_nope_head_dim + self.qk_rope_head_dim
+
+ def is_mla_layer(self, idx: int) -> bool:
+ """Softmax (MLA) layers sit last in each group; any remainder
+ layers past the final full group are MLA too (torch ref
+ ``BailingMoeV3DecoderLayer.__init__``)."""
+ g = self.layer_group_size
+ full = self.num_hidden_layers // g * g
+ return (idx + 1) % g == 0 or idx >= full
+
+
+class ShortConv1d(nn.Module):
+ """Causal depthwise conv with silu and a rolling cache.
+
+ Adapted from mlx-lm ``kimi_linear.ShortConv1d`` (same wire: the
+ checkpoint stores ``*_conv1d.weight`` of shape [channels, ksize, 1]).
+ """
+
+ def __init__(self, channels: int, kernel_size: int):
+ super().__init__()
+ self.kernel_size = kernel_size
+ self.conv = nn.Conv1d(
+ channels, channels, kernel_size, groups=channels, bias=False
+ )
+
+ def __call__(
+ self,
+ x: mx.array,
+ state: mx.array | None = None,
+ ) -> tuple[mx.array, mx.array]:
+ B, T, C = x.shape
+ if state is None:
+ state = mx.zeros((B, self.kernel_size - 1, C), dtype=x.dtype)
+ conv_input = mx.concatenate([state, x], axis=1)
+ out = nn.silu(self.conv(conv_input))
+ if self.kernel_size == 1:
+ # ``-(k-1)`` would be ``-0`` and retain the WHOLE input,
+ # growing the cache every step (codex r3 #3).
+ new_state = conv_input[:, :0, :]
+ else:
+ new_state = conv_input[:, -(self.kernel_size - 1) :, :]
+ return out, new_state
+
+
+def _kda_gate(
+ f: mx.array,
+ A_log: mx.array, # noqa: N803 — the checkpoint's parameter name
+ dt_bias: mx.array,
+ *,
+ safe_gate: bool,
+ lower_bound: float,
+) -> mx.array:
+ """Per-channel log-decay gate.
+
+ fla ``fused_recurrent_kda_fwd_kernel`` (L166-171):
+
+ g_pre = f + dt_bias
+ USE_LOWER_BOUND: g = lower_bound * sigmoid(exp(A_log) * g_pre)
+ else: g = -exp(A_log) * softplus(g_pre)
+
+ Ling 3.0 ships ``kda_safe_gate=True`` / ``kda_lower_bound=-5`` — the
+ sigmoid law, clamping the log-decay into (lower_bound, 0). Computed
+ in float32 like the kernel.
+ """
+ f = f.astype(mx.float32) + dt_bias.astype(mx.float32)
+ a = mx.exp(A_log.astype(mx.float32))
+ if safe_gate:
+ return lower_bound * mx.sigmoid(a[..., None] * f)
+ return -a[..., None] * nn.softplus(f)
+
+
+def _kda_update(
+ q: mx.array,
+ k: mx.array,
+ v: mx.array,
+ g_log: mx.array,
+ beta: mx.array,
+ state: mx.array | None,
+) -> tuple[mx.array, mx.array]:
+ """Run the KDA delta-rule recurrence via mlx-lm's fused gated-delta
+ path (the same Metal kernel / ops pair qwen3-next and kimi_linear
+ ship on), with our precomputed safe-gate decay.
+
+ ``gated_delta_ops``/``gated_delta_kernel`` take the decay as a
+ MULTIPLIER (``compute_g`` returns ``exp(...)``), vectorized per
+ channel ``[B, T, H, Dk]`` — so the log-space safe gate is
+ exponentiated here. A per-token Python loop over the full prompt
+ (the naive fla reference) is prohibitively slow at 131K context
+ (codex r1 #1); the kernel path chunks the recurrence on Metal.
+ """
+ from mlx_lm.models.gated_delta import gated_delta_kernel, gated_delta_ops
+
+ g = mx.exp(g_log)
+ if state is None:
+ B = q.shape[0]
+ state = mx.zeros((B, v.shape[-2], v.shape[-1], k.shape[-1]), dtype=mx.float32)
+ # The Metal kernel tiles Dk in 32-lane strips (``n_per_t = Dk/32``);
+ # head dims below 32 would generate a zero-length array and fail the
+ # metallib build. Production checkpoints use head_dim 128; tiny test
+ # configs take the ops path.
+ use_kernel = (
+ gated_delta_kernel is not None
+ and k.shape[-1] % 32 == 0
+ and mx.default_device() == mx.gpu
+ and mx.metal.is_available()
+ )
+ fn = gated_delta_kernel if use_kernel else gated_delta_ops
+ return fn(q, k, v, g, beta, state, None)
+
+
+class BailingKDA(nn.Module):
+ """Kimi Delta Attention with the Ling V3 safe gate."""
+
+ def __init__(self, args: ModelArgs, layer_idx: int):
+ super().__init__()
+ self.num_heads = args.num_attention_heads
+ self.head_dim = args.head_dim
+ self.proj_dim = self.num_heads * self.head_dim
+ self.conv_kernel = args.short_conv_kernel_size
+ self.safe_gate = args.kda_safe_gate
+ self.lower_bound = float(args.kda_lower_bound)
+ self.no_kda_lora = args.no_kda_lora
+ self.scale = float(self.head_dim) ** -0.5
+
+ hidden = args.hidden_size
+ self.q_proj = nn.Linear(hidden, self.proj_dim, bias=False)
+ self.k_proj = nn.Linear(hidden, self.proj_dim, bias=False)
+ self.v_proj = nn.Linear(hidden, self.proj_dim, bias=False)
+ self.q_conv1d = ShortConv1d(self.proj_dim, self.conv_kernel)
+ self.k_conv1d = ShortConv1d(self.proj_dim, self.conv_kernel)
+ self.v_conv1d = ShortConv1d(self.proj_dim, self.conv_kernel)
+
+ if self.no_kda_lora:
+ self.f_proj = nn.Linear(hidden, self.proj_dim, bias=False)
+ self.g_proj = nn.Linear(hidden, self.proj_dim, bias=False)
+ else:
+ # Ling-2.6 / flash variants use LoRA pairs. The bottleneck is
+ # head_dim BY DESIGN — the authoritative reference hard-codes
+ # it the same way (modeling_bailing_moe_v3.BailingMoeV3
+ # KimiDeltaAttention: ``f_a_proj = Linear(hidden_size,
+ # head_dim)``) and no bailing config revision publishes a
+ # separate f/g LoRA-rank field to wire.
+ self.f_a_proj = nn.Linear(hidden, self.head_dim, bias=False)
+ self.f_b_proj = nn.Linear(self.head_dim, self.proj_dim, bias=False)
+ self.g_a_proj = nn.Linear(hidden, self.head_dim, bias=False)
+ self.g_b_proj = nn.Linear(self.head_dim, self.proj_dim, bias=False)
+
+ self.b_proj = nn.Linear(hidden, self.num_heads, bias=False)
+ self.A_log = mx.zeros((self.num_heads,))
+ self.dt_bias = mx.zeros((self.proj_dim,))
+ self.o_norm = nn.RMSNorm(self.head_dim, eps=args.rms_norm_eps)
+ self.o_proj = nn.Linear(self.proj_dim, hidden, bias=False)
+
+ def __call__(
+ self,
+ x: mx.array,
+ mask: mx.array | None = None,
+ cache: Any | None = None,
+ ) -> mx.array:
+ B, T, _ = x.shape
+ dtype = x.dtype
+
+ if cache is not None:
+ q_state, k_state, v_state, ssm_state = cache
+ else:
+ q_state = k_state = v_state = ssm_state = None
+
+ q_conv, q_state = self.q_conv1d(self.q_proj(x), q_state)
+ k_conv, k_state = self.k_conv1d(self.k_proj(x), k_state)
+ v_conv, v_state = self.v_conv1d(self.v_proj(x), v_state)
+ if cache is not None:
+ cache[0] = q_state
+ cache[1] = k_state
+ cache[2] = v_state
+
+ q = q_conv.reshape(B, T, self.num_heads, self.head_dim)
+ k = k_conv.reshape(B, T, self.num_heads, self.head_dim)
+ v = v_conv.reshape(B, T, self.num_heads, self.head_dim)
+
+ # fla applies L2-norm to q/k in-kernel plus scale=d^-0.5. Use the
+ # exact l2norm form (x / (||x|| + eps)) so the eps placement
+ # matches the kernel bit-for-bit; the kimi_linear rms_norm
+ # folding trick puts eps under the sqrt and costs ~2e-4 parity.
+ qf = q.astype(mx.float32)
+ kf = k.astype(mx.float32)
+ q = self.scale * qf / (mx.linalg.norm(qf, axis=-1, keepdims=True) + 1e-6)
+ k = kf / (mx.linalg.norm(kf, axis=-1, keepdims=True) + 1e-6)
+
+ if self.no_kda_lora:
+ f = self.f_proj(x)
+ gate = self.g_proj(x)
+ else:
+ f = self.f_b_proj(self.f_a_proj(x))
+ gate = self.g_b_proj(self.g_a_proj(x))
+ f = f.reshape(B, T, self.num_heads, self.head_dim)
+ g = _kda_gate(
+ f,
+ self.A_log,
+ self.dt_bias.reshape(self.num_heads, self.head_dim),
+ safe_gate=self.safe_gate,
+ lower_bound=self.lower_bound,
+ )
+ beta = mx.sigmoid(self.b_proj(x).astype(mx.float32))
+
+ out, ssm_state = _kda_update(q, k, v, g, beta, ssm_state)
+ if cache is not None:
+ cache[3] = ssm_state
+
+ gate = gate.reshape(B, T, self.num_heads, self.head_dim)
+ out = self.o_norm(out.astype(dtype)) * mx.sigmoid(gate)
+ return self.o_proj(out.reshape(B, T, -1))
+
+
+class BailingMLA(nn.Module):
+ """DeepSeek-style MLA plus the V3 head-wise output gate."""
+
+ def __init__(self, args: ModelArgs):
+ super().__init__()
+ self.num_heads = args.num_attention_heads
+ self.qk_nope_head_dim = args.qk_nope_head_dim
+ self.qk_rope_head_dim = args.qk_rope_head_dim
+ self.qk_head_dim = args.qk_head_dim
+ self.v_head_dim = args.v_head_dim
+ self.kv_lora_rank = args.kv_lora_rank
+ self.q_lora_rank = args.q_lora_rank
+ self.scale = self.qk_head_dim**-0.5
+ self.gate_kind = args.gated_attention_proj_granularity_type
+
+ hidden = args.hidden_size
+ bias = args.use_qkv_bias
+ if self.q_lora_rank is None:
+ # bias=False BY REFERENCE: the authoritative torch modeling
+ # hard-codes the direct q_proj without bias (use_qkv_bias
+ # applies only to the LoRA a-projections and kv_a/dense) —
+ # mirroring it exactly, not an omission.
+ self.q_proj = nn.Linear(
+ hidden, self.num_heads * self.qk_head_dim, bias=False
+ )
+ else:
+ self.q_a_proj = nn.Linear(hidden, self.q_lora_rank, bias=bias)
+ self.q_a_layernorm = nn.RMSNorm(self.q_lora_rank, eps=args.rms_norm_eps)
+ self.q_b_proj = nn.Linear(
+ self.q_lora_rank, self.num_heads * self.qk_head_dim, bias=False
+ )
+ self.kv_a_proj_with_mqa = nn.Linear(
+ hidden, self.kv_lora_rank + self.qk_rope_head_dim, bias=bias
+ )
+ self.kv_a_layernorm = nn.RMSNorm(self.kv_lora_rank, eps=args.rms_norm_eps)
+ self.kv_b_proj = nn.Linear(
+ self.kv_lora_rank,
+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
+ bias=False,
+ )
+ if self.gate_kind == "head_wise":
+ self.g_proj = nn.Linear(hidden, self.num_heads, bias=False)
+ elif self.gate_kind == "element_wise":
+ self.g_proj = nn.Linear(
+ hidden, self.num_heads * self.v_head_dim, bias=False
+ )
+ # Checkpoint names the output projection ``dense``.
+ self.dense = nn.Linear(self.num_heads * self.v_head_dim, hidden, bias=bias)
+ # RoPE follows the upstream torch implementation exactly:
+ # transformers 4.x ``default`` (rope_scaling=null) computes
+ # inv_freq = 1/theta^(2i/dim), emb = cat((freqs, freqs)), then
+ # apply_rotary_pos_emb_interleave rotates consecutive pairs
+ # (x[2i], x[2i+1]) by pos*inv_freq[i]. We hand-roll it
+ # (_rope_interleave_torch) because mx.fast.rope's offset is a single
+ # absolute position: prefill with offset=0 would skip rotation.
+ if args.rope_scaling:
+ raise NotImplementedError(
+ "bailing_hybrid: rope_scaling is not supported by the "
+ f"vendored backbone yet (got {args.rope_scaling!r})"
+ )
+ self.rope_theta = args.rope_theta
+
+ def __call__(
+ self,
+ x: mx.array,
+ mask: mx.array | None = None,
+ cache: Any | None = None,
+ ) -> mx.array:
+ B, L, _ = x.shape
+
+ if self.q_lora_rank is None:
+ q = self.q_proj(x)
+ else:
+ q = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(x)))
+ q = q.reshape(B, L, self.num_heads, self.qk_head_dim).transpose(0, 2, 1, 3)
+ q_nope, q_pe = mx.split(q, [self.qk_nope_head_dim], axis=-1)
+
+ compressed = self.kv_a_proj_with_mqa(x)
+ kv_latent, k_pe = mx.split(compressed, [self.kv_lora_rank], axis=-1)
+ kv_latent = self.kv_a_layernorm(kv_latent)
+ k_pe = k_pe.reshape(B, L, 1, self.qk_rope_head_dim).transpose(0, 2, 1, 3)
+
+ kv = self.kv_b_proj(kv_latent)
+ kv = kv.reshape(
+ B, L, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
+ ).transpose(0, 2, 1, 3)
+ k_nope, values = mx.split(kv, [self.qk_nope_head_dim], axis=-1)
+
+ offset = cache.offset if cache is not None else 0
+ positions = mx.arange(L) + offset
+ q_pe = _rope_interleave_torch(q_pe, positions, self.rope_theta)
+ k_pe = _rope_interleave_torch(k_pe, positions, self.rope_theta)
+ k_pe = mx.broadcast_to(k_pe, (B, self.num_heads, L, self.qk_rope_head_dim))
+
+ queries = mx.concatenate([q_nope, q_pe], axis=-1)
+ keys = mx.concatenate([k_nope, k_pe], axis=-1)
+
+ if cache is not None:
+ keys, values = cache.update_and_fetch(keys, values)
+
+ out = scaled_dot_product_attention(
+ queries,
+ values=values,
+ keys=keys,
+ cache=cache,
+ scale=self.scale,
+ mask=mask,
+ )
+ out = out.transpose(0, 2, 1, 3).reshape(B, L, -1)
+ if self.gate_kind == "head_wise":
+ gate = mx.sigmoid(self.g_proj(x)) # [B, L, H]
+ out = out.reshape(B, L, self.num_heads, self.v_head_dim)
+ out = out * gate[..., None]
+ out = out.reshape(B, L, -1)
+ elif self.gate_kind == "element_wise":
+ out = out * mx.sigmoid(self.g_proj(x))
+ return self.dense(out)
+
+
+class BailingMLP(nn.Module):
+ def __init__(self, args: ModelArgs, intermediate: int):
+ super().__init__()
+ self.gate_proj = nn.Linear(args.hidden_size, intermediate, bias=False)
+ self.up_proj = nn.Linear(args.hidden_size, intermediate, bias=False)
+ self.down_proj = nn.Linear(intermediate, args.hidden_size, bias=False)
+
+ def __call__(self, x: mx.array) -> mx.array:
+ return self.down_proj(nn.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class BailingGate(nn.Module):
+ """Sigmoid-score router with noaux_tc expert-bias group selection.
+
+ DeepSeek-V3 routing law (torch ref ``BailingMoeV3Gate``): selection
+ scores = sigmoid(logits) + expert_bias, grouped into ``n_group``;
+ the ``topk_group`` best groups (by sum of each group's top-2
+ selection scores) survive; the top-k experts within the surviving
+ groups are chosen by selection score; the WEIGHTS are the raw
+ sigmoid scores of the chosen experts (bias excluded), normalized if
+ ``norm_topk_prob`` and scaled by ``routed_scaling_factor``.
+ """
+
+ def __init__(self, args: ModelArgs):
+ super().__init__()
+ self.args = args
+ self.weight = mx.zeros((args.num_experts, args.hidden_size))
+ if args.moe_router_enable_expert_bias:
+ self.expert_bias = mx.zeros((args.num_experts,))
+
+ def __call__(self, x: mx.array) -> tuple[mx.array, mx.array]:
+ a = self.args
+ scores = mx.sigmoid((x @ self.weight.T).astype(mx.float32))
+ select = scores
+ if a.moe_router_enable_expert_bias:
+ select = scores + self.expert_bias
+
+ B = select.shape[:-1]
+ k_drop = a.n_group - a.topk_group
+ if k_drop > 0:
+ grouped = select.reshape(*B, a.n_group, a.num_experts // a.n_group)
+ # Group score = sum of the top-2 selection scores in the group.
+ top2 = mx.topk(grouped, 2, axis=-1)
+ group_scores = top2.sum(axis=-1)
+ drop = mx.argpartition(group_scores, kth=k_drop - 1, axis=-1)[..., :k_drop]
+ # ``put_along_axis`` broadcasts the trailing size-1 index dim
+ # across the experts-per-group axis (numpy semantics), so the
+ # single write masks EVERY expert slot of each dropped group —
+ # unlike torch scatter, no expand is needed. Pinned by
+ # ``test_gate_group_drop_masks_whole_group`` and by the
+ # reference parity run (1.5e-6 on a drop-path config).
+ masked = mx.put_along_axis(
+ grouped,
+ mx.expand_dims(drop, -1),
+ mx.array(-float("inf"), grouped.dtype),
+ axis=-2,
+ )
+ select = masked.reshape(*B, a.num_experts)
+ # topk_group == n_group keeps every group — nothing to drop
+ # (codex r3 #1: argpartition(kth=-1) would fault).
+
+ k = a.num_experts_per_tok
+ idx = mx.argpartition(-select, kth=k - 1, axis=-1)[..., :k]
+ w = mx.take_along_axis(scores, idx, axis=-1)
+ if a.norm_topk_prob:
+ w = w / (w.sum(axis=-1, keepdims=True) + 1e-20)
+ w = w * a.routed_scaling_factor
+ return idx, w
+
+
+class BailingPrerouter(nn.Module):
+ """One-MoE-block-ahead expert selector (arXiv 2308.12066 style).
+
+ Input is the current layer's MoE input (post-attention RMSNorm output)
+ plus optional current-layer top-8 one-hot and prev-token top-8 one-hot
+ (v4/v5 feature set). Output is 128 logits. Selection reuses the
+ sigmoid + group-limited top-k semantics of ``BailingGate`` (no expert
+ bias), so at deployment the pre-gate can fully replace the router for
+ its target layer.
+ """
+
+ def __init__(
+ self,
+ args: ModelArgs,
+ hidden: int | None = None,
+ input_dim: int | None = None,
+ ):
+ super().__init__()
+ self.args = args
+ self.hidden = hidden or args.prerouter_hidden
+ self.input_dim = input_dim or args.hidden_size
+ self.fc1 = nn.Linear(self.input_dim, self.hidden, bias=False)
+ self.fc2 = nn.Linear(self.hidden, args.num_experts, bias=False)
+ # v4/v5 residual linear path (zero by default; trained in v5).
+ self.linear_init = nn.Linear(
+ self.input_dim, args.num_experts, bias=False
+ )
+ self.linear_init.weight = mx.zeros(
+ (args.num_experts, self.input_dim), dtype=mx.float32
+ )
+
+ def __call__(
+ self,
+ x: mx.array,
+ topk_oh: mx.array | None = None,
+ prev_oh: mx.array | None = None,
+ ) -> mx.array:
+ dtype = self.fc1.weight.dtype
+ feats = [x.astype(dtype)]
+ if topk_oh is not None:
+ feats.append(topk_oh.astype(dtype))
+ if prev_oh is not None:
+ feats.append(prev_oh.astype(dtype))
+ xi = mx.concatenate(feats, axis=-1)
+ h = nn.gelu(self.fc1(xi))
+ return self.fc2(h) + self.linear_init(xi)
+
+
+class BailingSparseMoE(nn.Module):
+ def __init__(
+ self,
+ args: ModelArgs,
+ prerouter: BailingPrerouter | None = None,
+ use_prerouter: bool = False,
+ ):
+ super().__init__()
+ self.args = args
+ self.gate = BailingGate(args)
+ self.prerouter = prerouter
+ self.use_prerouter = use_prerouter
+ self.last_topk: mx.array | None = None
+ self.prev_topk_oh: mx.array | None = None
+ # v5 was trained with the TEACHER (original router) top-8 as the
+ # pre-gate's "current layer top-8" feature, so at decode we compute
+ # the original gate top-8 for the feature even when the pre-gate
+ # selects the executed experts. Set to "executed" to use the
+ # pre-gate's own selection instead (post-SFT ce_only semantics).
+ self.feature_topk = os.environ.get(
+ "PREROUTER_FEATURE_TOPK", "teacher")
+ # Same-token intra path computes the teacher feature inside the
+ # after-attention callback; the per-layer gate re-computation here
+ # would be dead work (nothing consumes last_topk in intra mode).
+ self.intra_mode = os.environ.get("PREROUTER_INTRA", "0") == "1"
+ self.experts = SwitchGLU(
+ args.hidden_size, args.moe_intermediate_size, args.num_experts
+ )
+ self.shared_experts = BailingMLP(
+ args,
+ args.moe_shared_expert_intermediate_size * args.num_shared_experts,
+ )
+
+ def _select_from_logits(self, logits: mx.array) -> tuple[mx.array, mx.array]:
+ """Same routing math as BailingGate, but from pre-gate logits
+ (expert bias excluded)."""
+ a = self.args
+ scores = mx.sigmoid(logits.astype(mx.float32))
+ B = scores.shape[:-1]
+ k_drop = a.n_group - a.topk_group
+ if k_drop > 0:
+ grouped = scores.reshape(
+ *B, a.n_group, a.num_experts // a.n_group
+ )
+ top2 = mx.topk(grouped, 2, axis=-1)
+ group_scores = top2.sum(axis=-1)
+ drop = mx.argpartition(
+ group_scores, kth=k_drop - 1, axis=-1
+ )[..., :k_drop]
+ masked = mx.put_along_axis(
+ grouped,
+ mx.expand_dims(drop, -1),
+ mx.array(-float("inf"), grouped.dtype),
+ axis=-2,
+ )
+ select = masked.reshape(*B, a.num_experts)
+ else:
+ select = scores
+ k = a.num_experts_per_tok
+ idx = mx.argpartition(-select, kth=k - 1, axis=-1)[..., :k]
+ w = mx.take_along_axis(scores, idx, axis=-1)
+ if a.norm_topk_prob:
+ w = w / (w.sum(axis=-1, keepdims=True) + 1e-20)
+ w = w * a.routed_scaling_factor
+ return idx, w
+
+ def __call__(
+ self, x: mx.array, prev_prerouter_logits: mx.array | None = None
+ ) -> mx.array:
+ if self.use_prerouter and prev_prerouter_logits is not None:
+ idx, w = self._select_from_logits(prev_prerouter_logits)
+ if (not self.intra_mode
+ and self.feature_topk == "teacher"):
+ feat_idx, _ = self.gate(x)
+ else:
+ feat_idx = idx
+ else:
+ idx, w = self.gate(x)
+ feat_idx = idx
+ self.last_topk = feat_idx if not self.intra_mode else None
+ routed = self.experts(x, idx)
+ out = (routed * w[..., None].astype(routed.dtype)).sum(axis=-2)
+ return out + self.shared_experts(x)
+
+
+class BailingDecoderLayer(nn.Module):
+ def __init__(self, args: ModelArgs, layer_idx: int):
+ super().__init__()
+ self.layer_idx = layer_idx
+ self.is_mla = args.is_mla_layer(layer_idx)
+ if self.is_mla:
+ self.attention = BailingMLA(args)
+ else:
+ self.attention = BailingKDA(args, layer_idx)
+ self.prerouter_enabled = bool(getattr(args, "prerouter_enabled", False))
+ self.prerouter_start_layer = int(getattr(args, "prerouter_start_layer", 7))
+ self.prerouter_logits: mx.array | None = None
+ self.m_in_cache: mx.array | None = None
+ self.after_attention_cb = None
+ self.has_prerouter = (
+ self.prerouter_enabled
+ and layer_idx >= self.prerouter_start_layer - 1
+ and layer_idx < args.num_hidden_layers - 1
+ )
+ self.use_prerouter = (
+ self.prerouter_enabled
+ and layer_idx >= self.prerouter_start_layer
+ and layer_idx < args.num_hidden_layers
+ )
+ if layer_idx >= args.first_k_dense_replace:
+ self.mlp = BailingSparseMoE(
+ args,
+ prerouter=(
+ BailingPrerouter(args)
+ if self.has_prerouter
+ else None
+ ),
+ use_prerouter=self.use_prerouter,
+ )
+ else:
+ self.mlp = BailingMLP(args, args.intermediate_size)
+ self.input_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)
+ self.post_attention_layernorm = nn.RMSNorm(
+ args.hidden_size, eps=args.rms_norm_eps
+ )
+
+ def __call__(
+ self,
+ x: mx.array,
+ mask: mx.array | None = None,
+ cache: Any | None = None,
+ prev_prerouter_logits: mx.array | None = None,
+ ) -> mx.array:
+ h = x + self.attention(self.input_layernorm(x), mask, cache)
+ m_in = self.post_attention_layernorm(h)
+ if self.has_prerouter:
+ # Defer the pre-gate MLP to a single post-eval batch (PrerouterStager
+ # stacks every layer's m_in), keeping the forward graph lean.
+ self.m_in_cache = m_in
+ self.prerouter_logits = None
+ else:
+ self.prerouter_logits = None
+ self.m_in_cache = None
+ if self.after_attention_cb is not None:
+ self.after_attention_cb(self.layer_idx, m_in)
+ if isinstance(self.mlp, BailingSparseMoE):
+ out = self.mlp(m_in, prev_prerouter_logits=prev_prerouter_logits)
+ else:
+ out = self.mlp(m_in)
+ return h + out
+
+
+class BailingModel(nn.Module):
+ def __init__(self, args: ModelArgs):
+ super().__init__()
+ self.args = args
+ self.word_embeddings = nn.Embedding(args.vocab_size, args.hidden_size)
+ self.layers = [
+ BailingDecoderLayer(args, i) for i in range(args.num_hidden_layers)
+ ]
+ self.norm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)
+ self.first_mla_idx = next(
+ i for i in range(args.num_hidden_layers) if args.is_mla_layer(i)
+ )
+
+ def __call__(
+ self,
+ inputs: mx.array,
+ cache=None,
+ input_embeddings: mx.array | None = None,
+ after_layer_cb=None,
+ before_layer_cb=None,
+ async_eval_per_layer: bool = False,
+ prerouter_cache: dict[int, mx.array] | None = None,
+ ) -> mx.array:
+ h = (
+ input_embeddings
+ if input_embeddings is not None
+ else self.word_embeddings(inputs)
+ )
+ if cache is None:
+ cache = [None] * len(self.layers)
+ mla_mask = create_attention_mask(h, cache[self.first_mla_idx])
+ # Hidden-state clip (LING_HIDDEN_CLIP, e.g. 1000): clamps each
+ # layer's output to [-v, +v] and zeroes NaN entries so a single
+ # fp16 overflow inside one layer cannot poison the whole network
+ # (observed: decode blowup -> all-NaN logits -> argmax falls back
+ # to token 0 ('!') death spiral). Off by default.
+ # (v7-deploy NAN_BANG_COLLAPSE_FIX.md parity.)
+ _clip_v = float(os.environ.get("LING_HIDDEN_CLIP", "0"))
+ for li, (layer, c) in enumerate(zip(self.layers, cache)):
+ if before_layer_cb is not None:
+ before_layer_cb(li)
+ mask = mla_mask if layer.is_mla else None
+ prev_prerouter_logits = None
+ if (
+ getattr(layer, "use_prerouter", False)
+ and prerouter_cache is not None
+ and li in prerouter_cache
+ ):
+ prev_prerouter_logits = prerouter_cache[li]
+ h = layer(h, mask, c, prev_prerouter_logits)
+ if _clip_v > 0:
+ h = mx.where(
+ mx.isnan(h), mx.zeros_like(h),
+ mx.clip(h, -_clip_v, _clip_v))
+ if after_layer_cb is not None:
+ after_layer_cb(li, h)
+ if async_eval_per_layer:
+ # Fire this layer's graph on the GPU while the CPU keeps
+ # building the next layer (overlaps the whole-layer weight
+ # load of the E3b prefill path). The final logits eval
+ # still waits for everything.
+ mx.async_eval(h)
+ return self.norm(h)
+
+
+class Model(nn.Module):
+ def __init__(self, args: ModelArgs):
+ super().__init__()
+ self.args = args
+ self.model_type = args.model_type
+ self.model = BailingModel(args)
+ self.tie_word_embeddings = args.tie_word_embeddings
+ if not self.tie_word_embeddings:
+ self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False)
+
+ def __call__(
+ self,
+ inputs: mx.array,
+ cache=None,
+ input_embeddings: mx.array | None = None,
+ after_layer_cb=None,
+ ) -> mx.array:
+ h = self.model(inputs, cache, input_embeddings, after_layer_cb)
+ if self.tie_word_embeddings:
+ return self.model.word_embeddings.as_linear(h)
+ return self.lm_head(h)
+
+ def sanitize(self, weights):
+ """Stack per-expert weights into SwitchGLU layout and drop any
+ MTP head (flash/2.6 ship ``num_nextn_predict_layers`` layers we
+ do not serve)."""
+ weights = {
+ k: v for k, v in weights.items() if ".mtp_" not in k and "mtp." not in k
+ }
+ n_layers = self.args.num_hidden_layers
+ for li in range(n_layers):
+ prefix = f"model.layers.{li}"
+ if f"{prefix}.mlp.experts.0.gate_proj.weight" not in weights and (
+ f"{prefix}.mlp.experts.0.gate_proj.scales" not in weights
+ ):
+ continue
+ for m in ("gate_proj", "up_proj", "down_proj"):
+ for part in ("weight", "scales", "biases"):
+ key0 = f"{prefix}.mlp.experts.0.{m}.{part}"
+ if key0 not in weights:
+ continue
+ joined = mx.stack(
+ [
+ weights.pop(f"{prefix}.mlp.experts.{e}.{m}.{part}")
+ for e in range(self.args.num_experts)
+ ]
+ )
+ weights[f"{prefix}.mlp.experts.{m}.{part}"] = joined
+ # Conv weights: the checkpoint stores them at ``*_conv1d.weight``
+ # in torch depthwise layout [C, 1, ksize] (or flat [C, ksize]);
+ # our ShortConv1d nests an nn.Conv1d at ``.conv`` expecting
+ # [C, ksize, 1].
+ for k in list(weights):
+ if k.endswith("_conv1d.weight"):
+ w = weights.pop(k)
+ if w.ndim == 2:
+ w = w[..., None]
+ elif w.ndim == 3 and w.shape[1] == 1:
+ w = w.swapaxes(1, 2)
+ weights[k[: -len(".weight")] + ".conv.weight"] = w
+ return weights
+
+ @property
+ def layers(self):
+ return self.model.layers
+
+ def make_cache(self):
+ return [
+ KVCache() if layer.is_mla else ArraysCache(size=4)
+ for layer in self.model.layers
+ ]
+
+ @property
+ def quant_predicate(self):
+ def predicate(path, module):
+ # Short-conv weights stay fp — tiny tensors, and quantized
+ # depthwise conv has no parity coverage (codex r3 #2).
+ if "_conv1d" in path:
+ return False
+ # Router quality is precision-critical (deepseek-v3
+ # precedent): keep the gate at 8-bit.
+ if "mlp.gate" in path and "gate_proj" not in path:
+ return {"group_size": 64, "bits": 8}
+ return True
+
+ return predicate
diff --git a/src/edge0/backends/mlx/_impl/qwen3_5.py b/src/edge0/backends/mlx/_impl/qwen3_5.py
new file mode 100644
index 0000000..c5cf965
--- /dev/null
+++ b/src/edge0/backends/mlx/_impl/qwen3_5.py
@@ -0,0 +1,545 @@
+# Copyright © 2026 Apple Inc.
+#
+# edge0 vendored copy of mlx-lm 0.31.0 ``mlx_lm/models/qwen3_5.py``,
+# byte-identical EXCEPT: (1) the import block rewritten to absolute
+# paths — infra bits (base/cache/gated_delta) point at the pinned
+# mlx_lm package, sibling model files point at this vendored package so
+# the prerouter block patch lands on OUR ``Qwen3NextSparseMoeBlock``;
+# (2) ``Qwen3_5TextModel.__call__`` gained the edge0 engine callback
+# hooks (before/after_layer_cb + async_eval_per_layer).
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Union
+
+import mlx.core as mx
+import mlx.nn as nn
+from mlx.nn.layers.distributed import shard_inplace, shard_linear, sum_gradients
+from mlx.utils import tree_map
+
+from mlx_lm.models.base import (
+ BaseModelArgs,
+ create_attention_mask,
+ create_ssm_mask,
+)
+from mlx_lm.models.cache import ArraysCache, KVCache
+from mlx_lm.models.gated_delta import gated_delta_update
+from edge0.backends.mlx._impl.qwen3_next import Qwen3NextAttention as Attention
+from edge0.backends.mlx._impl.qwen3_next import Qwen3NextMLP as MLP
+from edge0.backends.mlx._impl.qwen3_next import Qwen3NextRMSNormGated as RMSNormGated
+from edge0.backends.mlx._impl.qwen3_next import Qwen3NextSparseMoeBlock as SparseMoeBlock
+
+
+@dataclass
+class TextModelArgs(BaseModelArgs):
+ model_type: str = ""
+ hidden_size: int = 4096
+ intermediate_size: int = 14336
+ num_hidden_layers: int = 32
+ num_attention_heads: int = 32
+ rms_norm_eps: float = 1e-6
+ vocab_size: int = 151936
+ num_key_value_heads: int = 8
+ max_position_embeddings: int = 131072
+ linear_num_value_heads: int = 64
+ linear_num_key_heads: int = 16
+ linear_key_head_dim: int = 192
+ linear_value_head_dim: int = 128
+ linear_conv_kernel_dim: int = 4
+ tie_word_embeddings: bool = False
+ attention_bias: bool = False
+ head_dim: Optional[int] = None
+ full_attention_interval: int = 4
+
+ # MoE fields (optional, for Qwen3_5MoeForConditionalGeneration)
+ num_experts: int = 0
+ num_experts_per_tok: int = 0
+ decoder_sparse_step: int = 1
+ shared_expert_intermediate_size: int = 0
+ moe_intermediate_size: int = 0
+ norm_topk_prob: bool = True
+
+ # Rope parameters
+ rope_parameters: Optional[Dict[str, Union[float, str, bool, List[int]]]] = field(
+ default_factory=lambda: {
+ "type": "default",
+ "mrope_section": [11, 11, 10],
+ "rope_theta": 100000,
+ "partial_rotary_factor": 0.25,
+ }
+ )
+
+ # Derived from rope_parameters (set in __post_init__)
+ partial_rotary_factor: float = 0.25
+ rope_theta: float = 100000.0
+ rope_scaling: Optional[Dict[str, Union[float, str]]] = None
+
+ def __post_init__(self):
+ if self.head_dim is None:
+ self.head_dim = self.hidden_size // self.num_attention_heads
+
+ if self.rope_parameters:
+ if (
+ "type" not in self.rope_parameters
+ and "rope_type" in self.rope_parameters
+ ):
+ self.rope_parameters["type"] = self.rope_parameters.pop("rope_type")
+
+ self.partial_rotary_factor = self.rope_parameters.get(
+ "partial_rotary_factor", 0.25
+ )
+ self.rope_theta = self.rope_parameters.get("rope_theta", 100000.0)
+ self.rope_scaling = self.rope_parameters
+
+
+class GatedDeltaNet(nn.Module):
+ def __init__(self, config: TextModelArgs):
+ super().__init__()
+ self.hidden_size = config.hidden_size
+ self.num_v_heads = config.linear_num_value_heads
+ self.num_k_heads = config.linear_num_key_heads
+ self.head_k_dim = config.linear_key_head_dim
+ self.head_v_dim = config.linear_value_head_dim
+ self.key_dim = self.head_k_dim * self.num_k_heads
+ self.value_dim = self.head_v_dim * self.num_v_heads
+ if self.num_v_heads % self.num_k_heads != 0:
+ raise ValueError(
+ f"num_v_heads ({self.num_v_heads}) must be divisible by num_k_heads ({self.num_k_heads})"
+ )
+
+ self.conv_kernel_size = config.linear_conv_kernel_dim
+ self.layer_norm_epsilon = config.rms_norm_eps
+
+ self.conv_dim = self.key_dim * 2 + self.value_dim
+ self.conv1d = nn.Conv1d(
+ in_channels=self.conv_dim,
+ out_channels=self.conv_dim,
+ bias=False,
+ kernel_size=self.conv_kernel_size,
+ groups=self.conv_dim,
+ padding=0,
+ )
+
+ self.in_proj_qkv = nn.Linear(
+ self.hidden_size, self.key_dim * 2 + self.value_dim, bias=False
+ )
+ self.in_proj_z = nn.Linear(self.hidden_size, self.value_dim, bias=False)
+ self.in_proj_b = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)
+ self.in_proj_a = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)
+
+ self.dt_bias = mx.ones(self.num_v_heads)
+
+ A = mx.random.uniform(low=0, high=16, shape=(self.num_v_heads,))
+ self.A_log = mx.log(A)
+
+ self.norm = RMSNormGated(self.head_v_dim, eps=self.layer_norm_epsilon)
+
+ self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)
+
+ self.sharding_group = None
+
+ def __call__(
+ self,
+ inputs: mx.array,
+ mask: Optional[mx.array] = None,
+ cache: Optional[Any] = None,
+ ) -> mx.array:
+ B, S, _ = inputs.shape
+
+ if self.sharding_group is not None:
+ inputs = sum_gradients(self.sharding_group)(inputs)
+
+ qkv = self.in_proj_qkv(inputs)
+ z = self.in_proj_z(inputs).reshape(B, S, self.num_v_heads, self.head_v_dim)
+ b = self.in_proj_b(inputs)
+ a = self.in_proj_a(inputs)
+
+ if cache is not None and cache[0] is not None:
+ conv_state = cache[0]
+ else:
+ conv_state = mx.zeros(
+ (B, self.conv_kernel_size - 1, self.conv_dim),
+ dtype=inputs.dtype,
+ )
+
+ if mask is not None:
+ qkv = mx.where(mask[..., None], qkv, 0)
+ conv_input = mx.concatenate([conv_state, qkv], axis=1)
+ if cache is not None:
+ cache[0] = conv_input[:, -(self.conv_kernel_size - 1) :]
+ conv_out = nn.silu(self.conv1d(conv_input))
+
+ q, k, v = [
+ t.reshape(B, S, h, d)
+ for t, h, d in zip(
+ mx.split(conv_out, [self.key_dim, 2 * self.key_dim], -1),
+ [self.num_k_heads, self.num_k_heads, self.num_v_heads],
+ [self.head_k_dim, self.head_k_dim, self.head_v_dim],
+ )
+ ]
+
+ state = cache[1] if cache else None
+ inv_scale = k.shape[-1] ** -0.5
+ q = (inv_scale**2) * mx.fast.rms_norm(q, None, 1e-6)
+ k = inv_scale * mx.fast.rms_norm(k, None, 1e-6)
+
+ out, state = gated_delta_update(
+ q,
+ k,
+ v,
+ a,
+ b,
+ self.A_log,
+ self.dt_bias,
+ state,
+ mask,
+ use_kernel=not self.training,
+ )
+
+ if cache is not None:
+ cache[1] = state
+
+ out = self.norm(out, z)
+ out = self.out_proj(out.reshape(B, S, -1))
+
+ if self.sharding_group is not None:
+ out = mx.distributed.all_sum(out, group=self.sharding_group)
+
+ return out
+
+
+class DecoderLayer(nn.Module):
+ def __init__(self, args: TextModelArgs, layer_idx: int):
+ super().__init__()
+ self.is_linear = (layer_idx + 1) % args.full_attention_interval != 0
+ if self.is_linear:
+ self.linear_attn = GatedDeltaNet(args)
+ else:
+ self.self_attn = Attention(args)
+
+ self.input_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)
+ self.post_attention_layernorm = nn.RMSNorm(
+ args.hidden_size, eps=args.rms_norm_eps
+ )
+
+ if args.num_experts > 0:
+ self.mlp = SparseMoeBlock(args)
+ else:
+ self.mlp = MLP(args.hidden_size, args.intermediate_size)
+
+ def __call__(
+ self,
+ x: mx.array,
+ mask: Optional[mx.array] = None,
+ cache: Optional[Any] = None,
+ ) -> mx.array:
+ if self.is_linear:
+ r = self.linear_attn(self.input_layernorm(x), mask, cache)
+ else:
+ r = self.self_attn(self.input_layernorm(x), mask, cache)
+ h = x + r
+ out = h + self.mlp(self.post_attention_layernorm(h))
+ return out
+
+
+class Qwen3_5TextModel(nn.Module):
+ def __init__(self, args: TextModelArgs):
+ super().__init__()
+ self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)
+ self.layers = [
+ DecoderLayer(args=args, layer_idx=i) for i in range(args.num_hidden_layers)
+ ]
+ self.norm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)
+ self.ssm_idx = 0
+ self.fa_idx = args.full_attention_interval - 1
+
+ def __call__(
+ self,
+ inputs: mx.array,
+ cache: Optional[Any] = None,
+ input_embeddings: Optional[mx.array] = None,
+ before_layer_cb=None,
+ after_layer_cb=None,
+ async_eval_per_layer: bool = False,
+ ) -> mx.array:
+ if input_embeddings is not None:
+ hidden_states = input_embeddings
+ else:
+ hidden_states = self.embed_tokens(inputs)
+
+ if cache is None:
+ cache = [None] * len(self.layers)
+
+ fa_mask = create_attention_mask(hidden_states, cache[self.fa_idx])
+ ssm_mask = create_ssm_mask(hidden_states, cache[self.ssm_idx])
+
+ for li, (layer, c) in enumerate(zip(self.layers, cache)):
+ if before_layer_cb is not None:
+ before_layer_cb(li)
+ mask = ssm_mask if layer.is_linear else fa_mask
+ hidden_states = layer(hidden_states, mask=mask, cache=c)
+ if after_layer_cb is not None:
+ after_layer_cb(li, hidden_states)
+ if async_eval_per_layer:
+ # Fire this layer's graph on the GPU while the CPU keeps
+ # building the next layer (overlaps the whole-layer weight
+ # load of the E3b prefill path). The final logits eval
+ # still waits for everything.
+ mx.async_eval(hidden_states)
+
+ return self.norm(hidden_states)
+
+
+class TextModel(nn.Module):
+ def __init__(self, args: TextModelArgs):
+ super().__init__()
+ self.args = args
+ self.model_type = args.model_type
+ self.model = Qwen3_5TextModel(args)
+ if not args.tie_word_embeddings:
+ self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False)
+
+ def __call__(
+ self,
+ inputs: mx.array,
+ cache: Optional[Any] = None,
+ input_embeddings: Optional[mx.array] = None,
+ ) -> mx.array:
+ out = self.model(inputs, cache, input_embeddings=input_embeddings)
+ if self.args.tie_word_embeddings:
+ out = self.model.embed_tokens.as_linear(out)
+ else:
+ out = self.lm_head(out)
+ return out
+
+ @property
+ def layers(self):
+ return self.model.layers
+
+ def make_cache(self):
+ return [ArraysCache(size=2) if l.is_linear else KVCache() for l in self.layers]
+
+ def sanitize(self, weights):
+ has_mtp_weights = any("mtp." in k for k in weights)
+ has_unsanitized_conv1d = any(
+ "conv1d.weight" in k and v.shape[-1] != 1 for k, v in weights.items()
+ )
+ should_shift_norm_weights = has_mtp_weights or has_unsanitized_conv1d
+ weights = {k: v for k, v in weights.items() if "mtp." not in k}
+
+ if self.args.tie_word_embeddings:
+ weights.pop("lm_head.weight", None)
+
+ norm_keys = (
+ ".input_layernorm.weight",
+ ".post_attention_layernorm.weight",
+ "model.norm.weight",
+ ".q_norm.weight",
+ ".k_norm.weight",
+ )
+ for k, v in weights.items():
+ if "conv1d.weight" in k and v.shape[-1] != 1:
+ weights[k] = v.moveaxis(2, 1)
+ if should_shift_norm_weights and any(k.endswith(sfx) for sfx in norm_keys):
+ if v.ndim == 1:
+ weights[k] = v + 1.0
+ return weights
+
+ @property
+ def quant_predicate(self):
+ if self.args.num_experts <= 0:
+ return None
+
+ def predicate(path, _):
+ if path.endswith("mlp.gate") or path.endswith("shared_expert_gate"):
+ return {"group_size": 64, "bits": 8}
+ return True
+
+ return predicate
+
+ @property
+ def cast_predicate(self):
+ def predicate(path: str):
+ if path.endswith("A_log"):
+ return False
+ return True
+
+ return predicate
+
+
+@dataclass
+class ModelArgs(BaseModelArgs):
+ model_type: str
+ text_config: dict
+
+ @classmethod
+ def from_dict(cls, params):
+ if "text_config" not in params:
+ return cls(model_type=params["model_type"], text_config=params)
+ return super().from_dict(params)
+
+
+class Model(nn.Module):
+ def __init__(self, args: ModelArgs):
+ super().__init__()
+ self.args = args
+ self.model_type = args.model_type
+ self.language_model = TextModel(TextModelArgs.from_dict(args.text_config))
+
+ def __call__(
+ self,
+ inputs: mx.array,
+ cache=None,
+ input_embeddings: Optional[mx.array] = None,
+ ):
+ return self.language_model(
+ inputs, cache=cache, input_embeddings=input_embeddings
+ )
+
+ def sanitize(self, weights):
+ sanitized = {}
+ for key, value in weights.items():
+ if key.startswith("vision_tower") or key.startswith("model.visual"):
+ continue
+ if key.startswith("model.visual"):
+ continue
+ if key.startswith("model.language_model"):
+ key = key.replace("model.language_model", "language_model.model")
+ elif key.startswith("language_model."):
+ pass
+ else:
+ key = "language_model." + key
+ sanitized[key] = value
+ return self.language_model.sanitize(sanitized)
+
+ def shard(self, group=None):
+ group = group or mx.distributed.init()
+ N = group.size()
+ rank = group.rank()
+
+ # A sharding factory for the convolution in gated delta net
+ def conv_sharding(key_dim):
+ return lambda p, w: (0, [key_dim, 2 * key_dim])
+
+ def repeat_kv_layer_inplace(layer, h):
+ # No repeat needed cause we have more heads than nodes
+ if N <= h:
+ return
+
+ # Repeat function to apply to the layer weights
+ def _repeat(p):
+ s = p.shape
+ p = p.reshape(h, s[0] // h, *s[1:])
+ p = mx.repeat(p, N // h, axis=0)
+ p = p.reshape(-1, *s[1:])
+ return p
+
+ layer.update(tree_map(_repeat, layer.parameters()))
+
+ for layer in self.layers:
+ # Linear attention
+ if layer.is_linear:
+ kd = layer.linear_attn.key_dim
+ layer.linear_attn.sharding_group = group
+ shard_inplace(layer.linear_attn.conv1d, conv_sharding(kd), group=group)
+ layer.linear_attn.conv1d.groups //= N
+ shard_inplace(
+ layer.linear_attn.in_proj_qkv,
+ "all-to-sharded",
+ segments=[kd, 2 * kd],
+ group=group,
+ )
+ shard_inplace(
+ layer.linear_attn.in_proj_z, "all-to-sharded", group=group
+ )
+ shard_inplace(
+ layer.linear_attn.in_proj_b, "all-to-sharded", group=group
+ )
+ shard_inplace(
+ layer.linear_attn.in_proj_a, "all-to-sharded", group=group
+ )
+ layer.linear_attn.dt_bias = mx.contiguous(
+ mx.split(layer.linear_attn.dt_bias, N)[rank]
+ )
+ layer.linear_attn.A_log = mx.contiguous(
+ mx.split(layer.linear_attn.A_log, N)[rank]
+ )
+ shard_inplace(layer.linear_attn.out_proj, "sharded-to-all", group=group)
+ layer.linear_attn.num_k_heads //= N
+ layer.linear_attn.num_v_heads //= N
+ layer.linear_attn.key_dim //= N
+ layer.linear_attn.value_dim //= N
+ layer.linear_attn.conv_dim //= N
+
+ # Softmax attention
+ else:
+ layer.self_attn.o_proj = shard_linear(
+ layer.self_attn.o_proj, "sharded-to-all", group=group
+ )
+ layer.self_attn.q_proj = shard_linear(
+ layer.self_attn.q_proj, "all-to-sharded", group=group
+ )
+ repeat_kv_layer_inplace(
+ layer.self_attn.k_proj, layer.self_attn.num_key_value_heads
+ )
+ repeat_kv_layer_inplace(
+ layer.self_attn.v_proj, layer.self_attn.num_key_value_heads
+ )
+ layer.self_attn.k_proj = shard_linear(
+ layer.self_attn.k_proj, "all-to-sharded", group=group
+ )
+ layer.self_attn.v_proj = shard_linear(
+ layer.self_attn.v_proj, "all-to-sharded", group=group
+ )
+ layer.self_attn.num_attention_heads //= N
+ layer.self_attn.num_key_value_heads = max(
+ 1, layer.self_attn.num_key_value_heads // N
+ )
+
+ # MLP
+ if isinstance(layer.mlp, MLP):
+ layer.mlp.gate_proj = shard_linear(
+ layer.mlp.gate_proj, "all-to-sharded", group=group
+ )
+ layer.mlp.down_proj = shard_linear(
+ layer.mlp.down_proj, "sharded-to-all", group=group
+ )
+ layer.mlp.up_proj = shard_linear(
+ layer.mlp.up_proj, "all-to-sharded", group=group
+ )
+
+ # MoE
+ else:
+ layer.mlp.sharding_group = group
+ shard_inplace(
+ layer.mlp.shared_expert.gate_proj, "all-to-sharded", group=group
+ )
+ shard_inplace(
+ layer.mlp.shared_expert.down_proj, "sharded-to-all", group=group
+ )
+ shard_inplace(
+ layer.mlp.shared_expert.up_proj, "all-to-sharded", group=group
+ )
+ shard_inplace(
+ layer.mlp.switch_mlp.gate_proj, "all-to-sharded", group=group
+ )
+ shard_inplace(
+ layer.mlp.switch_mlp.down_proj, "sharded-to-all", group=group
+ )
+ shard_inplace(
+ layer.mlp.switch_mlp.up_proj, "all-to-sharded", group=group
+ )
+
+ @property
+ def layers(self):
+ return self.language_model.model.layers
+
+ def make_cache(self):
+ return self.language_model.make_cache()
+
+ @property
+ def quant_predicate(self):
+ return self.language_model.quant_predicate
+
+ @property
+ def cast_predicate(self):
+ return self.language_model.cast_predicate
diff --git a/src/edge0/backends/mlx/_impl/qwen3_5_moe.py b/src/edge0/backends/mlx/_impl/qwen3_5_moe.py
new file mode 100644
index 0000000..7930733
--- /dev/null
+++ b/src/edge0/backends/mlx/_impl/qwen3_5_moe.py
@@ -0,0 +1,56 @@
+# Copyright © 2026 Apple Inc.
+#
+# edge0 vendored copy of mlx-lm 0.31.0 ``mlx_lm/models/qwen3_5_moe.py``,
+# byte-identical EXCEPT the import block (absolute paths; the qwen3_5
+# sibling resolves to this vendored package).
+
+from dataclasses import dataclass
+
+from mlx_lm.models.base import BaseModelArgs
+from edge0.backends.mlx._impl.qwen3_5 import Model as Qwen3_5Model
+
+
+@dataclass
+class ModelArgs(BaseModelArgs):
+ model_type: str
+ text_config: dict
+
+ @classmethod
+ def from_dict(cls, params):
+ if "text_config" not in params:
+ return cls(model_type=params["model_type"], text_config=params)
+ return super().from_dict(params)
+
+
+class Model(Qwen3_5Model):
+
+ def sanitize(self, weights):
+ new_weights = {}
+ for key, value in weights.items():
+ if key.startswith("vision_tower") or key.startswith("model.visual"):
+ continue
+ if key.startswith("model.language_model"):
+ key = key.replace("model.language_model", "language_model.model")
+ elif key.startswith("language_model."):
+ pass
+ else:
+ key = "language_model." + key
+ new_weights[key] = value
+
+ for l in range(self.language_model.args.num_hidden_layers):
+ prefix = f"language_model.model.layers.{l}.mlp"
+ gate_up_key = f"{prefix}.experts.gate_up_proj"
+ if gate_up_key in new_weights:
+ gate_up = new_weights.pop(gate_up_key)
+ mid = gate_up.shape[-2] // 2
+ new_weights[f"{prefix}.switch_mlp.gate_proj.weight"] = gate_up[
+ ..., :mid, :
+ ]
+ new_weights[f"{prefix}.switch_mlp.up_proj.weight"] = gate_up[
+ ..., mid:, :
+ ]
+ new_weights[f"{prefix}.switch_mlp.down_proj.weight"] = new_weights.pop(
+ f"{prefix}.experts.down_proj"
+ )
+
+ return self.language_model.sanitize(new_weights)
diff --git a/src/edge0/backends/mlx/_impl/qwen3_next.py b/src/edge0/backends/mlx/_impl/qwen3_next.py
new file mode 100644
index 0000000..5062513
--- /dev/null
+++ b/src/edge0/backends/mlx/_impl/qwen3_next.py
@@ -0,0 +1,504 @@
+# Copyright © 2025 Apple Inc.
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import partial
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import mlx.core as mx
+import mlx.nn as nn
+from mlx.nn.layers.distributed import sum_gradients
+
+from mlx_lm.models.activations import swiglu
+from mlx_lm.models.base import (
+ BaseModelArgs,
+ create_attention_mask,
+ create_ssm_mask,
+ scaled_dot_product_attention,
+)
+from mlx_lm.models.cache import ArraysCache, KVCache
+from mlx_lm.models.gated_delta import gated_delta_update
+from mlx_lm.models.rope_utils import initialize_rope
+from mlx_lm.models.switch_layers import SwitchGLU
+
+
+@dataclass
+class ModelArgs(BaseModelArgs):
+ model_type: str
+ hidden_size: int
+ num_hidden_layers: int
+ intermediate_size: int
+ num_attention_heads: int
+ linear_num_value_heads: int
+ linear_num_key_heads: int
+ linear_key_head_dim: int
+ linear_value_head_dim: int
+ linear_conv_kernel_dim: int
+ num_experts: int
+ num_experts_per_tok: int
+ decoder_sparse_step: int
+ shared_expert_intermediate_size: int
+ mlp_only_layers: List[int]
+ moe_intermediate_size: int
+ rms_norm_eps: float
+ vocab_size: int
+ num_key_value_heads: int
+ rope_theta: float
+ partial_rotary_factor: float
+ max_position_embeddings: int
+ head_dim: int
+ norm_topk_prob: bool = False
+ tie_word_embeddings: bool = False
+ attention_bias: bool = False
+ rope_scaling: Optional[Dict[str, Union[float, str]]] = None
+ full_attention_interval: int = 4
+
+
+@partial(mx.compile, shapeless=True)
+def _precise_swiglu(h, gate, x):
+ gate = nn.silu(gate.astype(mx.float32))
+ x = x.astype(mx.float32)
+ return (gate * x).astype(h.dtype)
+
+
+class Qwen3NextRMSNormGated(nn.Module):
+ def __init__(self, hidden_size: int, eps: float = 1e-6):
+ super().__init__()
+ self.eps = eps
+ self.weight = mx.ones(hidden_size)
+
+ def __call__(
+ self, hidden_states: mx.array, gate: mx.array | None = None
+ ) -> mx.array:
+ x = mx.fast.rms_norm(hidden_states, self.weight, self.eps)
+ if gate is not None:
+ return _precise_swiglu(hidden_states, gate, x)
+ else:
+ return x.astype(hidden_states.dtype)
+
+
+class Qwen3NextAttention(nn.Module):
+ def __init__(self, args: ModelArgs):
+ super().__init__()
+ self.num_key_value_heads = args.num_key_value_heads
+ self.num_attention_heads = args.num_attention_heads
+ self.head_dim = args.head_dim
+ self.scale = self.head_dim**-0.5
+
+ self.q_proj = nn.Linear(
+ args.hidden_size,
+ self.num_attention_heads * self.head_dim * 2,
+ bias=args.attention_bias,
+ )
+ self.k_proj = nn.Linear(
+ args.hidden_size,
+ self.num_key_value_heads * self.head_dim,
+ bias=args.attention_bias,
+ )
+ self.v_proj = nn.Linear(
+ args.hidden_size,
+ self.num_key_value_heads * self.head_dim,
+ bias=args.attention_bias,
+ )
+ self.o_proj = nn.Linear(
+ self.num_attention_heads * self.head_dim,
+ args.hidden_size,
+ bias=args.attention_bias,
+ )
+
+ self.q_norm = nn.RMSNorm(self.head_dim, eps=args.rms_norm_eps)
+ self.k_norm = nn.RMSNorm(self.head_dim, eps=args.rms_norm_eps)
+
+ self.rope = initialize_rope(
+ int(self.head_dim * args.partial_rotary_factor),
+ base=args.rope_theta,
+ traditional=False,
+ scaling_config=args.rope_scaling,
+ max_position_embeddings=args.max_position_embeddings,
+ )
+
+ def __call__(
+ self,
+ x: mx.array,
+ mask: Optional[mx.array] = None,
+ cache: Optional[Any] = None,
+ ) -> mx.array:
+ B, L, D = x.shape
+
+ q_proj_output = self.q_proj(x)
+ queries, gate = mx.split(
+ q_proj_output.reshape(B, L, self.num_attention_heads, -1), 2, axis=-1
+ )
+ gate = gate.reshape(B, L, -1)
+
+ keys, values = self.k_proj(x), self.v_proj(x)
+
+ queries = self.q_norm(queries).transpose(0, 2, 1, 3)
+ keys = self.k_norm(keys.reshape(B, L, self.num_key_value_heads, -1)).transpose(
+ 0, 2, 1, 3
+ )
+ values = values.reshape(B, L, self.num_key_value_heads, -1).transpose(
+ 0, 2, 1, 3
+ )
+
+ if cache is not None:
+ queries = self.rope(queries, offset=cache.offset)
+ keys = self.rope(keys, offset=cache.offset)
+ keys, values = cache.update_and_fetch(keys, values)
+ else:
+ queries = self.rope(queries)
+ keys = self.rope(keys)
+
+ output = scaled_dot_product_attention(
+ queries, keys, values, cache=cache, scale=self.scale, mask=mask
+ )
+ output = output.transpose(0, 2, 1, 3).reshape(B, L, -1)
+
+ return self.o_proj(output * mx.sigmoid(gate))
+
+
+class Qwen3NextMLP(nn.Module):
+ def __init__(self, dim, hidden_dim):
+ super().__init__()
+ self.gate_proj = nn.Linear(dim, hidden_dim, bias=False)
+ self.down_proj = nn.Linear(hidden_dim, dim, bias=False)
+ self.up_proj = nn.Linear(dim, hidden_dim, bias=False)
+
+ def __call__(self, x) -> mx.array:
+ return self.down_proj(swiglu(self.gate_proj(x), self.up_proj(x)))
+
+
+class Qwen3NextGatedDeltaNet(nn.Module):
+ def __init__(self, config: ModelArgs):
+ super().__init__()
+ self.hidden_size = config.hidden_size
+ self.num_v_heads = config.linear_num_value_heads
+ self.num_k_heads = config.linear_num_key_heads
+ self.head_k_dim = config.linear_key_head_dim
+ self.head_v_dim = config.linear_value_head_dim
+ self.key_dim = self.head_k_dim * self.num_k_heads
+ self.value_dim = self.head_v_dim * self.num_v_heads
+ if self.num_v_heads % self.num_k_heads != 0:
+ raise ValueError(
+ f"num_v_heads ({self.num_v_heads}) must be divisible by num_k_heads ({self.num_k_heads})"
+ )
+
+ self.conv_kernel_size = config.linear_conv_kernel_dim
+ self.layer_norm_epsilon = config.rms_norm_eps
+
+ self.conv_dim = self.key_dim * 2 + self.value_dim
+ self.conv1d = nn.Conv1d(
+ in_channels=self.conv_dim,
+ out_channels=self.conv_dim,
+ bias=False,
+ kernel_size=self.conv_kernel_size,
+ groups=self.conv_dim,
+ padding=0,
+ )
+
+ self.in_proj_qkvz = nn.Linear(
+ self.hidden_size, self.key_dim * 2 + self.value_dim * 2, bias=False
+ )
+ self.in_proj_ba = nn.Linear(self.hidden_size, self.num_v_heads * 2, bias=False)
+
+ self.dt_bias = mx.ones(self.num_v_heads)
+
+ A = mx.random.uniform(low=0, high=16, shape=(self.num_v_heads,))
+ self.A_log = mx.log(A)
+
+ self.norm = Qwen3NextRMSNormGated(self.head_v_dim, eps=self.layer_norm_epsilon)
+
+ self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)
+
+ def fix_query_key_value_ordering(
+ self, mixed_qkvz: mx.array, mixed_ba: mx.array
+ ) -> mx.array:
+ nk, dn, nv, dv = (
+ self.num_k_heads,
+ self.head_k_dim,
+ self.num_v_heads,
+ self.head_v_dim,
+ )
+ mixed_qkvz = mixed_qkvz.reshape(*mixed_qkvz.shape[:-1], nk, -1)
+ mixed_ba = mixed_ba.reshape(*mixed_ba.shape[:-1], nk, -1)
+ q, k, v, z = mx.split(mixed_qkvz, [dn, 2 * dn, 2 * dn + nv // nk * dv], axis=-1)
+ b, a = mx.split(mixed_ba, [nv // nk], axis=-1)
+ return (
+ q,
+ k,
+ v.reshape(*v.shape[:2], -1, dv),
+ z.reshape(*z.shape[:2], -1, dv),
+ b.reshape(*b.shape[:2], nv),
+ a.reshape(*a.shape[:2], nv),
+ )
+
+ def __call__(
+ self,
+ inputs: mx.array,
+ mask: Optional[mx.array] = None,
+ cache: Optional[Any] = None,
+ ) -> mx.array:
+ B, S, _ = inputs.shape
+ q, k, v, z, b, a = self.fix_query_key_value_ordering(
+ self.in_proj_qkvz(inputs), self.in_proj_ba(inputs)
+ )
+
+ if cache is not None and cache[0] is not None:
+ conv_state = cache[0]
+ else:
+ conv_state = mx.zeros(
+ (B, self.conv_kernel_size - 1, self.conv_dim),
+ dtype=inputs.dtype,
+ )
+
+ mixed_qkv = mx.concatenate(
+ [q.reshape(B, S, -1), k.reshape(B, S, -1), v.reshape(B, S, -1)], axis=-1
+ )
+ if mask is not None:
+ mixed_qkv = mx.where(mask[..., None], mixed_qkv, 0)
+ conv_input = mx.concatenate([conv_state, mixed_qkv], axis=1)
+
+ if cache is not None:
+ n_keep = self.conv_kernel_size - 1
+ if cache.lengths is not None:
+ ends = mx.clip(cache.lengths, 0, S)
+ positions = (ends[:, None] + mx.arange(n_keep))[..., None]
+ cache[0] = mx.take_along_axis(conv_input, positions, axis=1)
+ else:
+ cache[0] = conv_input[:, -n_keep:, :]
+
+ conv_out = nn.silu(self.conv1d(conv_input))
+
+ q, k, v = [
+ t.reshape(B, S, h, d)
+ for t, h, d in zip(
+ mx.split(conv_out, [self.key_dim, 2 * self.key_dim], -1),
+ [self.num_k_heads, self.num_k_heads, self.num_v_heads],
+ [self.head_k_dim, self.head_k_dim, self.head_v_dim],
+ )
+ ]
+
+ state = cache[1] if cache else None
+ inv_scale = k.shape[-1] ** -0.5
+ q = (inv_scale**2) * mx.fast.rms_norm(q, None, 1e-6)
+ k = inv_scale * mx.fast.rms_norm(k, None, 1e-6)
+
+ out, state = gated_delta_update(
+ q,
+ k,
+ v,
+ a,
+ b,
+ self.A_log,
+ self.dt_bias,
+ state,
+ mask,
+ use_kernel=not self.training,
+ )
+
+ if cache is not None:
+ cache[1] = state
+ cache.advance(S)
+
+ out = self.norm(out, z)
+ return self.out_proj(out.reshape(B, S, -1))
+
+
+class Qwen3NextSparseMoeBlock(nn.Module):
+ def __init__(self, args: ModelArgs):
+ super().__init__()
+ dim = args.hidden_size
+ intermediate_size = args.moe_intermediate_size
+ shared_expert_intermediate_size = args.shared_expert_intermediate_size
+
+ self.norm_topk_prob = args.norm_topk_prob
+ self.num_experts = num_experts = args.num_experts
+ self.top_k = args.num_experts_per_tok
+
+ self.gate = nn.Linear(dim, num_experts, bias=False)
+ self.switch_mlp = SwitchGLU(dim, intermediate_size, num_experts)
+
+ self.shared_expert = Qwen3NextMLP(dim, shared_expert_intermediate_size)
+ self.shared_expert_gate = nn.Linear(dim, 1, bias=False)
+
+ self.sharding_group = None
+
+ def __call__(
+ self,
+ x: mx.array,
+ ) -> mx.array:
+ if self.sharding_group is not None:
+ x = sum_gradients(self.sharding_group)(x)
+
+ gates = self.gate(x)
+ gates = mx.softmax(gates, axis=-1, precise=True)
+
+ k = self.top_k
+ inds = mx.argpartition(gates, kth=-k, axis=-1)[..., -k:]
+ scores = mx.take_along_axis(gates, inds, axis=-1)
+ if self.norm_topk_prob:
+ scores = scores / scores.sum(axis=-1, keepdims=True)
+
+ y = self.switch_mlp(x, inds)
+ y = (y * scores[..., None]).sum(axis=-2)
+
+ shared_y = self.shared_expert(x)
+ shared_y = mx.sigmoid(self.shared_expert_gate(x)) * shared_y
+
+ y = y + shared_y
+
+ if self.sharding_group is not None:
+ y = mx.distributed.all_sum(y, group=self.sharding_group)
+
+ return y
+
+
+class Qwen3NextDecoderLayer(nn.Module):
+ def __init__(self, args: ModelArgs, layer_idx: int):
+ super().__init__()
+ self.is_linear = (layer_idx + 1) % args.full_attention_interval != 0
+ if self.is_linear:
+ self.linear_attn = Qwen3NextGatedDeltaNet(args)
+ else:
+ self.self_attn = Qwen3NextAttention(args)
+
+ self.input_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)
+ self.post_attention_layernorm = nn.RMSNorm(
+ args.hidden_size, eps=args.rms_norm_eps
+ )
+ if (layer_idx not in args.mlp_only_layers) and (
+ args.num_experts > 0 and (layer_idx + 1) % args.decoder_sparse_step == 0
+ ):
+ self.mlp = Qwen3NextSparseMoeBlock(args)
+ else:
+ self.mlp = Qwen3NextMLP(args.hidden_size, args.intermediate_size)
+
+ def __call__(
+ self,
+ x: mx.array,
+ mask: Optional[mx.array] = None,
+ cache: Optional[Any] = None,
+ ) -> mx.array:
+ if self.is_linear:
+ r = self.linear_attn(self.input_layernorm(x), mask, cache)
+ else:
+ r = self.self_attn(self.input_layernorm(x), mask, cache)
+ h = x + r
+ out = h + self.mlp(self.post_attention_layernorm(h))
+ return out
+
+
+class Qwen3NextModel(nn.Module):
+ def __init__(self, args: ModelArgs):
+ super().__init__()
+ self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)
+ self.layers = [
+ Qwen3NextDecoderLayer(args=args, layer_idx=i)
+ for i in range(args.num_hidden_layers)
+ ]
+ self.norm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)
+ self.ssm_idx = 0
+ self.fa_idx = args.full_attention_interval - 1
+
+ def __call__(
+ self,
+ inputs: mx.array,
+ cache: Optional[Any] = None,
+ before_layer_cb=None,
+ after_layer_cb=None,
+ async_eval_per_layer: bool = False,
+ ) -> mx.array:
+ hidden_states = self.embed_tokens(inputs)
+
+ if cache is None:
+ cache = [None] * len(self.layers)
+
+ fa_mask = create_attention_mask(hidden_states, cache[self.fa_idx])
+ ssm_mask = create_ssm_mask(hidden_states, cache[self.ssm_idx])
+
+ for li, (layer, c) in enumerate(zip(self.layers, cache)):
+ if before_layer_cb is not None:
+ before_layer_cb(li)
+ mask = ssm_mask if layer.is_linear else fa_mask
+ hidden_states = layer(hidden_states, mask=mask, cache=c)
+ if after_layer_cb is not None:
+ after_layer_cb(li, hidden_states)
+ if async_eval_per_layer:
+ # Fire this layer's graph on the GPU while the CPU keeps
+ # building the next layer (overlaps the whole-layer weight
+ # load of the E3b prefill path). The final logits eval
+ # still waits for everything.
+ mx.async_eval(hidden_states)
+
+ return self.norm(hidden_states)
+
+
+class Model(nn.Module):
+ def __init__(self, args: ModelArgs):
+ super().__init__()
+ self.args = args
+ self.model_type = args.model_type
+ self.model = Qwen3NextModel(args)
+ if not args.tie_word_embeddings:
+ self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False)
+
+ def __call__(
+ self,
+ inputs: mx.array,
+ cache: Optional[Any] = None,
+ ) -> mx.array:
+ out = self.model(inputs, cache)
+ if self.args.tie_word_embeddings:
+ out = self.model.embed_tokens.as_linear(out)
+ else:
+ out = self.lm_head(out)
+ return out
+
+ @property
+ def layers(self):
+ return self.model.layers
+
+ def make_cache(self):
+ return [ArraysCache(size=2) if l.is_linear else KVCache() for l in self.layers]
+
+ def sanitize(self, weights):
+ if "model.layers.0.mlp.experts.0.up_proj.weight" not in weights:
+ return weights
+ weights = {key: value for key, value in weights.items() if "mtp." not in key}
+
+ if self.args.tie_word_embeddings:
+ weights.pop("lm_head.weight", None)
+
+ for l in range(self.args.num_hidden_layers):
+ prefix = f"model.layers.{l}.mlp"
+ for n in ["up_proj", "down_proj", "gate_proj"]:
+ to_join = [
+ weights.pop(f"{prefix}.experts.{e}.{n}.weight")
+ for e in range(self.args.num_experts)
+ ]
+ weights[f"{prefix}.switch_mlp.{n}.weight"] = mx.stack(to_join)
+
+ norm_keys = (
+ ".input_layernorm.weight",
+ ".post_attention_layernorm.weight",
+ "model.norm.weight",
+ ".q_norm.weight",
+ ".k_norm.weight",
+ )
+ for k, v in weights.items():
+ if "conv1d.weight" in k and v.shape[-1] != 1:
+ weights[k] = v.moveaxis(2, 1)
+ if any(k.endswith(sfx) for sfx in norm_keys):
+ if v.ndim == 1:
+ weights[k] = v + 1.0
+ return weights
+
+ @property
+ def quant_predicate(self):
+ def predicate(path, _):
+ if path.endswith("mlp.gate") or path.endswith("shared_expert_gate"):
+ return {"group_size": 64, "bits": 8}
+ return True
+
+ return predicate
diff --git a/src/edge0/backends/mlx/backend.py b/src/edge0/backends/mlx/backend.py
new file mode 100644
index 0000000..d4f52b7
--- /dev/null
+++ b/src/edge0/backends/mlx/backend.py
@@ -0,0 +1,25 @@
+"""MLX backend: namespace assembly.
+
+``core`` is mlx.core (arrays, ops, eval/compile, random), ``nn`` is
+mlx.nn (module factory), ``io`` and ``quant`` are edge0 wrappers around
+mlx-lm and mlx's quantized gather kernels.
+"""
+
+from __future__ import annotations
+
+import mlx.core as core # noqa: F401
+import mlx.nn as nn # noqa: F401
+
+from edge0.backends.mlx import io, quant # noqa: F401
+
+
+class BackendImpl:
+ """Handle to the active backend implementation."""
+
+ name = "mlx"
+ description = "MLX / Metal on Apple Silicon"
+
+ @property
+ def version(self) -> str:
+ import mlx
+ return mlx.__version__
diff --git a/src/edge0/backends/mlx/io.py b/src/edge0/backends/mlx/io.py
new file mode 100644
index 0000000..242a4ea
--- /dev/null
+++ b/src/edge0/backends/mlx/io.py
@@ -0,0 +1,155 @@
+"""MLX backend: model / tokenizer / tensor-store loading."""
+
+from __future__ import annotations
+
+import json
+import mmap
+import struct
+
+import numpy as np
+import mlx.core as mx
+from mlx_lm.utils import load_model as _load_model
+
+from edge0.backends.base import TensorStore
+
+
+def load_model(model_path, lazy=True, strict=False, model_config=None,
+ get_model_classes=None):
+ """Load a checkpoint skeleton via mlx-lm.
+
+ ``get_model_classes`` (callable config -> ``(Model, ModelArgs)``)
+ selects a vendored class pair instead of mlx-lm's registry; its
+ ``ModelArgs`` fields can be overridden through ``model_config`` —
+ the hook used by the edge0 model adapters (e.g. prerouter wiring
+ flags). ``lazy=True`` keeps weights as mmap-backed lazy arrays so
+ the skeleton occupies no RAM (only the small non-MoE parts are ever
+ materialized).
+ """
+ from pathlib import Path
+ if not isinstance(model_path, Path):
+ model_path = Path(model_path)
+ return _load_model(model_path, lazy=lazy, strict=strict,
+ model_config=model_config,
+ get_model_classes=get_model_classes)
+
+
+def load_tokenizer(model_path):
+ """Load a HuggingFace tokenizer from a LOCAL directory only.
+
+ mlx-lm 0.31's ``load_tokenizer`` funnels the path through its
+ ``_download`` (hf hub snapshot logic), which prompts for
+ ``trust_remote_code`` when the checkpoint carries an ``auto_map`` —
+ undesirable for a local-only serving stack. The transformers
+ ``AutoTokenizer`` loads a local directory without any hub round trip.
+ """
+ from transformers import AutoTokenizer
+ return AutoTokenizer.from_pretrained(
+ model_path, local_files_only=True, trust_remote_code=True)
+
+
+def open_shards(model_dir: str) -> list:
+ """Open every ``model*.safetensors`` shard as a byte-range mmap
+ (qwen-style multi-shard checkpoints; a single-file checkpoint yields
+ a one-element list)."""
+ import glob
+ import os
+ from edge0.streaming.mmap import SafetensorsMmap
+ shards = []
+ for path in sorted(glob.glob(os.path.join(
+ os.fspath(model_dir), "model*.safetensors"))):
+ shards.append(SafetensorsMmap(path))
+ if not shards:
+ raise FileNotFoundError(
+ f"no model*.safetensors shards under {model_dir}")
+ return shards
+
+
+def load_safetensors(path: str, dtype=None) -> dict[str, "mx.array"]:
+ """Load every tensor of a (small) safetensors file as mx arrays.
+
+ Used for adapter / prerouter weight files; dtype converts (e.g.
+ mx.float16) without copying when it matches the stored dtype.
+ """
+ import mlx.core as mx
+ store = SafeTensorsStore(path)
+ try:
+ out = {}
+ for name in store.keys():
+ arr = store.get(name)
+ if dtype is not None and arr.dtype != dtype:
+ arr = arr.astype(dtype)
+ out[name] = arr
+ return out
+ finally:
+ store.close()
+
+
+class SafeTensorsStore(TensorStore):
+ """mmap-backed safetensors store with per-tensor lazy access.
+
+ Tensors are returned as ``mx.array`` views (zero copy). ``get`` never
+ materializes the whole file.
+ """
+
+ def __init__(self, path: str):
+ import mlx.core as mx
+ self._mx = mx
+ self._path = path
+ with open(path, "rb") as f:
+ header_len = struct.unpack("<Q", f.read(8))[0]
+ header_bytes = f.read(header_len)
+ header = json.loads(header_bytes)
+ self._metadata = header.pop("__metadata__", {})
+ self._entries = {
+ name: {
+ "offset": meta["data_offsets"][0] + 8 + header_len,
+ "size": meta["data_offsets"][1] - meta["data_offsets"][0],
+ "dtype": meta["dtype"],
+ "shape": tuple(meta["shape"]),
+ }
+ for name, meta in header.items()
+ }
+ self._keys = sorted(self._entries)
+ self._file = open(path, "rb")
+ self._mm = mmap.mmap(self._file.fileno(), 0, access=mmap.ACCESS_READ)
+
+ @property
+ def path(self) -> str:
+ return self._path
+
+ def keys(self) -> list[str]:
+ return list(self._keys)
+
+ def metadata(self) -> dict:
+ return dict(self._metadata)
+
+ def get(self, name: str):
+ e = self._entries.get(name)
+ if e is None:
+ raise KeyError(f"{self.path}: no tensor {name!r}")
+ buf = np.frombuffer(self._mm, dtype=np.uint8, count=e["size"],
+ offset=e["offset"])
+ arr = np.frombuffer(buf, dtype=_DTYPES[e["dtype"]],
+ count=int(np.prod(e["shape"])))
+ return self._mx.array(arr.reshape(e["shape"]))
+
+ def close(self):
+ self._mm.close()
+ self._file.close()
+
+ def __del__(self):
+ try:
+ self.close()
+ except Exception:
+ pass
+
+
+# Some numpy builds lack ``np.bfloat16`` (absent on this machine's
+# 2.5.3); mlx has a native bfloat16 we fall back to — the array comes
+# back as an mlx array either way.
+_BF16 = getattr(np, "bfloat16", None) or mx.bfloat16
+_DTYPES = {
+ "F64": np.float64, "F32": np.float32, "F16": np.float16,
+ "BF16": _BF16, "I64": np.int64, "I32": np.int32,
+ "I16": np.int16, "I8": np.int8, "U8": np.uint8, "BOOL": np.bool_,
+}
diff --git a/src/edge0/backends/mlx/quant.py b/src/edge0/backends/mlx/quant.py
new file mode 100644
index 0000000..b1ca11d
--- /dev/null
+++ b/src/edge0/backends/mlx/quant.py
@@ -0,0 +1,26 @@
+"""MLX backend: quantized gather kernels (bit-compatible with mlx-lm)."""
+
+from __future__ import annotations
+
+import mlx.core as mx
+
+
+def gather_qmm(x, w, scales, biases, rhs_indices, transpose=True,
+ group_size=64, bits=4, mode="affine",
+ sorted_indices=False):
+ """Quantized matmul over a gathered subset of expert rows.
+
+ ``w`` is a stacked [num_experts, out, in] weight tensor; rows are
+ gathered by ``rhs_indices`` before dequantizing. Same kernel and
+ defaults as the resident (non-streaming) MoE path, so outputs are
+ bit-identical to a resident ``SwitchGLU``.
+ """
+ return mx.gather_qmm(
+ x, w, scales, biases, rhs_indices=rhs_indices, transpose=transpose,
+ group_size=group_size, bits=bits, mode=mode,
+ sorted_indices=sorted_indices)
+
+
+def swiglu(up: mx.array, gate: mx.array) -> mx.array:
+ """SiLU gated activation: silu(gate) * up."""
+ return mx.nn.silu(gate) * up
diff --git a/src/edge0/cli.py b/src/edge0/cli.py
new file mode 100644
index 0000000..0598d26
--- /dev/null
+++ b/src/edge0/cli.py
@@ -0,0 +1,253 @@
+"""edge0 command-line entry point (``edge0``).
+
+Commands:
+
+ edge0 demo one-command quickstart: find a checkpoint, generate
+ edge0 serve start the HTTP server (one model, queued generations)
+ edge0 chat one-shot prompt -> answer on the terminal
+ edge0 models list registered tiers and their default profiles
+ edge0 convert-adapters one-shot legacy npz -> safetensors migration
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+from edge0 import models # noqa: F401 (populates MODEL_REGISTRY)
+from edge0.registry import MODEL_REGISTRY
+
+# Checkpoint locations the demo command probes when no --model-dir is
+# given (the dev boxes these tiers were validated on).
+DEMO_DEFAULTS = {
+ "edge0-35b": "/Users/linyu/Documents/qwen35-v7-deploy/model",
+ "edge0-10b": "/Users/linyu/Documents/ling-mlx-server/v7-deploy/model",
+}
+
+DEMO_PROMPTS = {
+ "edge0-35b": "Hello! Write one short sentence about Zhuhai.",
+ "edge0-10b": "你好,用一句话介绍珠海。",
+}
+
+
+def cmd_models(args) -> int:
+ for name in sorted(MODEL_REGISTRY):
+ mod = MODEL_REGISTRY[name]
+ cfg = mod.Config.from_pretrained(None) # tier defaults
+ print(f"{name} (port {cfg.port}, target {cfg.target_tok_s} tok/s, "
+ f"peak ≈ {cfg.peak_active_mem_mb:.0f} MB)")
+ print(f" experts={cfg.moe_spec.num_experts} "
+ f"top_k={cfg.moe_spec.top_k} "
+ f"quant={cfg.moe_spec.quant.bits}bit/"
+ f"g{cfg.moe_spec.quant.group_size} "
+ f"staged_n={cfg.options.staged_n} "
+ f"prefill_full={cfg.options.prefill_full_layers} "
+ f"hot={cfg.options.hot_per_layer}")
+ pr = cfg.prerouter
+ if pr is not None:
+ owners = pr.owners
+ owners_txt = (f"owners={owners[0]}..{owners[-1]} ({len(owners)})"
+ if owners else "owners=default")
+ print(f" prerouter: start={pr.start_layer} hidden={pr.hidden} "
+ f"dtype={pr.dtype} K={cfg.prerouter_top_k} {owners_txt} "
+ f"weights={pr.weights_file}")
+ if cfg.lora:
+ print(f" lora: {cfg.lora} (r={cfg.lora_r} alpha={cfg.lora_alpha})")
+ return 0
+
+
+def _engine_kwargs(args) -> dict:
+ """Map CLI flags to from_pretrained overrides (absent key = keep the
+ tier default; explicit None/' ' disables)."""
+ kw: dict = {}
+ if getattr(args, "no_prerouter", False):
+ kw["prerouter"] = None
+ if getattr(args, "no_lora", False):
+ kw["lora"] = ""
+ return kw
+
+
+def _resolve_model(args) -> tuple[str | None, str | None]:
+ """vLLM-style model argument resolution.
+
+ ``edge0 serve <model>`` accepts either a registered tier name
+ (``edge0-35b`` / ``edge0-10b`` — checkpoint found via the matching
+ env var, else the known dev-box default) or a checkpoint path
+ (``edge0 serve /root/edge0-35b`` — tier auto-detected from the
+ checkpoint's ``config.json``).
+
+ Returns ``(model_dir, name)``; either may stay None to keep the
+ legacy ``--model-dir`` / ``--name`` behaviour.
+ """
+ model = getattr(args, "model", None)
+ if not model:
+ return args.model_dir, args.name
+ if model in MODEL_REGISTRY:
+ short = (model.split("-", 1)[-1] if model.startswith("edge0-")
+ else model)
+ env = os.environ.get(f"EDGE0_{short.upper()}_MODEL")
+ return env or DEMO_DEFAULTS.get(model), model
+ return model, None # a path: tier auto-detected by the registry
+
+
+def cmd_demo(args) -> int:
+ from edge0 import AutoEngine
+ from edge0.server.chat import ChatMessage, ChatRequest, ChatSession
+
+ model_dir, name = _resolve_model(args)
+ if not model_dir:
+ # No explicit model: probe the known dev-box checkpoints.
+ for d in DEMO_DEFAULTS.values():
+ if os.path.isdir(d):
+ model_dir = d
+ break
+ if not model_dir or not os.path.isdir(model_dir):
+ print(
+ f"[edge0] no checkpoint for {name or 'model'}.\n"
+ "Pass the model explicitly, vLLM-style:\n"
+ f" edge0 demo {DEMO_DEFAULTS.get(name, '<checkpoint-dir>')}\n"
+ "or point at your own checkpoint:\n"
+ " edge0 demo /path/to/qwen35/model\n"
+ "Tier names are edge0-35b / edge0-10b (env EDGE0_<TIER>_MODEL "
+ "overrides the default search path).",
+ file=sys.stderr,
+ )
+ return 2
+ engine = AutoEngine.from_pretrained(model_dir, name=name,
+ **_engine_kwargs(args))
+ tok = engine._tok
+ if tok is None:
+ engine.close()
+ raise SystemExit("model has no tokenizer; cannot demo")
+ prompt = args.prompt or DEMO_PROMPTS.get(engine.name, "Hello!")
+ req = ChatRequest(model=engine.name, messages=[
+ ChatMessage(role="user", content=prompt)])
+ sess = ChatSession(engine, req)
+ tokens, meta = sess.run()
+ print(f"user : {prompt}")
+ print(f"edge0: {tok.decode(tokens)}")
+ print(f"# {len(tokens)} tokens in {meta['wall_s']}s",
+ file=sys.stderr)
+ engine.close()
+ return 0
+
+
+def cmd_chat(args) -> int:
+ from edge0 import AutoEngine
+
+ model_dir, name = _resolve_model(args)
+ if not model_dir or not os.path.isdir(model_dir):
+ raise SystemExit(
+ "chat requires a checkpoint; pass it as the model argument "
+ "(edge0 chat /path/to/model) or --model-dir")
+ engine = AutoEngine.from_pretrained(model_dir, name=name,
+ **_engine_kwargs(args))
+ tok = engine._tok
+ if tok is None:
+ raise SystemExit("model has no tokenizer; cannot chat")
+ if args.prompt:
+ prompts = [args.prompt]
+ elif not sys.stdin.isatty():
+ prompts = [line.rstrip("\n") for line in sys.stdin if line.strip()]
+ else:
+ raise SystemExit("pass --prompt or pipe input on stdin")
+ for p in prompts:
+ from edge0.server.chat import ChatMessage, ChatRequest, ChatSession
+ req = ChatRequest(model=name, messages=[
+ ChatMessage(role="user", content=p)])
+ sess = ChatSession(engine, req)
+ tokens, meta = sess.run()
+ print(tok.decode(tokens))
+ print(f"# {len(tokens)} tokens in {meta['wall_s']}s", file=sys.stderr)
+ engine.close()
+ return 0
+
+
+def cmd_serve(args) -> int:
+ from edge0 import AutoEngine
+ from edge0.server import QueueServer, run_server
+
+ model_dir, name = _resolve_model(args)
+ if not model_dir or not os.path.isdir(model_dir):
+ raise SystemExit(
+ "serve requires a checkpoint; pass it as the model argument "
+ "(edge0 serve /path/to/model) or --model-dir")
+ engine = AutoEngine.from_pretrained(model_dir, name=name,
+ **_engine_kwargs(args))
+ server = QueueServer(engine, model_name=name or engine.name)
+ print(f"[edge0] serving {server.model_name} on http://{args.host}:{args.port} "
+ f"(stream={'flask' if args.flask else 'stdlib'})",
+ file=sys.stderr)
+ run_server(server, host=args.host, port=args.port, use_flask=args.flask)
+ return 0
+
+
+def cmd_convert(args) -> int:
+ import runpy
+ sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve()
+ .parents[1] / "scripts"))
+ runpy.run_path(
+ str(__import__("pathlib").Path(__file__).resolve().parents[1]
+ / "scripts" / "convert_adapters_legacy.py"),
+ run_name="__main__",
+ )
+ return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+ ap = argparse.ArgumentParser(prog="edge0", description=__doc__)
+ sub = ap.add_subparsers(dest="cmd", required=True)
+
+ p = sub.add_parser("models", help="list registered model tiers")
+ p.set_defaults(fn=cmd_models)
+
+ p = sub.add_parser(
+ "demo",
+ help="one-command quickstart (vLLM-style model argument)")
+ p.add_argument("model", nargs="?", default=None,
+ help="tier name (edge0-35b) or checkpoint dir")
+ p.add_argument("--model-dir", default=None)
+ p.add_argument("--name", default=None)
+ p.add_argument("--prompt", default=None)
+ p.add_argument("--no-prerouter", action="store_true")
+ p.add_argument("--no-lora", action="store_true")
+ p.set_defaults(fn=cmd_demo)
+
+ p = sub.add_parser(
+ "chat",
+ help="one-shot prompt answering (vLLM-style model argument)")
+ p.add_argument("model", nargs="?", default=None,
+ help="tier name (edge0-35b) or checkpoint dir")
+ p.add_argument("--model-dir", default=None)
+ p.add_argument("--name", default=None)
+ p.add_argument("--prompt", default=None)
+ p.add_argument("--no-prerouter", action="store_true")
+ p.add_argument("--no-lora", action="store_true")
+ p.set_defaults(fn=cmd_chat)
+
+ p = sub.add_parser(
+ "serve",
+ help="run the HTTP server, vLLM-style: edge0 serve <model>")
+ p.add_argument("model", nargs="?", default=None,
+ help="tier name (edge0-35b) or checkpoint dir")
+ p.add_argument("--model-dir", default=None)
+ p.add_argument("--name", default=None)
+ p.add_argument("--host", default="127.0.0.1")
+ p.add_argument("--port", type=int, default=8000)
+ p.add_argument("--flask", action="store_true",
+ help="use the Flask transport (needs flask installed)")
+ p.add_argument("--no-prerouter", action="store_true")
+ p.add_argument("--no-lora", action="store_true")
+ p.set_defaults(fn=cmd_serve)
+
+ p = sub.add_parser("convert-adapters",
+ help="one-shot legacy npz -> safetensors migration")
+ p.set_defaults(fn=cmd_convert)
+
+ args = ap.parse_args(argv)
+ return args.fn(args)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/src/edge0/config.py b/src/edge0/config.py
new file mode 100644
index 0000000..f7cd6e3
--- /dev/null
+++ b/src/edge0/config.py
@@ -0,0 +1,37 @@
+"""Generation / sampling configuration (transformers-style)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class GenerationConfig:
+ """Sampling knobs for one model tier.
+
+ Attributes:
+ temperature: softmax temperature (0 disables).
+ top_p: nucleus truncation (<1.0 enables).
+ top_k: top-k truncation (>0 enables).
+ repetition_penalty: HF-style penalty applied to the history of
+ generated tokens.
+ max_new_tokens: default cap for generated tokens.
+ eos_ids: token ids that end generation.
+ seed: optional RNG seed.
+ first_token_greedy: always argmax the FIRST generated token
+ (qwen35-v7-deploy serve.py / THINK_FIX_20260907.md pattern):
+ after the think opener a randomly-sampled first token can
+ derail the whole block into '!' loops.
+ """
+
+ temperature: float = 0.7
+ top_p: float = 0.95
+ top_k: int = 64
+ repetition_penalty: float = 1.0
+ max_new_tokens: int = 512
+ eos_ids: tuple[int, ...] = ()
+ seed: int | None = None
+ first_token_greedy: bool = True
+
+ def is_eos(self, token_id: int) -> bool:
+ return token_id in self.eos_ids
diff --git a/src/edge0/engine/__init__.py b/src/edge0/engine/__init__.py
new file mode 100644
index 0000000..56a6161
--- /dev/null
+++ b/src/edge0/engine/__init__.py
@@ -0,0 +1,25 @@
+"""edge0 engines: streaming prefill/decode lifecycles per model family.
+
+``edge0.engine.base.Edge0Engine`` owns the shared prefill / step /
+generate loops; ``edge0.engine.qwen.Qwen35Engine`` and
+``edge0.engine.ling.Ling10BEngine`` implement the family hooks
+(streaming install, prerouter wiring, step-boundary staging).
+"""
+
+from edge0.engine.base import Edge0Engine
+from edge0.engine.hooks import (
+ make_history_prefetch,
+ make_intra_after_layer,
+ make_prefill_before_layer,
+)
+from edge0.engine.ling import Ling10BEngine
+from edge0.engine.qwen import Qwen35Engine
+
+__all__ = [
+ "Edge0Engine",
+ "Qwen35Engine",
+ "Ling10BEngine",
+ "make_history_prefetch",
+ "make_intra_after_layer",
+ "make_prefill_before_layer",
+]
diff --git a/src/edge0/engine/base.py b/src/edge0/engine/base.py
new file mode 100644
index 0000000..d13a22e
--- /dev/null
+++ b/src/edge0/engine/base.py
@@ -0,0 +1,177 @@
+"""Edge0 engine base: lifecycle + shared prefill / decode loops.
+
+Family engines (``edge0.engine.qwen`` / ``edge0.engine.ling``) implement
+the model-specific hooks; everything here is model-agnostic:
+
+* ``prefill`` — chunked prompt processing; the family hook decides
+ whole-layer (E3b) vs on-demand paths per chunk, and ``_prefill_end``
+ drops the prefill working set, refreshes hot pins and primes the
+ staged slots for the first decode step.
+* ``step`` — one-token forward + the family's step-boundary staging
+ (early submit for router layers, ``sync_actuals``, prerouter
+ ``stage_all`` + state swap).
+* ``generate`` — sampling loop with the family ``GenerationConfig``.
+
+Loop mechanics ported from the deployment engines (``engine_qwen.py`` /
+ling ``engine.py``) so the measured production behavior — one cheap sync
+point per step, fills overlapping the next forward — is preserved.
+"""
+
+from __future__ import annotations
+
+import time
+
+from edge0.backends import core
+
+from edge0.config import GenerationConfig
+from edge0.sampling import sample
+
+
+class Edge0Engine:
+ """Base streaming engine. Subclasses fill in the family hooks."""
+
+ name = "edge0"
+
+ def __init__(self, model_dir: str, cfg, tokenizer=None):
+ self.dir = model_dir
+ self.cfg = cfg
+ self._tok = tokenizer
+ self.prefill_chunk = getattr(cfg, "prefill_chunk", 2048)
+ self.pos = 0
+ self._prefill_active = False
+ self._last_logits = None
+ self._cap_mlx_cache()
+ t0 = time.time()
+ self._build()
+ print(f"[{self.name}] built in {time.time() - t0:.1f}s", flush=True)
+
+ # ---- family hooks -----------------------------------------------------
+
+ def _build(self):
+ raise NotImplementedError
+
+ def _forward(self, ids, intra_stage: bool = True) -> core.array:
+ raise NotImplementedError
+
+ def _step_pre(self, token_id: int) -> None:
+ """Called before a decode forward (early-submit fills)."""
+
+ def _step_post(self, token_id: int) -> None:
+ """Called after a decode forward's logits eval (staging/swap)."""
+
+ def _prefill_end(self) -> None:
+ """Called after the last prefill chunk."""
+
+ def _reset_state(self) -> None:
+ pass
+
+ def _lm_logits(self, h: core.array) -> core.array:
+ raise NotImplementedError
+
+ # ---- shared loops -----------------------------------------------------
+
+ def prefill(self, token_ids, chunk_size=None, on_progress=None):
+ if chunk_size is None:
+ chunk_size = self.prefill_chunk
+ total = len(token_ids)
+ done = 0
+ self._prefill_active = True
+ try:
+ for start in range(0, total, chunk_size):
+ chunk = token_ids[start:start + chunk_size]
+ self._last_logits = self._forward(chunk)
+ self.pos += len(chunk)
+ done += len(chunk)
+ if on_progress is not None:
+ on_progress(done, total)
+ finally:
+ self._prefill_active = False
+ self._prefill_end()
+ return total
+
+ def next_logits(self) -> core.array:
+ return self._last_logits
+
+ def step(self, token_id: int) -> core.array:
+ self._step_pre(token_id)
+ logits = self._forward([token_id])
+ self.pos += 1
+ self._step_post(token_id)
+ return logits
+
+ def generate(self, token_ids, gen_config: GenerationConfig | None = None,
+ max_new_tokens: int | None = None, on_token=None) -> list[int]:
+ """Prefill ``token_ids`` then sample until EOS or the cap."""
+ if gen_config is None:
+ gen_config = getattr(self.cfg, "gen", GenerationConfig())
+ if max_new_tokens is None:
+ max_new_tokens = gen_config.max_new_tokens
+ if len(token_ids) > 1:
+ self.prefill(token_ids)
+ elif len(token_ids) == 1:
+ self._last_logits = self._forward(token_ids)
+ self.pos += 1
+ out: list[int] = []
+ history = list(token_ids)
+ logits = self.next_logits()
+ first_token = True
+ for _ in range(max_new_tokens):
+ if logits is None:
+ raise RuntimeError("no logits available before first step")
+ if first_token and gen_config.first_token_greedy:
+ # The FIRST token is always greedy (qwen35-v7-deploy
+ # serve.py pattern, THINK_FIX_20260907.md §3.2): after the
+ # think opener a randomly-sampled first token can derail
+ # the whole block into '!' loops.
+ tid = int(core.argmax(logits, axis=-1).item())
+ else:
+ tid = sample(
+ logits,
+ temperature=gen_config.temperature,
+ top_k=gen_config.top_k,
+ top_p=gen_config.top_p,
+ repetition_penalty=gen_config.repetition_penalty,
+ history=history,
+ seed=gen_config.seed,
+ )
+ first_token = False
+ if gen_config.is_eos(tid):
+ break
+ out.append(tid)
+ history.append(tid)
+ # The deployment's step() returns the next logits and does NOT
+ # update _last_logits (engine_qwen.py:471-474), so the loop
+ # must carry the step return value — re-reading next_logits()
+ # here would replay the prefill-final logits forever (the
+ # greedy repetition bug: [353]*8 instead of [353, 2688, ...]).
+ logits = self.step(tid)
+ if on_token is not None:
+ on_token(tid)
+ return out
+
+ def reset(self):
+ self._reset_state()
+ self.pos = 0
+ self._last_logits = None
+
+ def stats(self) -> dict:
+ layers = getattr(self, "_all_stream_layers", {})
+ return {li: exp.stats() for li, exp in layers.items()}
+
+ def close(self):
+ for exp in getattr(self, "_all_stream_layers", {}).values():
+ try:
+ exp.close()
+ except Exception: # noqa: BLE001 — best-effort shutdown
+ pass
+
+ # ---- misc -------------------------------------------------------------
+
+ @staticmethod
+ def _cap_mlx_cache():
+ import os
+ try:
+ mb = int(os.environ.get("MLX_CACHE_LIMIT_MB", "256"))
+ core.set_cache_limit(mb * 1024 * 1024)
+ except Exception: # noqa: BLE001 — best-effort
+ pass
diff --git a/src/edge0/engine/hooks.py b/src/edge0/engine/hooks.py
new file mode 100644
index 0000000..c075dc5
--- /dev/null
+++ b/src/edge0/engine/hooks.py
@@ -0,0 +1,87 @@
+"""Prefill / decode callback factories.
+
+All callbacks are ports of the deployment engines' proven hooks
+(``engine_qwen.py`` / ling ``engine.py``); each is a pure function
+building a callback from a ``dict[int, StreamingSwitchGLU]`` so the
+same machinery serves both model families.
+"""
+
+from __future__ import annotations
+
+
+def make_prefill_before_layer(all_stream_layers, *, full_n: int = 0,
+ hot_n: int = 0, hot_window: int = 4):
+ """Before-layer prefill hook: whole-layer load-drop (E3b) + the
+ sliding hot-expert window.
+
+ Before layer ``li`` runs: drop layer ``li-1``'s whole-layer set (its
+ async_eval was already submitted, so the GPU queue keeps the arrays
+ alive until evaluated), then either load layer ``li``'s full stacked
+ tensors (``full_n`` unset / ``li < full_n``) or build the hot-stack
+ window (``hot_n``): numpy backing for layers ``li..li+ahead``,
+ GPU materialization for the window, dematerialize trailing layers.
+ """
+ w = max(1, hot_window)
+ ahead = max(1, w // 2)
+
+ def before_layer(li: int) -> None:
+ exp_old = all_stream_layers.get(li - 1)
+ if exp_old is not None:
+ exp_old.clear_full_layer()
+ if full_n and li >= full_n:
+ return
+ if hot_n:
+ for lj in range(li, min(li + ahead + 1,
+ len(all_stream_layers))):
+ exp_w = all_stream_layers.get(lj)
+ if exp_w is None:
+ continue
+ exp_w.load_hot_layer(hot_n) # numpy backing (cheap)
+ exp_w.materialize_hot() # GPU window
+ for lj in range(0, max(0, li - (w - ahead))):
+ exp_t = all_stream_layers.get(lj)
+ if exp_t is not None:
+ exp_t.dematerialize_hot()
+ if not full_n:
+ return
+ exp = all_stream_layers.get(li)
+ if exp is not None:
+ exp.load_full_layer()
+
+ return before_layer
+
+
+def make_intra_after_layer(all_stream_layers, enabled: bool):
+ """Intra-step staging: right after layer ``li`` runs, submit the next
+ step's layer-``li`` fill from ``last_used`` (the router's actuals from
+ the previous step, already on the CPU — no mid-forward tolist, which
+ segfaults in mlx 0.30.4 when a pool thread forces a GPU sync while the
+ main thread is still building the graph)."""
+ if not enabled:
+ return None
+
+ def after_layer(li: int, hidden_states=None) -> None:
+ exp = all_stream_layers.get(li)
+ if exp is None or not exp.last_used:
+ return
+ exp.stage_experts(list(exp.last_used))
+
+ return after_layer
+
+
+def make_history_prefetch(all_stream_layers, enabled: bool):
+ """Zero-cost decode prefetch: submit each layer's previous-token
+ expert set (``last_used``) so the SSD loads overlap this step's GPU
+ compute. The no-prerouter replacement for next-token prediction —
+ exploits adjacent-token expert locality (~0.4 overlap measured)."""
+ if not enabled:
+ return None
+
+ def history_prefetch() -> None:
+ for li in range(1, len(all_stream_layers)):
+ e = all_stream_layers.get(li)
+ if e is None or not e.last_used:
+ continue
+ e.prefetch(list(e.last_used))
+
+ return history_prefetch
diff --git a/src/edge0/engine/ling.py b/src/edge0/engine/ling.py
new file mode 100644
index 0000000..b0f3936
--- /dev/null
+++ b/src/edge0/engine/ling.py
@@ -0,0 +1,246 @@
+"""edge0-10b engine: Ling 3.0 hybrid (MLA + MoE, 10B tier).
+
+Port of the deployment's ling ``engine.py`` hybrid-prerouter path:
+
+* the vendored ``bailing_hybrid.py`` model already owns its prerouter
+ heads and consumes ``prerouter_cache`` logits inside
+ ``BailingSparseMoE``'s ``_select_from_logits`` (sigmoid + group-limited
+ top-k, routed-scaling 2.5) — the engine only feeds the cache and
+ submits the staged fills.
+* step boundary: ONE stacked head batch -> ONE tolist -> per-consumer
+ ``stage_experts`` + ``pg_cache[consumer]`` logits; the next forward's
+ consuming MoE blocks re-select from those cached logits, so staged
+ set == routing set (zero drops) by construction.
+* prefill — E3b whole-layer load-drop (``prefill_full_layers``).
+"""
+
+from __future__ import annotations
+
+import os
+from types import SimpleNamespace
+
+from edge0.backends import core
+
+from edge0.backends.mlx._impl.bailing_hybrid import Model as BailingModel
+from edge0.backends.mlx._impl.bailing_hybrid import ModelArgs as BailingArgs
+from edge0.backends import io
+from edge0.backends.mlx.io import load_model, load_tokenizer
+from edge0.engine.base import Edge0Engine
+from edge0.engine.hooks import (
+ make_history_prefetch,
+ make_prefill_before_layer,
+)
+from edge0.prerouter.install import install_prerouter
+from edge0.prerouter.stager import LingPrerouterStager
+from edge0.streaming.install import install_streaming_experts
+from edge0.streaming.mmap import SafetensorsMmap
+
+
+def _get_model_classes(config):
+ """mlx-lm class hook: serve the vendored bailing backbone."""
+ return BailingModel, BailingArgs
+
+
+def load_installed(model_dir: str, cfg):
+ """Load the ling skeleton and install streaming twins, LoRA and the
+ hybrid prerouter weights (shared by ``build_model``/``build_engine``).
+
+ Returns ``(model, model_config, shards, installs)``.
+ """
+ model, model_config = load_model(
+ model_dir, lazy=True, strict=False,
+ model_config={"model_type": "bailing_hybrid",
+ "prerouter_enabled": cfg.prerouter is not None,
+ "prerouter_start_layer":
+ getattr(cfg.prerouter, "start_layer", 7),
+ "prerouter_hidden":
+ getattr(cfg.prerouter, "hidden", 512)},
+ get_model_classes=_get_model_classes)
+ shards = [SafetensorsMmap(os.path.join(
+ os.fspath(model_dir), "model.safetensors"))]
+ spec = cfg.moe_spec
+ opts = cfg.options
+ n_layers = model_config["num_hidden_layers"]
+ installed = install_streaming_experts(
+ model, shards, spec, options=opts, num_layers=n_layers)
+ all_stream = {li: t for li, t in enumerate(installed) if t is not None}
+ stream_layers = {li: t for li, t in all_stream.items() if t._staged_mode}
+
+ if cfg.lora:
+ from edge0.adapters.lora import install_lora
+ install_lora(model, cfg.lora, r=cfg.lora_r, alpha=cfg.lora_alpha)
+
+ pg_state = pg_stager = None
+ if cfg.prerouter and cfg.prerouter.weights_file:
+ pg_state, heads = install_prerouter(
+ model=model, spec=spec, pspec=cfg.prerouter, n_layers=n_layers)
+ pg_stager = LingPrerouterStager(
+ model=model, spec=spec, pspec=cfg.prerouter,
+ state=pg_state, stream_layers=stream_layers,
+ top_k=cfg.prerouter_top_k)
+ print(f"[edge0-10b] prerouter installed: {len(heads)} heads, "
+ f"start={cfg.prerouter.start_layer}, "
+ f"K={cfg.prerouter_top_k}", flush=True)
+ installs = dict(all_stream_layers=all_stream,
+ stream_layers=stream_layers,
+ pg_state=pg_state, pg_stager=pg_stager)
+ return model, model_config, shards, installs
+
+
+class Ling10BEngine(Edge0Engine):
+ """Streaming Ling-3.0 engine (staged decode + hybrid prerouter)."""
+
+ name = "edge0-10b"
+
+ def __init__(self, model_dir: str, cfg, tokenizer=None,
+ think: bool = False):
+ # Deployment parity: the ling server's THINK_MODE knob becomes the
+ # engine's ``think`` flag (start_server.sh exports THINK_MODE=0).
+ self.think = think
+ self._chat_tpl = None
+ # NAN_BANG_COLLAPSE_FIX.md parity: hidden clip default 1000 unless
+ # the deployer overrides LING_HIDDEN_CLIP explicitly. One fp16
+ # overflow inside a layer otherwise poisons the whole net into
+ # all-NaN logits -> argmax fallback token 0 ('!') death spiral.
+ if "LING_HIDDEN_CLIP" not in os.environ:
+ os.environ["LING_HIDDEN_CLIP"] = "1000"
+ super().__init__(model_dir, cfg, tokenizer=tokenizer)
+
+ def _build(self):
+ cfg = self.cfg
+ (self.model, self.model_config, self.shards,
+ inst) = load_installed(cfg.model_dir, cfg)
+ self._all_stream_layers = inst["all_stream_layers"]
+ self._stream_layers = inst["stream_layers"]
+ self._pg_state = inst["pg_state"]
+ self._pg_stager = inst["pg_stager"]
+ opts = cfg.options
+ if self._tok is None:
+ try:
+ self._tok = load_tokenizer(cfg.model_dir)
+ except Exception: # noqa: BLE001 — tokenizer optional for CLI
+ pass
+
+ self._prefill_before_layer = make_prefill_before_layer(
+ self._all_stream_layers,
+ full_n=getattr(opts, "prefill_full_layers", 0),
+ hot_n=0, hot_window=1)
+ self._history_prefetch = make_history_prefetch(
+ self._all_stream_layers, enabled=cfg.prefetch_history)
+
+ self.cache = self.model.make_cache()
+
+ # ---- forward ----------------------------------------------------------
+
+ def _forward(self, ids, intra_stage: bool = True) -> core.array:
+ inputs = core.array(ids)[None, :]
+ opts = self.cfg.options
+ prefill_multi = len(ids) > 1 and self._prefill_active
+ full_layer = bool(
+ prefill_multi and self._all_stream_layers
+ and opts.full_layer_prefill)
+ h = self.model.model(
+ inputs, cache=self.cache,
+ before_layer_cb=self._prefill_before_layer if prefill_multi
+ else None,
+ after_layer_cb=None,
+ async_eval_per_layer=bool(prefill_multi and full_layer),
+ prerouter_cache=(self._pg_stager.pg_cache
+ if self._pg_stager is not None else None))
+ logits = self.model.lm_head(h[0, -1])
+ core.eval(logits)
+ # Step boundary: ONE stacked head batch -> ONE tolist -> fills.
+ if (self._pg_stager is not None and not self._prefill_active
+ and len(ids) == 1):
+ self._pg_stager.stage_all()
+ return logits
+
+ # ---- step hooks -------------------------------------------------------
+
+ def _step_pre(self, token_id: int) -> None:
+ if self._pg_stager is None and self._history_prefetch is not None:
+ self._history_prefetch()
+
+ def _step_post(self, token_id: int) -> None:
+ for exp in self._stream_layers.values():
+ exp.sync_actuals()
+ if self._pg_stager is None:
+ # History staging: each layer's next set is its previous
+ # actuals (adjacent-token expert locality).
+ for exp in self._stream_layers.values():
+ if exp.last_used:
+ exp.stage_experts(list(exp.last_used))
+ exp.swap_staged()
+
+ # ---- prefill / reset --------------------------------------------------
+
+ def _prefill_end(self) -> None:
+ for exp in self._all_stream_layers.values():
+ exp.clear_full_layer()
+ if self._pg_stager is not None:
+ # Deployment parity: prefill's tail stages the FIRST decode
+ # token's expert sets (engine.py:919 calls stage_all right
+ # after prefill). Without this the first decode step has no
+ # prerouter logits, falls back to the true gate for one step,
+ # and the whole prediction chain shifts by one token
+ # (observed: [23982, 2862, ...] vs correct [23982, 4264, ...]).
+ self._pg_stager.stage_all()
+ if self._stream_layers:
+ for exp in self._stream_layers.values():
+ exp.stage_from_prefill()
+
+ def _reset_state(self) -> None:
+ self.cache = self.model.make_cache()
+ if self._pg_stager is not None:
+ self._pg_stager.reset()
+ self._pg_state.reset()
+
+ def _lm_logits(self, h: core.array) -> core.array:
+ return self.model.lm_head(h[0, -1])
+
+ # ---- chat template (deployment parity) -------------------------------
+
+ def _chat_template(self):
+ """Render the deployment's ``chat_template.jinja`` (same file and
+ renderer as the ling server's ``encode_chat``, engine.py)."""
+ from jinja2 import BaseLoader, Environment, StrictUndefined
+
+ if self._chat_tpl is None:
+ src = open(
+ os.path.join(self.dir, "chat_template.jinja"),
+ encoding="utf-8",
+ ).read()
+ self._chat_tpl = Environment(
+ loader=BaseLoader(), undefined=StrictUndefined,
+ autoescape=False,
+ ).from_string(src)
+ return self._chat_tpl
+
+ def encode_chat(self, messages, think=None) -> list:
+ """Tokenize chat messages with the deployment chat template.
+
+ ``think`` mirrors THINK_MODE: True renders "detailed thinking on"
+ (the model answers with a reasoning preamble), False renders
+ "detailed thinking off" (direct answer). Defaults to the
+ engine's ``think`` flag.
+ """
+ if think is None:
+ think = self.think
+ msgs = [
+ SimpleNamespace(
+ role=m.get("role"),
+ content=m.get("content") or "",
+ reasoning_content=m.get("reasoning_content") or "",
+ tool_calls=m.get("tool_calls"),
+ )
+ for m in messages
+ ]
+ text = self._chat_template().render(
+ messages=msgs,
+ add_generation_prompt=True,
+ enable_thinking=bool(think),
+ tools=None,
+ )
+ # transformers encode would prepend/append special tokens by
+ # default; the template text is already complete.
+ return self._tok.encode(text, add_special_tokens=False)
diff --git a/src/edge0/engine/qwen.py b/src/edge0/engine/qwen.py
new file mode 100644
index 0000000..75286cd
--- /dev/null
+++ b/src/edge0/engine/qwen.py
@@ -0,0 +1,189 @@
+"""edge0-35b engine: Qwen3.5-MoE (K=4 tier).
+
+Port of the deployment's ``engine_qwen.py`` trained-prerouter path:
+
+* prefill — E3b whole-layer load-drop (with the sliding hot-expert
+ window), driven by the callback hooks added to the vendored
+ ``qwen3_next.py``.
+* decode — the prerouter head (class-level block patch) supplies the
+ routing; at the step boundary ONE stacked head batch is evaluated,
+ ONE tolist extracts every layer's next expert set, and the staged
+ fills overlap the next forward (0 drops: staged set == routing set).
+* router layers (< start, >= n-1) are staged from their actuals.
+"""
+
+from __future__ import annotations
+
+from edge0.backends import core
+
+from edge0.backends.mlx._impl.qwen3_5_moe import Model as Qwen35Model
+from edge0.backends.mlx._impl.qwen3_5_moe import ModelArgs as Qwen35Args
+from edge0.backends import io
+from edge0.backends.mlx.io import load_model, load_tokenizer, open_shards
+from edge0.engine.base import Edge0Engine
+from edge0.engine.hooks import (
+ make_history_prefetch,
+ make_intra_after_layer,
+ make_prefill_before_layer,
+)
+from edge0.prerouter.install import install_prerouter
+from edge0.prerouter.stager import CrossTokenStager
+from edge0.streaming.install import install_streaming_experts
+
+
+def _get_model_classes(config):
+ """mlx-lm class hook: serve the vendored qwen3_5_moe backbone."""
+ return Qwen35Model, Qwen35Args
+
+
+def load_installed(model_dir: str, cfg):
+ """Load the qwen skeleton and install streaming twins, LoRA and the
+ trained prerouter (shared by ``build_model`` and ``build_engine``).
+
+ Returns ``(model, model_config, shards, installs)`` where
+ ``installs`` carries the layer maps and prerouter state the engine
+ drives at the step boundary.
+ """
+ model, model_config = load_model(
+ model_dir, lazy=True, strict=False,
+ model_config={"model_type": "qwen3_5_moe"},
+ get_model_classes=_get_model_classes)
+ shards = open_shards(model_dir)
+ spec = cfg.moe_spec
+ opts = cfg.options
+ # qwen config.json nests the text params under ``text_config``; mlx-lm
+ # returns the raw dict, so resolve defensively.
+ _tc = model_config.get("text_config", model_config)
+ n_layers = int(_tc.get("num_hidden_layers", 40))
+ installed = install_streaming_experts(
+ model, shards, spec, options=opts, num_layers=n_layers)
+ all_stream = {li: t for li, t in enumerate(installed) if t is not None}
+ stream_layers = {li: t for li, t in all_stream.items() if t._staged_mode}
+
+ if cfg.lora:
+ from edge0.adapters.lora import install_lora
+ install_lora(model, cfg.lora, r=cfg.lora_r, alpha=cfg.lora_alpha)
+
+ pg_state = pg_stager = None
+ if cfg.prerouter and cfg.prerouter.weights_file:
+ pg_state, heads = install_prerouter(
+ model=model, spec=spec, pspec=cfg.prerouter, n_layers=n_layers)
+ pg_stager = CrossTokenStager(
+ model=model, spec=spec, pspec=cfg.prerouter,
+ state=pg_state, stream_layers=stream_layers,
+ top_k=cfg.prerouter_top_k)
+ print(f"[edge0-35b] prerouter installed: {len(heads)} heads, "
+ f"start={cfg.prerouter.start_layer}, "
+ f"K={cfg.prerouter_top_k}", flush=True)
+ installs = dict(all_stream_layers=all_stream,
+ stream_layers=stream_layers,
+ pg_state=pg_state, pg_stager=pg_stager)
+ return model, model_config, shards, installs
+
+
+class Qwen35Engine(Edge0Engine):
+ """Streaming Qwen3.5-MoE engine (staged decode + trained prerouter)."""
+
+ name = "edge0-35b"
+
+ def _build(self):
+ cfg = self.cfg
+ (self.model, self.model_config, self.shards,
+ inst) = load_installed(cfg.model_dir, cfg)
+ self._all_stream_layers = inst["all_stream_layers"]
+ self._stream_layers = inst["stream_layers"]
+ self._pg_state = inst["pg_state"]
+ self._pg_stager = inst["pg_stager"]
+ self._lm = self.model.language_model
+ opts = cfg.options
+ if self._tok is None:
+ try:
+ self._tok = load_tokenizer(cfg.model_dir)
+ except Exception: # noqa: BLE001 — tokenizer optional for CLI
+ pass
+
+ self._prefill_before_layer = make_prefill_before_layer(
+ self._all_stream_layers,
+ full_n=getattr(opts, "prefill_full_layers", 0),
+ hot_n=opts.prefill_hot, hot_window=cfg.hot_window)
+ self._intra_after_layer = make_intra_after_layer(
+ self._all_stream_layers, enabled=cfg.intra_staging)
+ self._history_prefetch = make_history_prefetch(
+ self._all_stream_layers, enabled=cfg.prefetch_history)
+
+ self.cache = self._lm.make_cache()
+
+ # ---- forward ----------------------------------------------------------
+
+ def _forward(self, ids, intra_stage: bool = True) -> core.array:
+ inputs = core.array(ids)[None, :]
+ opts = self.cfg.options
+ prefill_multi = len(ids) > 1 and self._prefill_active
+ full_layer = bool(
+ prefill_multi and self._all_stream_layers
+ and opts.full_layer_prefill)
+ before_cb = self._prefill_before_layer if prefill_multi else None
+ if full_layer:
+ after_cb = None
+ else:
+ after_cb = (self._intra_after_layer
+ if (intra_stage and self._intra_after_layer is not None
+ and len(ids) == 1)
+ else None)
+ h = self._lm.model(
+ inputs, cache=self.cache, before_layer_cb=before_cb,
+ after_layer_cb=after_cb,
+ async_eval_per_layer=bool(prefill_multi and full_layer))
+ logits = self._lm.lm_head(h[0, -1])
+ core.eval(logits)
+ # Step boundary: ONE stacked head batch -> ONE tolist -> fills.
+ if (self._pg_stager is not None and not self._prefill_active
+ and len(ids) == 1):
+ self._pg_stager.stage_all()
+ self._pg_state.swap()
+ return logits
+
+ # ---- step hooks -------------------------------------------------------
+
+ def _step_pre(self, token_id: int) -> None:
+ if self._pg_stager is not None:
+ # Router layers (no owner): fill from their previous actuals.
+ st = self._pg_state
+ for li, exp in self._all_stream_layers.items():
+ if li < st.start or li >= st.n - 1:
+ if exp.last_used:
+ exp.stage_experts(list(exp.last_used))
+ elif self._history_prefetch is not None:
+ self._history_prefetch()
+
+ def _step_post(self, token_id: int) -> None:
+ if self._pg_stager is not None:
+ st = self._pg_state
+ for li, exp in self._all_stream_layers.items():
+ if li < st.start or li >= st.n - 1:
+ exp.sync_actuals()
+ return
+ if self._stream_layers:
+ for exp in self._all_stream_layers.values():
+ exp.sync_actuals()
+ exp.stage_experts(list(exp.last_used))
+ exp.swap_staged()
+
+ # ---- prefill / reset --------------------------------------------------
+
+ def _prefill_end(self) -> None:
+ for exp in self._all_stream_layers.values():
+ exp.clear_full_layer()
+ for exp in self._all_stream_layers.values():
+ exp.refresh_hot_pins()
+ if self._stream_layers:
+ for exp in self._stream_layers.values():
+ exp.stage_from_prefill()
+
+ def _reset_state(self) -> None:
+ self.cache = self._lm.make_cache()
+ if self._pg_state is not None:
+ self._pg_state.reset()
+
+ def _lm_logits(self, h: core.array) -> core.array:
+ return self._lm.lm_head(h[0, -1])
diff --git a/src/edge0/models/__init__.py b/src/edge0/models/__init__.py
new file mode 100644
index 0000000..9804d40
--- /dev/null
+++ b/src/edge0/models/__init__.py
@@ -0,0 +1,17 @@
+"""Model adapters: one package per tier, registered on import.
+
+``edge0.models.base`` carries the shared ``ModelConfig``; the tier
+packages (``edge0.models.edge0_35b``, ``edge0.models.edge0_10b``)
+define their family configs and the ``build_model`` / ``build_engine``
+entry points, then call ``register_model`` so ``AutoConfig`` /
+``AutoModel`` / ``AutoEngine`` can resolve them by name.
+"""
+
+from __future__ import annotations
+
+from edge0.models.base import ModelConfig
+
+# Importing the tier packages populates edge0.registry.MODEL_REGISTRY.
+from edge0.models import edge0_10b, edge0_35b # noqa: F401
+
+__all__ = ["ModelConfig", "edge0_10b", "edge0_35b"]
diff --git a/src/edge0/models/base.py b/src/edge0/models/base.py
new file mode 100644
index 0000000..ad3bac8
--- /dev/null
+++ b/src/edge0/models/base.py
@@ -0,0 +1,96 @@
+"""Model adapters: per-model configs, profiles and build entry points.
+
+The registry contract (``edge0.registry``) is: an adapter module exposes
+``Config`` (a ``ModelConfig`` subclass with ``from_pretrained``),
+``build_model`` and ``build_engine``. ``Config`` carries everything the
+engines need — MoE spec, streaming options, prerouter spec, LoRA, sampling
+defaults, server port and the acceptance profile (tokens/s + peak-active
+memory targets measured on the production machine).
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field, fields, replace
+from pathlib import Path
+
+from edge0.config import GenerationConfig
+from edge0.moe.spec import MoESpec
+from edge0.prerouter.spec import PrerouterSpec
+from edge0.streaming.options import LayerOptions
+
+# Repository-root artifacts (gitignored; produced once by
+# scripts/convert_adapters_legacy.py from the training npz exports, then
+# the source npz are discarded).
+ARTIFACTS_DIR = Path(__file__).resolve().parents[3] / "artifacts"
+
+
+def artifact(name: str) -> str:
+ """Absolute path of one converted adapter file under ``artifacts/``."""
+ return str(ARTIFACTS_DIR / name)
+
+
+@dataclass
+class ModelConfig:
+ """Everything one edge0 model tier needs to run.
+
+ Subclasses fix the family defaults (specs, presets, generation, port,
+ acceptance targets); ``from_pretrained`` merges user overrides.
+ """
+
+ name: str
+ model_dir: str
+ moe_spec: MoESpec
+ options: LayerOptions
+ prerouter: PrerouterSpec | None = None
+ prerouter_top_k: int = 0 # 0 -> options.top_k
+ lora: str = "" # safetensors path; "" disables
+ lora_r: int = 16
+ lora_alpha: float = 32.0
+ gen: GenerationConfig = field(default_factory=GenerationConfig)
+ prefill_chunk: int = 2048
+ hot_window: int = 4
+ intra_staging: bool = False
+ prefetch_history: bool = True
+ port: int = 8000
+ # acceptance profile (measured on the production Mac)
+ target_tok_s: float = 0.0
+ peak_active_mem_mb: float = 0.0
+
+ @classmethod
+ def from_pretrained(cls, model_dir: str | None = None, **overrides):
+ """Build the family config, resolving artifact paths and applying
+ per-field overrides (any public attribute may be overridden)."""
+ if model_dir is None:
+ model_dir = overrides.pop("model_dir", None) or ""
+ base = cls._defaults(model_dir)
+ for key, value in overrides.items():
+ if not hasattr(base, key):
+ raise TypeError(
+ f"unknown {cls.__name__} override {key!r} "
+ f"(known: {sorted(f.name for f in fields(base))})")
+ base = replace(base, **{key: value})
+ return base
+
+ @classmethod
+ def _defaults(cls, model_dir: str) -> "ModelConfig":
+ raise NotImplementedError
+
+
+def resolve_prerouter(pspec: PrerouterSpec, weights_file: str) -> PrerouterSpec:
+ """Fill the prerouter weights path (specs are frozen; only used when
+ the caller didn't already set one)."""
+ if pspec.weights_file:
+ return pspec
+ return replace(pspec, weights_file=weights_file)
+
+
+def resolve_lora(lora: str, model_dir: str, r: int, alpha: float):
+ """Sanitize the LoRA override: a bare model name resolves to the
+ artifact of the same tier, ``model_dir`` means "keep the training
+ weights in place", empty string disables."""
+ if lora == "model_dir":
+ return (os.path.join(model_dir, "lora.safetensors")
+ if os.path.isfile(os.path.join(model_dir, "lora.safetensors"))
+ else "")
+ return lora
diff --git a/src/edge0/models/edge0_10b/__init__.py b/src/edge0/models/edge0_10b/__init__.py
new file mode 100644
index 0000000..155bf24
--- /dev/null
+++ b/src/edge0/models/edge0_10b/__init__.py
@@ -0,0 +1,100 @@
+"""edge0-10b adapter: Ling 3.0 hybrid (MLA + MoE, 10B tier).
+
+Family facts (production profile, verified against the v7 checkpoint):
+
+* 24 layers (layer 0 dense), 128 routed experts (4-bit affine,
+ group 64), K=8 native, sigmoid + group-limited routing
+ (n_group 8, topk_group 4, routed scaling 2.5, norm_topk_prob),
+ 1 resident shared expert, separate gate/up/down stacked tensors
+ under ``model.layers.N.mlp.experts``.
+* hybrid prerouter baked into the vendored model: 22 heads
+ (explicit owners 1..22), start_layer 1, hidden 512, fp16,
+ feature_topk "executed", consumed inside ``BailingSparseMoE``
+ from ``prerouter_cache`` logits (patch_call=False).
+* serving: port 8083; acceptance ≈33 tok/s, peak active ≈1.4 GB.
+"""
+
+from __future__ import annotations
+
+import sys
+
+from edge0.config import GenerationConfig
+from edge0.models.base import ModelConfig, artifact
+from edge0.moe.spec import MoESpec, QuantSpec, RouterKind, WeightLayout
+from edge0.prerouter.spec import PrerouterSpec
+from edge0.registry import register_model
+from edge0.streaming.options import LayerOptions
+
+
+class Ling10BConfig(ModelConfig):
+ """edge0-10b family config (Ling 3.0 hybrid, K=8)."""
+
+ @classmethod
+ def _defaults(cls, model_dir: str) -> "Ling10BConfig":
+ return cls(
+ name="edge0-10b",
+ model_dir=model_dir,
+ moe_spec=MoESpec(
+ num_experts=128, top_k=8, intermediate_size=512,
+ router=RouterKind.SIGMOID_GROUP, norm_topk_prob=True,
+ routed_scaling=2.5, n_group=8, topk_group=4,
+ shared_experts=1,
+ quant=QuantSpec(bits=4, group_size=64, mode="affine"),
+ layout=WeightLayout.SEPARATE,
+ key_template="model.layers.{layer}.mlp.experts",
+ block_path="model.layers.{layer}.mlp",
+ layer_path="model.layers.{layer}",
+ ),
+ options=LayerOptions.prod_k8(),
+ prerouter=PrerouterSpec(
+ kind=RouterKind.SIGMOID_GROUP,
+ start_layer=7, hidden=512, dtype="fp16",
+ feature_topk="executed",
+ owners=tuple(range(6, 23)),
+ weights_file=artifact("prerouter_edge0_10b.safetensors"),
+ patch_call=False,
+ ),
+ prerouter_top_k=8,
+ lora=artifact("lora_edge0_10b.safetensors"),
+ lora_r=16, lora_alpha=32.0,
+ gen=GenerationConfig(
+ temperature=0.7, top_p=0.95, top_k=64,
+ repetition_penalty=1.1, max_new_tokens=2048,
+ eos_ids=(156895,)),
+ prefill_chunk=2048,
+ hot_window=1,
+ intra_staging=False,
+ prefetch_history=True,
+ port=8083,
+ target_tok_s=33.0,
+ peak_active_mem_mb=1400.0,
+ )
+
+
+def build_model(model_dir: str | None = None, **overrides):
+ """Load the ling skeleton with streaming twins / LoRA / prerouter
+ installed (the engine's own build path, DRY-shared)."""
+ from edge0.engine.ling import load_installed
+
+ cfg = Ling10BConfig.from_pretrained(model_dir, **overrides)
+ model, _mcfg, _shards, _installs = load_installed(cfg.model_dir, cfg)
+ return model
+
+
+def build_engine(model_dir: str | None = None, **overrides):
+ """Build a ready-to-generate Ling10BEngine.
+
+ ``think`` (THINK_MODE parity, default False) selects the
+ chat-template thinking mode for ``encode_chat``.
+ """
+ from edge0.engine.ling import Ling10BEngine
+
+ think = overrides.pop("think", False)
+ cfg = Ling10BConfig.from_pretrained(model_dir, **overrides)
+ return Ling10BEngine(cfg.model_dir, cfg, think=think)
+
+
+register_model("edge0-10b", sys.modules[__name__])
+
+
+Config = Ling10BConfig # registry contract: adapter.Config
diff --git a/src/edge0/models/edge0_35b/__init__.py b/src/edge0/models/edge0_35b/__init__.py
new file mode 100644
index 0000000..90805a4
--- /dev/null
+++ b/src/edge0/models/edge0_35b/__init__.py
@@ -0,0 +1,93 @@
+"""edge0-35b adapter: Qwen3.5-MoE (K=4 tier, 35B-class).
+
+Family facts (production profile, verified against the round-7
+checkpoint):
+
+* 40 layers, 256 routed experts (4-bit affine, group 64), K=4,
+ softmax->top-k routing with norm_topk_prob, 1 resident shared
+ expert, separate gate/up/down stacked tensors under
+ ``language_model.model.layers.N.mlp.switch_mlp``.
+* trained prerouter: 33 heads (owners 6..38), start_layer 7,
+ hidden 512, fp16, consumed directly by the patched MoE block at
+ decode (patch_call=True, cross-token staged decode, K=4).
+* serving: port 8085; acceptance ≈13 tok/s, peak active ≈3.3 GB.
+"""
+
+from __future__ import annotations
+
+import sys
+
+from edge0.config import GenerationConfig
+from edge0.models.base import ModelConfig, artifact
+from edge0.moe.spec import MoESpec, QuantSpec, RouterKind, WeightLayout
+from edge0.prerouter.spec import PrerouterSpec
+from edge0.registry import register_model
+from edge0.streaming.options import LayerOptions
+
+
+class Qwen35Config(ModelConfig):
+ """edge0-35b family config (Qwen3.5-MoE, K=4)."""
+
+ @classmethod
+ def _defaults(cls, model_dir: str) -> "Qwen35Config":
+ return cls(
+ name="edge0-35b",
+ model_dir=model_dir,
+ moe_spec=MoESpec(
+ num_experts=256, top_k=4, intermediate_size=512,
+ router=RouterKind.SOFTMAX_TOPK, norm_topk_prob=True,
+ shared_experts=1,
+ quant=QuantSpec(bits=4, group_size=64, mode="affine"),
+ layout=WeightLayout.SEPARATE,
+ key_template="language_model.model.layers.{layer}."
+ "mlp.switch_mlp",
+ block_path="language_model.model.layers.{layer}.mlp",
+ layer_path="language_model.model.layers.{layer}",
+ ),
+ options=LayerOptions.staged_k4(),
+ prerouter=PrerouterSpec(
+ kind=RouterKind.SOFTMAX_TOPK,
+ start_layer=7, hidden=512, dtype="fp16",
+ feature_topk="executed",
+ weights_file=artifact("prerouter_edge0_35b_k4.safetensors"),
+ patch_call=True,
+ ),
+ prerouter_top_k=4,
+ lora=artifact("lora_edge0_35b_k4.safetensors"),
+ lora_r=16, lora_alpha=32.0,
+ gen=GenerationConfig(
+ temperature=0.6, top_p=0.95, top_k=64,
+ repetition_penalty=1.0, max_new_tokens=2048,
+ eos_ids=(248046, 248044)),
+ prefill_chunk=2048,
+ hot_window=4,
+ intra_staging=False,
+ prefetch_history=True,
+ port=8085,
+ target_tok_s=13.0,
+ peak_active_mem_mb=3400.0,
+ )
+
+
+def build_model(model_dir: str | None = None, **overrides):
+ """Load the qwen skeleton with streaming twins / LoRA / prerouter
+ installed (the engine's own build path, DRY-shared)."""
+ from edge0.engine.qwen import load_installed
+
+ cfg = Qwen35Config.from_pretrained(model_dir, **overrides)
+ model, _mcfg, _shards, _installs = load_installed(cfg.model_dir, cfg)
+ return model
+
+
+def build_engine(model_dir: str | None = None, **overrides):
+ """Build a ready-to-generate Qwen35Engine."""
+ from edge0.engine.qwen import Qwen35Engine
+
+ cfg = Qwen35Config.from_pretrained(model_dir, **overrides)
+ return Qwen35Engine(cfg.model_dir, cfg)
+
+
+register_model("edge0-35b", sys.modules[__name__])
+
+
+Config = Qwen35Config # registry contract: adapter.Config
diff --git a/src/edge0/moe/__init__.py b/src/edge0/moe/__init__.py
new file mode 100644
index 0000000..70d1301
--- /dev/null
+++ b/src/edge0/moe/__init__.py
@@ -0,0 +1,18 @@
+"""MoE specs and routing math (shared by resident and streaming paths)."""
+
+from edge0.moe.routing import group_select_from_logits, select_from_logits
+from edge0.moe.spec import (
+ MoESpec,
+ QuantSpec,
+ RouterKind,
+ WeightLayout,
+)
+
+__all__ = [
+ "MoESpec",
+ "QuantSpec",
+ "RouterKind",
+ "WeightLayout",
+ "select_from_logits",
+ "group_select_from_logits",
+]
diff --git a/src/edge0/moe/routing.py b/src/edge0/moe/routing.py
new file mode 100644
index 0000000..0a9d2e4
--- /dev/null
+++ b/src/edge0/moe/routing.py
@@ -0,0 +1,70 @@
+"""Routing math, shared by resident and streaming MoE paths.
+
+These functions must stay bit-identical to the vendored base
+implementations (they are extracted from them verbatim); parity tests pin
+this.
+"""
+
+from __future__ import annotations
+
+from edge0.backends import core
+
+
+def select_from_logits(logits, top_k: int, norm: bool = True):
+ """Softmax-topk router: precise softmax -> top-k -> renormalize.
+
+ Mirrors the Qwen3.5-MoE HF router semantics (norm_topk_prob=True).
+ Returns ``(inds [..., k], scores [..., k])``.
+ """
+ gates = core.softmax(logits, axis=-1, precise=True)
+ inds = core.argpartition(gates, kth=-top_k, axis=-1)[..., -top_k:]
+ scores = core.take_along_axis(gates, inds, axis=-1)
+ if norm:
+ scores = scores / scores.sum(axis=-1, keepdims=True)
+ return inds, scores
+
+
+def group_select_from_logits(logits, top_k: int, n_group: int,
+ topk_group: int, routed_scaling: float,
+ norm: bool = True, expert_bias=None):
+ """Sigmoid + group-limited top-k router (DeepSeek-V3 / Bailing law).
+
+ Selection scores are ``sigmoid(logits) + expert_bias``; the
+ ``topk_group`` best groups (by sum of each group's top-2 selection
+ scores) survive; the top-k experts within surviving groups are chosen
+ by selection score; the WEIGHTS are the raw sigmoid scores of the
+ chosen experts (bias excluded), normalized if ``norm`` and scaled by
+ ``routed_scaling``.
+
+ Returns ``(inds [..., k], scores [..., k])``.
+ """
+ scores = core.sigmoid(logits.astype(core.float32))
+ select = scores
+ if expert_bias is not None:
+ select = scores + expert_bias
+
+ b_shape = select.shape[:-1]
+ k_drop = n_group - topk_group
+ if k_drop > 0:
+ grouped = select.reshape(
+ *b_shape, n_group, select.shape[-1] // n_group)
+ # Group score = sum of the top-2 selection scores in the group.
+ top2 = core.topk(grouped, 2, axis=-1)
+ group_scores = top2.sum(axis=-1)
+ drop = core.argpartition(group_scores, kth=k_drop - 1,
+ axis=-1)[..., :k_drop]
+ masked = core.put_along_axis(
+ grouped,
+ core.expand_dims(drop, -1),
+ core.array(-float("inf"), grouped.dtype),
+ axis=-2,
+ )
+ select = masked.reshape(*b_shape, select.shape[-1])
+ # topk_group == n_group keeps every group — nothing to drop.
+
+ idx = core.argpartition(-select, kth=top_k - 1, axis=-1)[..., :top_k]
+ w = core.take_along_axis(scores, idx, axis=-1)
+ if norm:
+ w = w / (w.sum(axis=-1, keepdims=True) + 1e-20)
+ w = w * routed_scaling
+ return idx, w
diff --git a/src/edge0/moe/spec.py b/src/edge0/moe/spec.py
new file mode 100644
index 0000000..176c43c
--- /dev/null
+++ b/src/edge0/moe/spec.py
@@ -0,0 +1,147 @@
+"""MoE abstraction: routing kinds, quantized weight layouts, and the spec
+that drives both the resident math and the streaming layer.
+
+Every MoE block in a supported model is described by one ``MoESpec``; the
+streaming subsystem consumes the spec (layout + key template) so one
+generic ``StreamingSwitchGLU`` serves all models.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Literal
+
+
+class RouterKind(Enum):
+ """Router math families supported by edge0."""
+
+ SOFTMAX_TOPK = "softmax_topk"
+ """precise softmax -> top-k -> renormalize (Qwen3.5-MoE style,
+ norm_topk_prob=True)."""
+
+ SIGMOID_GROUP = "sigmoid_group"
+ """sigmoid scores + group-limited top-k + routed scaling
+ (DeepSeek-V3 / Bailing style)."""
+
+
+class WeightLayout(Enum):
+ """How expert projections are stored in the checkpoint."""
+
+ SEPARATE = "separate"
+ """gate_proj / up_proj / down_proj as three stacked tensors."""
+
+ FUSED_GATE_UP = "fused_gate_up"
+ """gate_proj+up_proj fused into one stacked tensor (rows doubled
+ along the out axis); down_proj separate."""
+
+
+@dataclass(frozen=True)
+class QuantSpec:
+ """Weight quantization description (per-tensor defaults)."""
+
+ bits: int = 4
+ group_size: int = 64
+ mode: str = "affine"
+
+
+@dataclass(frozen=True)
+class MoESpec:
+ """Everything the framework needs to know about a model's MoE blocks.
+
+ Attributes:
+ num_experts: routed experts per layer.
+ top_k: experts selected per token (routing width).
+ intermediate_size: expert FFN hidden size (documentation only;
+ the math derives sizes from the weight tensors).
+ router: routing math family.
+ norm_topk_prob: renormalize selection weights.
+ routed_scaling: multiplier applied to selection weights
+ (sigmoid-group routers); None for softmax-topk.
+ n_group / topk_group: group-limited routing parameters
+ (sigmoid-group routers only).
+ shared_experts: number of resident shared experts (never streamed).
+ quant: weight quantization.
+ layout: weight layout.
+ key_template: safetensors key prefix for layer ``{layer}``, with
+ ``{proj}`` / ``{part}`` placeholders appended by the consumer,
+ e.g. ``"language_model.model.layers.{layer}.mlp.switch_mlp"``.
+ block_path: dotted attribute path from the loaded model object to
+ the MoE block, with ``{layer}``, e.g.
+ ``"language_model.model.layers.{layer}.mlp.switch_mlp"``.
+ layer_path: dotted attribute path to the DECODER LAYER object
+ (the block's owner), with ``{layer}``, e.g.
+ ``"language_model.model.layers.{layer}"``. Used by prerouter
+ stagers to read per-layer caches; default derived from
+ ``block_path`` by stripping the trailing attribute parts.
+ expert_row_axis: axis of the stacked [num_experts, ...] tensors
+ along which one expert is a contiguous slice (always 0 for the
+ supported layouts; kept for documentation).
+ """
+
+ num_experts: int
+ top_k: int
+ intermediate_size: int
+ router: RouterKind = RouterKind.SOFTMAX_TOPK
+ norm_topk_prob: bool = True
+ routed_scaling: float | None = None
+ n_group: int | None = None
+ topk_group: int | None = None
+ shared_experts: int = 0
+ quant: QuantSpec = field(default_factory=QuantSpec)
+ layout: WeightLayout = WeightLayout.SEPARATE
+ key_template: str = ""
+ block_path: str = ""
+ layer_path: str = ""
+ expert_row_axis: int = 0
+
+ def keys(self, layer: int, proj: str, part: str) -> str:
+ """Resolve one safetensors key for a layer's projection part."""
+ prefix = self.key_template.format(layer=layer)
+ return f"{prefix}.{proj}.{part}"
+
+ def block_of(self, model, layer: int):
+ """Resolve the MoE block object inside a loaded model."""
+ obj = model
+ for part in self.block_path.format(layer=layer).split("."):
+ if part.isdigit():
+ obj = obj[int(part)] # layers are plain lists in most families
+ else:
+ obj = getattr(obj, part)
+ return obj
+
+ def layer_of(self, model, layer: int):
+ """Resolve the decoder layer object (the block's owner).
+
+ Falls back to ``block_path`` minus the trailing attribute
+ segments (the convention: a block lives at
+ ``<layer>.<mlp>.<block>``, so the layer is two segments up).
+ """
+ if self.layer_path:
+ path = self.layer_path.format(layer=layer)
+ else:
+ path = ".".join(
+ self.block_path.format(layer=layer).split(".")[:-2])
+ obj = model
+ for part in path.split("."):
+ if part.isdigit():
+ obj = obj[int(part)]
+ else:
+ obj = getattr(obj, part)
+ return obj
+
+ @property
+ def bundle_projs(self) -> tuple[str, ...]:
+ """Projections a cached expert bundle contains, in stack order.
+
+ Order matches the math signature directly: up, gate, down
+ (``_swiglu(up, gate)``); fused layout packs gate rows on top of
+ up rows so ``split(x_gu, 2)`` yields ``(gate, up)``.
+ """
+ if self.layout is WeightLayout.FUSED_GATE_UP:
+ return ("gate_up_proj", "down_proj")
+ return ("up_proj", "gate_proj", "down_proj")
+
+ @property
+ def fuse_gu(self) -> bool:
+ return self.layout is WeightLayout.FUSED_GATE_UP
diff --git a/src/edge0/prerouter/__init__.py b/src/edge0/prerouter/__init__.py
new file mode 100644
index 0000000..a646b23
--- /dev/null
+++ b/src/edge0/prerouter/__init__.py
@@ -0,0 +1,31 @@
+"""Prerouter: trained cross-token routing predictor heads + step-boundary
+staging.
+
+Public API::
+
+ from edge0.prerouter import (PrerouterSpec, PrerouterState,
+ PrerouterHead, install_prerouter,
+ CrossTokenStager, LingPrerouterStager)
+"""
+
+from edge0.prerouter.heads import PrerouterHead, select_from_logits, topk_onehot
+from edge0.prerouter.install import install_prerouter
+from edge0.prerouter.spec import PrerouterSpec
+from edge0.prerouter.stager import (
+ CrossTokenStager,
+ LingPrerouterStager,
+ PrerouterStager,
+)
+from edge0.prerouter.state import PrerouterState
+
+__all__ = [
+ "PrerouterSpec",
+ "PrerouterState",
+ "PrerouterHead",
+ "PrerouterStager",
+ "CrossTokenStager",
+ "LingPrerouterStager",
+ "install_prerouter",
+ "select_from_logits",
+ "topk_onehot",
+]
diff --git a/src/edge0/prerouter/heads.py b/src/edge0/prerouter/heads.py
new file mode 100644
index 0000000..a08867c
--- /dev/null
+++ b/src/edge0/prerouter/heads.py
@@ -0,0 +1,73 @@
+"""Prerouter head math — faithful port of the qwen trained-head
+implementation (the legacy ``v7_trained`` file) with identifiers
+renamed.
+
+Semantics (cross-token):
+ * features = concat[hidden, this-token executed top-k one-hot,
+ previous-token executed top-k one-hot];
+ * head = fc1(features) -> exact(erf) gelu -> fc2 + linear_init on the
+ SAME concat features;
+ * fp16 weights by default (the training export precision, and the
+ precision ling v7 production ran the heads at); fp32 for debugging.
+"""
+
+from __future__ import annotations
+
+import math
+
+from edge0.backends import core
+from edge0.backends import nn
+
+_EYE_CACHE: dict[int, core.array] = {}
+
+
+def _eye(num_experts: int) -> core.array:
+ e = _EYE_CACHE.get(num_experts)
+ if e is None:
+ e = core.eye(num_experts)
+ _EYE_CACHE[num_experts] = e
+ return e
+
+
+def topk_onehot(inds: core.array, num_experts: int) -> core.array:
+ """inds [B,T,k] -> one-hot [B,T,num_experts]."""
+ idm = _eye(num_experts)
+ oh = core.take(idm, inds, axis=0) # [B,T,k,E]
+ return oh.sum(axis=-2)
+
+
+def gelu_erf(x: core.array) -> core.array:
+ """Exact GELU (erf) matching torch ``F.gelu`` default."""
+ return 0.5 * x * (1.0 + core.erf(x / math.sqrt(2.0)))
+
+
+def select_from_logits(logits: core.array, top_k: int):
+ """Mirror the qwen HF router math: precise softmax -> top-k ->
+ renormalize. Returns (inds [..., k], scores [..., k])."""
+ gates = core.softmax(logits, axis=-1, precise=True)
+ inds = core.argpartition(gates, kth=-top_k, axis=-1)[..., -top_k:]
+ scores = core.take_along_axis(gates, inds, axis=-1)
+ scores = scores / scores.sum(axis=-1, keepdims=True)
+ return inds, scores
+
+
+class PrerouterHead(nn.Module):
+ """Trained prerouter head (fp16 by default; fp32 via ``dtype``)."""
+
+ def __init__(self, hidden: int, num_experts: int, prerouter_hidden: int,
+ dtype=core.float16):
+ super().__init__()
+ self._dtype = dtype
+ f_in = hidden + 2 * num_experts
+ self.fc1 = nn.Linear(f_in, prerouter_hidden, bias=False)
+ self.fc2 = nn.Linear(prerouter_hidden, num_experts, bias=False)
+ self.linear_init = nn.Linear(f_in, num_experts, bias=False)
+
+ def __call__(self, h: core.array, executed_oh: core.array,
+ prev_oh: core.array) -> core.array:
+ feats = core.concatenate([h, executed_oh, prev_oh], axis=-1)
+ if feats.dtype != self._dtype:
+ feats = feats.astype(self._dtype)
+ # linear_init consumes the full concat features, same as training.
+ return self.linear_init(feats) + self.fc2(
+ gelu_erf(self.fc1(feats)))
diff --git a/src/edge0/prerouter/install.py b/src/edge0/prerouter/install.py
new file mode 100644
index 0000000..13fa101
--- /dev/null
+++ b/src/edge0/prerouter/install.py
@@ -0,0 +1,188 @@
+"""Install prerouter heads into a loaded base model.
+
+Two conventions exist (both supported here):
+
+* qwen (``patch_call=True``): heads are NEW ``PrerouterHead`` modules
+ attached to the MoE blocks, and the MoE block's ``__call__`` is patched
+ at class level so decode routes through the prerouter's
+ ``pred_inds``/``pred_scores`` (100% replacement, no matching/drops).
+ Prefill is untouched (router path). The patch mirrors the deployment's
+ proven ``_trained_call``, with guards per instance so non-prerouter
+ engines keep the normal router.
+* ling (``patch_call=False``): the vendored model already contains one
+ ``BailingPrerouter`` per MoE layer and consumes ``prerouter_cache``
+ logits internally; this installer only fills the head weights (and
+ zeroes ``linear_init`` when the checkpoint has none).
+"""
+
+from __future__ import annotations
+
+from edge0.backends import core
+from edge0.backends import nn
+
+from edge0.backends.mlx._impl.qwen3_next import Qwen3NextSparseMoeBlock
+from edge0.backends.mlx.io import load_safetensors
+from edge0.moe.spec import MoESpec
+from edge0.prerouter.heads import PrerouterHead, topk_onehot
+from edge0.prerouter.spec import PrerouterSpec
+from edge0.prerouter.state import PrerouterState
+
+_PATCHED_MARK = "_edge0_prerouter_patched"
+
+
+def _parse_weights(weights: dict[str, core.array], dtype):
+ """``layers.<N>.<part>.weight`` keys -> ``{N: {"<part>.weight": mx}}``.
+
+ Accepts both ``layers.N.fc1.weight`` and the nested
+ ``layers.N.mlp.prerouter.fc1.weight`` form."""
+ heads: dict[int, dict[str, core.array]] = {}
+ for key, arr in weights.items():
+ parts = key.split(".")
+ if len(parts) < 4 or parts[0] != "layers":
+ continue
+ try:
+ owner = int(parts[1])
+ except ValueError:
+ continue
+ if parts[-2] not in ("fc1", "fc2", "linear_init"):
+ continue
+ if arr.dtype != dtype:
+ arr = arr.astype(dtype)
+ heads.setdefault(owner, {})[f"{parts[-2]}.weight"] = arr
+ return heads
+
+
+def install_prerouter(
+ *,
+ model,
+ spec: MoESpec,
+ pspec: PrerouterSpec,
+ weights: dict[str, core.array] | None = None,
+ state: PrerouterState | None = None,
+ n_layers: int,
+) -> tuple[PrerouterState, dict[int, nn.Module]]:
+ """Attach prerouter heads to a loaded model.
+
+ Returns ``(state, heads)`` where ``state`` is the cross-token
+ double-buffer the engine drives at the step boundary and ``heads``
+ maps owner layer -> head module. ``weights`` is a flat
+ ``name -> core.array`` dict (from ``load_safetensors``); when None the
+ file from ``pspec.weights_file`` is loaded instead.
+ """
+ if weights is None:
+ if not pspec.weights_file:
+ raise ValueError(
+ "install_prerouter needs weights or PrerouterSpec.weights_file")
+ weights = load_safetensors(pspec.weights_file)
+ dtype = core.float16 if pspec.dtype == "fp16" else core.float32
+ head_weights = _parse_weights(weights, dtype)
+ missing = [n for n in pspec.owner_layers(n_layers) if n not in head_weights]
+ if missing:
+ raise ValueError(
+ f"prerouter weights missing owner layers {missing[:5]}... "
+ f"(have {sorted(head_weights)})")
+
+ if state is None:
+ state = PrerouterState(n_layers, spec.num_experts,
+ pspec.start_layer,
+ owners=pspec.owner_layers(n_layers))
+
+ heads: dict[int, nn.Module] = {}
+ for owner in pspec.owner_layers(n_layers):
+ block = spec.block_of(model, owner)
+ w = head_weights[owner]
+ if pspec.patch_call:
+ # hidden = the MoE block's input width, read off the router's
+ # gate projection (weight is [num_experts, dim]).
+ dim = block.gate.weight.shape[1]
+ head = PrerouterHead(dim, spec.num_experts, pspec.hidden,
+ dtype=dtype)
+ # hidden size: MoESpec doesn't carry it; read from the gate
+ # projection weight shape.
+ head.fc1.weight = w["fc1.weight"]
+ head.fc2.weight = w["fc2.weight"]
+ head.linear_init.weight = w["linear_init.weight"]
+ block.prerouter_head = head
+ block.prerouter_li = owner
+ block.prerouter_start = pspec.start_layer
+ block.prerouter_enabled = True
+ block.prerouter_state = state
+ heads[owner] = head
+ else:
+ pg = getattr(block, "prerouter", None)
+ if pg is None:
+ raise AttributeError(
+ f"layer {owner}: patch_call=False but block has no "
+ "'prerouter' head (wrong model family?)")
+ pg.fc1 = nn.Linear(w["fc1.weight"].shape[1],
+ w["fc1.weight"].shape[0], bias=False)
+ pg.fc1.weight = w["fc1.weight"]
+ pg.fc2 = nn.Linear(w["fc2.weight"].shape[1],
+ w["fc2.weight"].shape[0], bias=False)
+ pg.fc2.weight = w["fc2.weight"]
+ wk = w.get("linear_init.weight")
+ if wk is not None:
+ pg.linear_init = nn.Linear(wk.shape[1], wk.shape[0],
+ bias=False)
+ pg.linear_init.weight = wk
+ else:
+ pg.linear_init.weight = core.zeros(
+ pg.linear_init.weight.shape, dtype=dtype)
+ heads[owner] = pg
+
+ if pspec.patch_call and not getattr(
+ Qwen3NextSparseMoeBlock, _PATCHED_MARK, False):
+ _patch_qwen_consume()
+ return state, heads
+
+
+def _patch_qwen_consume():
+ """Patch ``Qwen3NextSparseMoeBlock.__call__`` (deploy-proven
+ ``_trained_call`` semantics).
+
+ A class-level patch is required because implicit ``moe(x)`` calls look
+ up the type, not the instance. Guarded per instance: only blocks with
+ ``prerouter_enabled`` and single-token inputs (decode) route through
+ the prerouter; everything else takes the original router path."""
+ orig_call = Qwen3NextSparseMoeBlock.__call__
+
+ def prerouter_call(self, x):
+ sm = getattr(self, "switch_mlp", None)
+ st = getattr(self, "prerouter_state", None)
+ if not (getattr(self, "prerouter_enabled", False) and st is not None
+ and x.ndim == 3 and x.shape[1] == 1
+ and getattr(sm, "_staged_replace", None) is False
+ and sm is not None):
+ return orig_call(self, x)
+ li = self.prerouter_li
+ # This layer CONSUMES the prediction produced by layer li-1's head
+ # at the previous token (prev-layer + prev-token double shift).
+ pred = st.pred_inds[li] if li >= st.start else None
+ if pred is not None:
+ inds = pred
+ scores = st.pred_scores[li]
+ else:
+ # First decode tokens / router layers below start: fall back to
+ # the real router (matches training's pos-0 fallback).
+ gates = core.softmax(self.gate(x), axis=-1, precise=True)
+ k = self.top_k
+ inds = core.argpartition(gates, kth=-k, axis=-1)[..., -k:]
+ scores = core.take_along_axis(gates, inds, axis=-1)
+ if self.norm_topk_prob:
+ scores = scores / scores.sum(axis=-1, keepdims=True)
+
+ oh = topk_onehot(inds, self.num_experts)
+ st.oh[li] = oh
+
+ # Capture for the head this layer OWNS (runs at the step boundary).
+ self.prerouter_m_in = x
+ self.prerouter_oh = oh
+
+ y = self.switch_mlp(x, inds)
+ y = (y * scores[..., None]).sum(axis=-2)
+ shared_y = self.shared_expert(x)
+ shared_y = core.sigmoid(self.shared_expert_gate(x)) * shared_y
+ return y + shared_y
+
+ Qwen3NextSparseMoeBlock.__call__ = prerouter_call
+ setattr(Qwen3NextSparseMoeBlock, _PATCHED_MARK, True)
diff --git a/src/edge0/prerouter/spec.py b/src/edge0/prerouter/spec.py
new file mode 100644
index 0000000..4789109
--- /dev/null
+++ b/src/edge0/prerouter/spec.py
@@ -0,0 +1,65 @@
+"""Prerouter abstraction: the trained cross-token routing predictor.
+
+A prerouter head is OWNED by layer N: it runs on layer N's MoE-input
+hidden state (post-attention norm output) at token t and predicts layer
+N+1's routing. Layer N+1 consumes the prediction from layer N's head
+at token t-1 (cross-token double shift: prev-layer + prev-token), so the
+staged expert set for a decode step IS the prerouter's routing — zero
+drops by construction.
+
+Feature vector per head: ``concat[hidden, this-token top-k one-hot,
+prev-token top-k one-hot]``; head = ``fc1 -> exact(erf) gelu -> fc2 +
+linear_init`` on the same features.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from edge0.moe.spec import RouterKind
+
+
+@dataclass(frozen=True)
+class PrerouterSpec:
+ """Configuration of one model's prerouter.
+
+ Attributes:
+ kind: routing math family the prerouter head's logits are turned
+ into expert selections with. ``SOFTMAX_TOPK`` (qwen:
+ precise softmax -> top-k -> renormalize) or
+ ``SIGMOID_GROUP`` (ling: sigmoid + group-limited top-k, the
+ model's own ``_select_from_logits``).
+ start_layer: first CONSUMING layer; the first owner is
+ ``start_layer - 1`` and the last owner ``n_layers - 2``.
+ hidden: prerouter hidden width (512 for both shipped models).
+ dtype: head compute dtype — "fp16" (default, training export
+ precision) or "fp32" (debug only).
+ feature_topk: which one-hot feeds the "this-token top-k" feature.
+ "executed": the top-k the block actually routed (qwen
+ trained semantics — executed == prerouter-selected at decode);
+ "teacher": the original gate's top-k, recomputed per layer
+ (ling v7 training semantics).
+ owners: explicit owner list override (defaults to
+ ``range(start_layer - 1, n_layers - 1)``).
+ weights_file: safetensors path with per-owner head weights
+ (keys ``layers.<N>.fc1.weight`` etc.); may be provided later
+ via ``install_prerouter``.
+ patch_call: for models whose MoE block is NOT prerouter-aware
+ (qwen3_next), install a class-level ``__call__`` patch so
+ decode routes through ``pred_inds``. Models with baked hooks
+ (bailing_hybrid) set this False.
+ """
+
+ kind: RouterKind = RouterKind.SOFTMAX_TOPK
+ start_layer: int = 7
+ hidden: int = 512
+ dtype: str = "fp16"
+ feature_topk: str = "executed" # "executed" | "teacher"
+ owners: tuple[int, ...] | None = None
+ weights_file: str = ""
+ patch_call: bool = False
+
+ def owner_layers(self, n_layers: int) -> list[int]:
+ if self.owners is not None:
+ return list(self.owners)
+ return list(range(self.start_layer - 1, n_layers - 1))
diff --git a/src/edge0/prerouter/stager.py b/src/edge0/prerouter/stager.py
new file mode 100644
index 0000000..2c8a613
--- /dev/null
+++ b/src/edge0/prerouter/stager.py
@@ -0,0 +1,244 @@
+"""Prerouter step-boundary stager.
+
+After each forward's logits eval, run every owner head as ONE stacked
+batch (the base graph is already materialized), turn the logits into
+expert selections with ONE eval + ONE tolist, and submit the next
+decode step's staged slots per consumer layer — so the pool's SSD
+builds overlap the next forward entirely (no per-layer sync points
+inside the graph).
+
+Two feature/consume conventions exist across the supported models, both
+implemented here:
+
+* qwen (``CrossTokenStager``): heads installed on the MoE blocks, the
+ block patch captures ``(m_in, executed one-hot)``; the stager makes
+ the selection itself (precise softmax -> top-k -> renormalize) and
+ stores ``pred_inds`` / ``pred_scores`` keyed by the consuming layer.
+* ling (``LingPrerouterStager``): heads baked into the vendored model,
+ features read from the model's own caches (``m_in_cache`` /
+ ``last_topk`` teacher feature / ``prev_topk_oh``); the stager stores
+ the raw LOGITS in ``pg_cache`` and the consuming MoE block performs
+ the sigmoid group-limited selection internally.
+"""
+
+from __future__ import annotations
+
+from edge0.backends import core
+
+from edge0.moe.routing import group_select_from_logits, select_from_logits
+from edge0.moe.spec import MoESpec, RouterKind
+from edge0.prerouter.heads import topk_onehot
+from edge0.prerouter.spec import PrerouterSpec
+from edge0.prerouter.state import PrerouterState
+from edge0.streaming.layer import StreamingSwitchGLU
+
+
+def _topk_onehot_pos(idx, num_experts: int, pos: int = -1):
+ """One-hot set of the top-k experts at position ``pos`` of ``idx``
+ ([B, T, K] -> [B, 1, E], OR over the K slots) — ling feature
+ semantics. Returns None when ``idx`` has no such position (e.g. a
+ T=1 prefill asking for pos=-2) so the caller can fall back to the
+ zeros feature (the deployment's ``_topk_onehot`` crashes on this
+ case; the T=1 prompt is one BPE token like 你好 = [34355])."""
+ if idx is None or idx.ndim < 3:
+ return None
+ t = idx.shape[-2]
+ lo = pos if pos >= 0 else t + pos
+ if lo < 0 or lo >= t:
+ return None
+ idx = idx[..., pos, :][..., None, :] # [B, 1, K]
+ ar = core.arange(num_experts, dtype=idx.dtype)
+ oh = (idx[..., None] == ar) # [B, 1, K, E]
+ return oh.sum(axis=-2).astype(core.float32)
+
+
+class PrerouterStager:
+ """Base: one-batch collect -> stack -> select -> eval -> tolist ->
+ per-consumer stage_experts."""
+
+ def __init__(
+ self,
+ *,
+ model,
+ spec: MoESpec,
+ pspec: PrerouterSpec,
+ state: PrerouterState,
+ stream_layers: dict[int, StreamingSwitchGLU],
+ top_k: int,
+ ):
+ self.model = model
+ self.spec = spec
+ self.pspec = pspec
+ self.state = state
+ self.stream_layers = stream_layers
+ self.top_k = top_k
+ self.num_experts = spec.num_experts
+ self.records: list | None = None
+ self.cur_step = 0
+
+ # ---- feature hooks (overridden per family) ----------------------------
+
+ def _features(self, owner: int):
+ """Return (m_in, cur_oh, prev_oh) for the owner layer's head."""
+ raise NotImplementedError
+
+ def _select(self, logits: core.array):
+ """Turn stacked head logits into (inds, scores)."""
+ raise NotImplementedError
+
+ def _store(self, consumer: int, logits_i, inds_i, scores_i,
+ experts: list[int]) -> None:
+ """Record this consumer's prediction + submit its staged fill."""
+ raise NotImplementedError
+
+ def _note(self, owner: int, cur_oh) -> None:
+ """Per-owner side effects (e.g. roll the prev-token feature)."""
+
+ # ---- the step-boundary batch ------------------------------------------
+
+ def stage_all(self) -> dict[int, list[int]]:
+ metas: list[int] = []
+ inputs: list[core.array] = []
+ cur_ohs: list[core.array] = []
+ prev_ohs: list[core.array] = []
+ for owner in self.state.owners:
+ feats = self._features(owner)
+ if feats is None:
+ continue
+ m_in, cur_oh, prev_oh = feats
+ metas.append(owner)
+ inputs.append(m_in)
+ cur_ohs.append(cur_oh)
+ prev_ohs.append(prev_oh)
+ if not metas:
+ return {}
+ per_logits = []
+ for owner, m_in, co, po in zip(metas, inputs, cur_ohs, prev_ohs):
+ head = self._head_of(owner)
+ per_logits.append(head(m_in, co, po))
+ logits = core.stack(per_logits, axis=0) # [n,1,1,E]
+ inds_all, scores_all = self._select(logits)
+ core.eval(inds_all, scores_all)
+ flat = inds_all.reshape(len(metas), -1).tolist() # ONE tolist
+ expert_sets: dict[int, list[int]] = {}
+ for i, owner in enumerate(metas):
+ consumer = owner + 1
+ experts = sorted(set(int(v) for v in flat[i]))
+ expert_sets[consumer] = experts
+ self._store(consumer, logits[i], inds_all[i],
+ scores_all[i], experts)
+ self._note(owner, cur_ohs[i])
+ if self.records is not None:
+ for owner, consumer in zip(metas, metas):
+ self.records.append(
+ (self.cur_step, consumer, expert_sets[consumer]))
+ self.cur_step += 1
+ return expert_sets
+
+ def _head_of(self, owner: int):
+ raise NotImplementedError
+
+
+class CrossTokenStager(PrerouterStager):
+ """qwen semantics: heads on the MoE blocks, selection made here,
+ ``pred_inds`` / ``pred_scores`` keyed by the consuming layer."""
+
+ def __init__(self, *, model, spec: MoESpec, pspec: PrerouterSpec,
+ state: PrerouterState,
+ stream_layers: dict[int, StreamingSwitchGLU], top_k: int):
+ super().__init__(model=model, spec=spec, pspec=pspec,
+ state=state, stream_layers=stream_layers,
+ top_k=top_k)
+
+ def _head_of(self, owner: int):
+ block = self.spec.block_of(self.model, owner)
+ return block.prerouter_head
+
+ def _features(self, owner: int):
+ block = self.spec.block_of(self.model, owner)
+ m_in = getattr(block, "prerouter_m_in", None)
+ oh = getattr(block, "prerouter_oh", None)
+ if m_in is None or oh is None:
+ return None
+ prev_oh = self.state.oh_prev[owner]
+ if prev_oh is None:
+ prev_oh = core.zeros((1, 1, self.num_experts), dtype=oh.dtype)
+ return m_in, oh, prev_oh
+
+ def _select(self, logits: core.array):
+ return select_from_logits(logits, self.top_k)
+
+ def _store(self, consumer: int, logits_i, inds_i, scores_i,
+ experts: list[int]):
+ st = self.state
+ st.pred_inds[consumer] = inds_i
+ st.pred_scores[consumer] = scores_i
+ st.logits[consumer - 1] = logits_i
+ exp = self.stream_layers.get(consumer)
+ if exp is not None:
+ exp.stage_experts(experts)
+
+ def _note(self, owner: int, cur_oh) -> None:
+ pass
+
+
+class LingPrerouterStager(PrerouterStager):
+ """ling semantics: heads baked into the vendored model, raw logits
+ cached per consumer layer (the MoE block re-selects internally with
+ its own ``_select_from_logits``); teacher top-8 feature; the
+ prev-token feature is rolled on the block."""
+
+ def __init__(self, *, model, spec: MoESpec, pspec: PrerouterSpec,
+ state: PrerouterState,
+ stream_layers: dict[int, StreamingSwitchGLU], top_k: int):
+ super().__init__(model=model, spec=spec, pspec=pspec,
+ state=state, stream_layers=stream_layers,
+ top_k=top_k)
+ self.pg_cache: dict[int, core.array] = {}
+
+ def _head_of(self, owner: int):
+ block = self.spec.block_of(self.model, owner)
+ return block.prerouter
+
+ def _features(self, owner: int):
+ layer = self.spec.layer_of(self.model, owner)
+ block = self.spec.block_of(self.model, owner)
+ m = getattr(layer, "m_in_cache", None)
+ if m is None:
+ return None
+ m_in = m[..., -1:, :]
+ last_topk = getattr(block, "last_topk", None)
+ cur_oh = _topk_onehot_pos(last_topk, self.num_experts, pos=-1)
+ prev_oh = getattr(block, "prev_topk_oh", None)
+ if prev_oh is None:
+ # Initial staging right after prefill: the "prev token" feature
+ # is the second-to-last prefill position.
+ prev_oh = _topk_onehot_pos(last_topk, self.num_experts, pos=-2)
+ if cur_oh is None:
+ cur_oh = core.zeros((1, 1, self.num_experts), dtype=core.float32)
+ if prev_oh is None:
+ prev_oh = core.zeros((1, 1, self.num_experts), dtype=core.float32)
+ return m_in, cur_oh, prev_oh
+
+ def _select(self, logits: core.array):
+ return group_select_from_logits(
+ logits, self.top_k,
+ n_group=self.spec.n_group, topk_group=self.spec.topk_group,
+ routed_scaling=self.spec.routed_scaling or 1.0)
+
+ def _store(self, consumer: int, logits_i, inds_i, scores_i,
+ experts: list[int]):
+ self.pg_cache[consumer] = logits_i
+ exp = self.stream_layers.get(consumer)
+ if exp is not None:
+ exp.stage_experts(experts)
+
+ def _note(self, owner: int, cur_oh) -> None:
+ block = self.spec.block_of(self.model, owner)
+ # Remember this token's top-k so the next decode step can feed the
+ # "prev-token" one-hot feature.
+ block.prev_topk_oh = cur_oh
+
+ def reset(self):
+ self.pg_cache = {}
+ self.cur_step = 0
diff --git a/src/edge0/prerouter/state.py b/src/edge0/prerouter/state.py
new file mode 100644
index 0000000..c996314
--- /dev/null
+++ b/src/edge0/prerouter/state.py
@@ -0,0 +1,42 @@
+"""Prerouter decode-time state (cross-token, cross-layer double buffer).
+
+After each decode step call ``swap()``:
+ ``logits_prev[owner] = logits[owner]`` (owner = producing layer N)
+ ``oh_prev[layer] = oh[layer]`` (per-layer executed one-hots)
+``pred_inds`` / ``pred_scores`` are keyed by the CONSUMING layer (N+1)
+and hold the selection made at the step boundary; each is consumed
+exactly once on the next step by that layer, then overwritten.
+"""
+
+from __future__ import annotations
+
+
+class PrerouterState:
+ def __init__(self, n_layers: int, num_experts: int, start_layer: int,
+ owners=None):
+ self.n = n_layers
+ self.num_experts = num_experts
+ self.start = start_layer # first consuming layer
+ if owners is None:
+ owners = list(range(start_layer - 1, n_layers - 1))
+ self.owners = list(owners)
+ self.logits: list = [None] * n_layers # owner output, this token
+ self.logits_prev: list = [None] * n_layers # owner output, prev token
+ self.pred_inds: list = [None] * n_layers # keyed by consumer N+1
+ self.pred_scores: list = [None] * n_layers # keyed by consumer N+1
+ self.oh: list = [None] * n_layers # executed one-hots, now
+ self.oh_prev: list = [None] * n_layers # executed one-hots, prev
+
+ def swap(self):
+ self.logits_prev = self.logits
+ self.logits = [None] * self.n
+ self.oh_prev = self.oh
+ self.oh = [None] * self.n
+
+ def reset(self):
+ self.logits = [None] * self.n
+ self.logits_prev = [None] * self.n
+ self.pred_inds = [None] * self.n
+ self.pred_scores = [None] * self.n
+ self.oh = [None] * self.n
+ self.oh_prev = [None] * self.n
diff --git a/src/edge0/registry.py b/src/edge0/registry.py
new file mode 100644
index 0000000..3827c40
--- /dev/null
+++ b/src/edge0/registry.py
@@ -0,0 +1,111 @@
+"""Transformers-style model registry: name -> adapter class.
+
+Adapters live in ``edge0.models`` and register themselves on import;
+``AutoConfig`` / ``AutoModel`` / ``AutoEngine`` resolve a model by
+explicit name, by the checkpoint's ``config.json`` ``model_type`` (via
+``TYPE_ALIASES``), or by the checkpoint directory basename. Unknown
+names raise ``KeyError`` listing the registered models.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+MODEL_REGISTRY: dict[str, type] = {}
+TYPE_ALIASES: dict[str, str] = {
+ # qwen3_5_moe family
+ "qwen3_5_moe_text": "edge0-35b",
+ "qwen3_5_moe": "edge0-35b",
+ # ling family (config.json carries architectures, not model_type)
+ "bailing_hybrid": "edge0-10b",
+ "bailing_moe_linear": "edge0-10b",
+ "BailingMoeV3ForCausalLM": "edge0-10b",
+}
+
+
+def register_model(name: str, adapter: type) -> None:
+ if name in MODEL_REGISTRY:
+ raise ValueError(f"model {name!r} already registered")
+ MODEL_REGISTRY[name] = adapter
+
+
+@dataclass(frozen=True)
+class _RegisteredModel:
+ """One entry of the registry, for the KeyError listing."""
+
+ name: str
+ adapter: type
+
+ def __str__(self) -> str: # pragma: no cover - cosmetic
+ return f"{self.name} ({self.adapter.__module__}.{self.adapter.__name__})"
+
+
+def _resolve_name(model_dir: str | None, name: str | None) -> str:
+ if name:
+ key = name
+ elif model_dir:
+ key = _model_type_from_dir(model_dir)
+ else:
+ raise ValueError("AutoX.from_pretrained needs model_dir or name")
+ if key in MODEL_REGISTRY:
+ return key
+ if key in TYPE_ALIASES:
+ return TYPE_ALIASES[key]
+ known = ", ".join(sorted(MODEL_REGISTRY))
+ raise KeyError(
+ f"unknown model {key!r} (resolved from name={name!r}, "
+ f"model_dir={model_dir!r}); registered: {known}")
+
+def _model_type_from_dir(model_dir: str) -> str:
+ """config.json model_type (or architectures[0] for checkpoints that
+ omit model_type, e.g. the ling family), falling back to the
+ directory basename."""
+ import json
+ import os
+ cfg_path = os.path.join(model_dir, "config.json")
+ if os.path.isfile(cfg_path):
+ try:
+ with open(cfg_path) as f:
+ cfg = json.load(f)
+ mt = str(cfg.get("model_type", "") or "")
+ if not mt:
+ archs = cfg.get("architectures") or []
+ if archs:
+ mt = str(archs[0])
+ if mt:
+ return mt
+ except Exception:
+ pass
+ base = os.path.basename(os.path.normpath(model_dir))
+ return base if base not in ("", ".") else ""
+
+
+class AutoConfig:
+ """Resolve a model's config class from the registry."""
+
+ @classmethod
+ def from_pretrained(cls, model_dir=None, name=None, **kwargs):
+ from edge0 import models # noqa: F401 (populates MODEL_REGISTRY)
+ key = _resolve_name(model_dir, name)
+ return MODEL_REGISTRY[key].Config.from_pretrained(
+ model_dir, **kwargs)
+
+
+class AutoModel:
+ """Load a base model + install streaming experts / prerouter / LoRA."""
+
+ @classmethod
+ def from_pretrained(cls, model_dir=None, name=None, **kwargs):
+ from edge0 import models # noqa: F401
+ key = _resolve_name(model_dir, name)
+ return MODEL_REGISTRY[key].build_model(model_dir, **kwargs)
+
+
+class AutoEngine:
+ """Build a ready-to-generate engine for a model tier."""
+
+ @classmethod
+ def from_pretrained(cls, model_dir=None, name=None, **kwargs):
+ from edge0 import models # noqa: F401
+ key = _resolve_name(model_dir, name)
+ return MODEL_REGISTRY[key].build_engine(model_dir, **kwargs)
diff --git a/src/edge0/sampling.py b/src/edge0/sampling.py
new file mode 100644
index 0000000..8d51aa7
--- /dev/null
+++ b/src/edge0/sampling.py
@@ -0,0 +1,61 @@
+"""Token sampling — port of the deployment's HF-style sampler
+(vectorized repetition penalty; one host-sync categorical draw)."""
+
+from __future__ import annotations
+
+from edge0.backends import core
+
+
+def _mask_logits(logits, temperature, top_k, top_p):
+ """Temperature / top-k / top-p truncation, shared by ``sample`` and
+ any speculative verifier so both draw from the exact same
+ distribution."""
+ logits = logits.astype(core.float32)
+ if temperature > 0:
+ logits = logits / temperature
+ if top_k is not None and top_k > 0:
+ k = min(top_k, logits.shape[-1])
+ top = core.topk(logits, k, axis=-1)
+ # core.topk returns values in ASCENDING order, so the smallest
+ # surviving value is the FIRST entry (top[..., 0]), not the last.
+ threshold = top[..., :1]
+ logits = core.where(logits < threshold, float("-inf"), logits)
+ if top_p is not None and top_p < 1.0:
+ sorted_vals = core.sort(logits, axis=-1)[..., ::-1]
+ cum = core.cumsum(core.softmax(sorted_vals, axis=-1), axis=-1)
+ cutoff = cum <= top_p
+ counts = core.sum(cutoff.astype(core.int32), axis=-1)
+ k = core.maximum(counts, 1)
+ threshold = core.take_along_axis(
+ sorted_vals, (k - 1)[..., None], axis=-1)
+ logits = core.where(logits < threshold, float("-inf"), logits)
+ if temperature <= 0:
+ # Deterministic decode (HF / llama.cpp semantics): temperature
+ # <= 0 means greedy — keep only the argmax (after any top-k
+ # truncation above), so categorical always draws it.
+ logits = core.where(
+ logits < core.max(logits), float("-inf"), logits)
+ return logits
+
+
+def sample(logits, temperature=0.7, top_k=None, top_p=None,
+ repetition_penalty=1.0, history=(), seed=None) -> int:
+ """Sample one token id from raw logits (HF-style penalty).
+
+ ``history`` is an iterable of already-generated token ids; the
+ penalty applies to them (vectorized, no per-token host syncs).
+ """
+ if repetition_penalty != 1.0 and history:
+ logits = logits.astype(core.float32)
+ ids = core.array(sorted(set(int(x) for x in history)),
+ dtype=core.int32)
+ if ids.size:
+ vals = core.take(logits, ids)
+ logits[ids] = core.where(
+ vals > 0,
+ vals / repetition_penalty,
+ vals * repetition_penalty)
+ logits = _mask_logits(logits, temperature, top_k, top_p)
+ if seed is not None:
+ core.random.seed(seed)
+ return int(core.random.categorical(logits[None]).item())
diff --git a/src/edge0/server/__init__.py b/src/edge0/server/__init__.py
new file mode 100644
index 0000000..473606a
--- /dev/null
+++ b/src/edge0/server/__init__.py
@@ -0,0 +1,12 @@
+"""edge0 serving layer: OpenAI-compatible chat endpoint over one engine."""
+
+from edge0.server.chat import (ChatMessage, ChatRequest, ChatSession,
+ QueueServer, decode_tokens, parse_chat_request,
+ sse_format)
+from edge0.server.app import (create_app, run_server, run_stdlib)
+
+__all__ = [
+ "ChatMessage", "ChatRequest", "ChatSession", "QueueServer",
+ "decode_tokens", "parse_chat_request", "sse_format",
+ "create_app", "run_server", "run_stdlib",
+]
diff --git a/src/edge0/server/app.py b/src/edge0/server/app.py
new file mode 100644
index 0000000..0de46a9
--- /dev/null
+++ b/src/edge0/server/app.py
@@ -0,0 +1,226 @@
+"""HTTP app layer (Flask optional; stdlib fallback).
+
+Preference order: if ``flask`` is importable it is used (the full
+OpenAI-compatible surface with SSE streaming); otherwise a minimal
+stdlib ``http.server`` handler serves the same JSON contract
+(non-streaming). Both share ``build_app_handlers`` so the route
+semantics stay identical.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import urlparse
+
+from edge0.server.chat import (QueueServer, parse_chat_request, sse_format,
+ decode_tokens)
+
+try: # pragma: no cover - environment dependent
+ from flask import Flask, Response, jsonify, request # type: ignore
+
+ _HAS_FLASK = True
+except ImportError: # pragma: no cover
+ _HAS_FLASK = False
+
+
+def _error(status: int, msg: str) -> tuple:
+ if _HAS_FLASK:
+ return jsonify({"error": {"message": msg, "type": "invalid_request"}}), status
+ return (json.dumps({"error": {"message": msg}}), status)
+
+
+def _split_think(text: str, think: bool):
+ """ling-mlx-server server.py parity: with thinking enabled the model
+ streams the reasoning block first and closes it with ``</think>``
+ (token 156904); everything before is ``reasoning_content``, after
+ is ``content``. A generation cut off inside the reasoning block
+ surfaces what it has so the UI isn't a blank wait."""
+ if think:
+ if "</think>" in text:
+ r, c = text.split("</think>", 1)
+ return r.strip(), c.strip()
+ return text.strip(), ""
+ return "", text
+
+
+def _chat_once(server: QueueServer, payload: dict):
+ req = parse_chat_request(payload)
+ tokens, meta = server.chat(req)
+ text = decode_tokens(server.engine, tokens)
+ think = bool(req.enable_thinking if req.enable_thinking is not None
+ else getattr(server.engine, "think", False))
+ reasoning, content = _split_think(text, think)
+ return {
+ "id": f"chatcmpl-{int(time.time() * 1000)}",
+ "object": "chat.completion",
+ "created": int(time.time()),
+ "model": server.model_name,
+ "choices": [{
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": content,
+ "reasoning_content": reasoning,
+ },
+ "finish_reason": "stop",
+ }],
+ "usage": meta["usage"],
+ }
+
+
+def _chat_stream(server: QueueServer, payload: dict):
+ req = parse_chat_request(payload)
+ events: list[bytes] = []
+
+ def on_token(tid: int):
+ text = decode_tokens(server.engine, [tid])
+ events.append(sse_format({
+ "id": "chatcmpl-1", "object": "chat.completion.chunk",
+ "created": int(time.time()), "model": server.model_name,
+ "choices": [{"index": 0,
+ "delta": {"content": text},
+ "finish_reason": None}],
+ }).encode("utf-8"))
+
+ tokens, meta = server.chat(req, on_token=on_token)
+ events.append(sse_format({
+ "id": "chatcmpl-1", "object": "chat.completion.chunk",
+ "created": int(time.time()), "model": server.model_name,
+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+ "usage": meta["usage"],
+ }).encode("utf-8"))
+ events.append(b"data: [DONE]\n\n")
+ return b"".join(events)
+
+
+def build_app_handlers(server: QueueServer):
+ """Return a handler dispatch dict shared by both transports."""
+
+ def handle_chat(payload: dict):
+ if payload.get("stream"):
+ if _HAS_FLASK:
+ return Response(_chat_stream(server, payload),
+ mimetype="text/event-stream")
+ return (_chat_stream(server, payload), 200,
+ {"Content-Type": "text/event-stream"})
+ return _chat_once(server, payload)
+
+ handlers = {
+ "GET /healthz": lambda: {"status": "ok", "model": server.model_name},
+ "GET /v1/models": lambda: {
+ "object": "list",
+ "data": [{
+ "id": server.model_name,
+ "object": "model",
+ "owned_by": "edge0",
+ }],
+ },
+ "POST /v1/chat/completions": handle_chat,
+ "POST /v1/completions": lambda p: _error(
+ 400, "text completions not supported; use /v1/chat/completions"),
+ }
+ return handlers
+
+
+def create_app(server: QueueServer):
+ """Flask app (raises ImportError when flask is missing)."""
+ if not _HAS_FLASK:
+ raise ImportError("flask is required for create_app(); "
+ "use run_stdlib instead")
+ app = Flask("edge0")
+ handlers = build_app_handlers(server)
+
+ @app.get("/healthz")
+ def healthz():
+ return handlers["GET /healthz"]()
+
+ @app.get("/v1/models")
+ def models():
+ return handlers["GET /v1/models"]()
+
+ @app.post("/v1/chat/completions")
+ def chat():
+ payload = request.get_json(force=True, silent=True) or {}
+ out = handlers["POST /v1/chat/completions"](payload)
+ if isinstance(out, Response):
+ return out
+ if isinstance(out, tuple) and out and isinstance(out[0], Response):
+ return out
+ return jsonify(out)
+
+ @app.post("/v1/completions")
+ def completions():
+ payload = request.get_json(force=True, silent=True) or {}
+ out = handlers["POST /v1/completions"](payload)
+ if isinstance(out, tuple):
+ return out
+ return jsonify(out)
+
+ return app
+
+
+class _StdlibHandler(BaseHTTPRequestHandler):
+ server_q = None # type: QueueServer
+
+ def _json(self, status: int, obj):
+ body = json.dumps(obj, ensure_ascii=False).encode("utf-8")
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def do_GET(self): # noqa: N802
+ path = urlparse(self.path).path
+ handlers = build_app_handlers(self.server_q)
+ if path == "/healthz":
+ self._json(200, handlers["GET /healthz"]())
+ elif path == "/v1/models":
+ self._json(200, handlers["GET /v1/models"]())
+ else:
+ self._json(404, {"error": {"message": f"no route {path}"}})
+
+ def do_POST(self): # noqa: N802
+ path = urlparse(self.path).path
+ handlers = build_app_handlers(self.server_q)
+ if path not in ("/v1/chat/completions", "/v1/completions"):
+ self._json(404, {"error": {"message": f"no route {path}"}})
+ return
+ length = int(self.headers.get("Content-Length", 0))
+ payload = json.loads(self.rfile.read(length) or b"{}")
+ out = handlers[("POST " + path)](payload)
+ if isinstance(out, tuple):
+ body, status, headers = out
+ self.send_response(status)
+ for k, v in headers.items():
+ self.send_header(k, v)
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+ else:
+ self._json(200, out)
+
+ def log_message(self, fmt, *args): # quiet by default
+ pass
+
+
+def run_stdlib(server: QueueServer, host: str, port: int):
+ handler = type("Edge0Handler", (_StdlibHandler,),
+ {"server_q": server})
+ httpd = ThreadingHTTPServer((host, port), handler)
+ httpd.serve_forever()
+
+
+def run_server(server: QueueServer, host: str = "127.0.0.1",
+ port: int = 8000, use_flask: bool | None = None):
+ """Run the HTTP server in the foreground (blocking)."""
+ if use_flask is None:
+ use_flask = _HAS_FLASK
+ if use_flask:
+ app = create_app(server)
+ app.run(host=host, port=port, threaded=True)
+ else:
+ run_stdlib(server, host, port)
diff --git a/src/edge0/server/chat.py b/src/edge0/server/chat.py
new file mode 100644
index 0000000..3babc68
--- /dev/null
+++ b/src/edge0/server/chat.py
@@ -0,0 +1,174 @@
+"""edge0 HTTP server: a single-process, one-model OpenAI-style endpoint.
+
+The engine is exclusive to one request at a time (a single MLX process
+can stream one generation well; concurrent requests would thrash the
+expert cache). Requests queue on a lock: each generation streams
+token-by-token over SSE, and the queue drains in FIFO order.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from edge0.engine.base import Edge0Engine
+
+
+@dataclass
+class ChatMessage:
+ role: str
+ content: str
+
+
+@dataclass
+class ChatRequest:
+ model: str
+ messages: list[ChatMessage]
+ temperature: float | None = None
+ top_p: float | None = None
+ top_k: int | None = None
+ max_tokens: int | None = None
+ seed: int | None = None
+ stream: bool = False
+ enable_thinking: bool | None = None
+ raw: dict[str, Any] = field(default_factory=dict)
+
+
+def parse_chat_request(payload: dict) -> ChatRequest:
+ msgs = []
+ for m in payload.get("messages", []):
+ role = str(m.get("role", "user"))
+ content = m.get("content", "")
+ if isinstance(content, list): # multi-part content: join text parts
+ content = "".join(
+ p.get("text", "") for p in content if isinstance(p, dict))
+ msgs.append(ChatMessage(role=role, content=str(content)))
+ return ChatRequest(
+ model=str(payload.get("model", "")),
+ messages=msgs,
+ temperature=payload.get("temperature"),
+ top_p=payload.get("top_p"),
+ top_k=payload.get("top_k"),
+ max_tokens=payload.get("max_tokens"),
+ seed=payload.get("seed"),
+ stream=bool(payload.get("stream", False)),
+ enable_thinking=payload.get("enable_thinking"),
+ raw=payload,
+ )
+
+
+class ChatSession:
+ """Tokenize + chat-template + generate for one request."""
+
+ def __init__(self, engine: Edge0Engine, req: ChatRequest):
+ self.engine = engine
+ self.req = req
+ self._tok = engine._tok
+
+ def prompt_ids(self) -> list[int]:
+ tok = self._tok
+ # ling-mlx-server parity (THINK_FIX_20260907.md): families with a
+ # vendored chat template (ling ``engine.encode_chat`` renders the
+ # deployment chat_template.jinja with enable_thinking) MUST go
+ # through it — the tokenizer's own apply_chat_template renders a
+ # DIFFERENT prompt (no think-mode system line, no ``<think>``
+ # opener), which is exactly the wrong-template bug.
+ encode = getattr(self.engine, "encode_chat", None)
+ if callable(encode):
+ think = self.req.enable_thinking
+ if think is None:
+ think = getattr(self.engine, "think", False)
+ return encode(
+ [m.__dict__ for m in self.req.messages], think=bool(think))
+ if hasattr(tok, "apply_chat_template"):
+ # Deployment parity with the correct demo (serve.py
+ # encode_chat): the qwen35 template's default IS the thinking
+ # mode (<think> opener). Passing enable_thinking=False on
+ # this template renders an EMPTY <think>\n\n</think> block
+ # instead of omitting thinking — a different prompt than the
+ # demo, so it must not be passed.
+ text = tok.apply_chat_template(
+ [m.__dict__ for m in self.req.messages],
+ tokenize=False, add_generation_prompt=True)
+ else:
+ text = self._chat_text()
+ ids = tok.encode(text)
+ if not ids:
+ ids = [tok.bos_token_id or 0]
+ return ids
+
+ def _chat_text(self) -> str:
+ parts = []
+ for m in self.req.messages:
+ if m.role == "system":
+ parts.append(f"<|im_start|>system\n{m.content}<|im_end|>\n")
+ elif m.role == "user":
+ parts.append(f"<|im_start|>user\n{m.content}<|im_end|>\n")
+ elif m.role == "assistant":
+ parts.append(f"<|im_start|>assistant\n{m.content}<|im_end|>\n")
+ parts.append("<|im_start|>assistant\n")
+ return "".join(parts)
+
+ def gen_config(self):
+ from edge0.config import GenerationConfig
+ base = getattr(self.engine.cfg, "gen", GenerationConfig())
+ kw = {}
+ for k in ("temperature", "top_p", "top_k", "max_new_tokens", "seed"):
+ v = getattr(self.req, {
+ "max_new_tokens": "max_tokens",
+ }.get(k, k), None)
+ if v is not None:
+ kw[k] = v
+ return GenerationConfig(**{**base.__dict__, **kw})
+
+ def run(self, on_token=None) -> tuple[list[int], dict]:
+ t0 = time.perf_counter()
+ # ling-mlx-server parity (THINK_FIX_20260907.md §3.1): clean
+ # per-request state. A previous degenerate/truncated turn leaves
+ # bad pregate cross-token state and KV behind, which makes every
+ # later request collapse from its first token.
+ self.engine.reset()
+ ids = self.prompt_ids()
+ gen = self.gen_config()
+ tokens = self.engine.generate(
+ ids, gen_config=gen, on_token=on_token)
+ usage = {
+ "prompt_tokens": len(ids),
+ "completion_tokens": len(tokens),
+ "total_tokens": len(ids) + len(tokens),
+ }
+ meta = {"wall_s": round(time.perf_counter() - t0, 3)}
+ return tokens, {"usage": usage, **meta}
+
+
+class QueueServer:
+ """Single-slot serving loop: serialize generations, stream via callback."""
+
+ def __init__(self, engine: Edge0Engine, model_name: str | None = None):
+ self.engine = engine
+ self.model_name = model_name or engine.name
+ self._lock = threading.Lock()
+
+ def health(self) -> dict:
+ st = self.engine.stats()
+ return {"status": "ok", "model": self.model_name,
+ "pos": self.engine.pos}
+
+ def chat(self, req: ChatRequest, on_token=None) -> dict:
+ with self._lock:
+ sess = ChatSession(self.engine, req)
+ return sess.run(on_token=on_token)
+
+
+def sse_format(data: dict) -> str:
+ return f"data: {json.dumps(data, ensure_ascii=False)}\n\n"
+
+
+def decode_tokens(engine: Edge0Engine, tokens: list[int]) -> str:
+ tok = engine._tok
+ if tok is None:
+ return ""
+ return tok.decode(tokens)
diff --git a/src/edge0/streaming/__init__.py b/src/edge0/streaming/__init__.py
new file mode 100644
index 0000000..8e412b0
--- /dev/null
+++ b/src/edge0/streaming/__init__.py
@@ -0,0 +1,17 @@
+"""Streaming expert offload: mmap stores, shared LRU caches, staged
+decode slots, whole-layer prefill and hot-expert pins."""
+
+from edge0.streaming.cache import PrefetchBuffer, SharedExpertCache
+from edge0.streaming.install import install_streaming_experts
+from edge0.streaming.layer import StreamingSwitchGLU
+from edge0.streaming.mmap import SafetensorsMmap
+from edge0.streaming.options import LayerOptions
+
+__all__ = [
+ "SafetensorsMmap",
+ "SharedExpertCache",
+ "PrefetchBuffer",
+ "StreamingSwitchGLU",
+ "LayerOptions",
+ "install_streaming_experts",
+]
diff --git a/src/edge0/streaming/cache.py b/src/edge0/streaming/cache.py
new file mode 100644
index 0000000..d426768
--- /dev/null
+++ b/src/edge0/streaming/cache.py
@@ -0,0 +1,81 @@
+"""Expert-bundle caches shared by all streaming MoE layers.
+
+* ``SharedExpertCache`` — global LRU across layers (one shared memory
+ budget: a layer that thrashes evicts entries from other layers, which is
+ exactly the behavior that keeps the whole-model budget flat).
+* ``PrefetchBuffer`` — cap-bounded buffer of eagerly prefetched bundles
+ that have NOT yet been promoted into the LRU.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections import OrderedDict
+
+
+class SharedExpertCache:
+ """Global LRU across all MoE layers (shared memory budget)."""
+
+ def __init__(self, slots: int):
+ self.slots = max(0, slots)
+ self._cache = OrderedDict()
+ self._lock = threading.Lock()
+
+ def get(self, key):
+ with self._lock:
+ v = self._cache.get(key)
+ if v is not None:
+ self._cache.move_to_end(key)
+ return v
+
+ def peek(self, key):
+ with self._lock:
+ return self._cache.get(key)
+
+ def put(self, key, bundle):
+ with self._lock:
+ self._cache[key] = bundle
+ if self.slots > 0 and len(self._cache) > self.slots:
+ self._cache.popitem(last=False)
+
+ def pop(self, key):
+ with self._lock:
+ return self._cache.pop(key, None)
+
+ def size(self):
+ with self._lock:
+ return len(self._cache)
+
+
+class PrefetchBuffer:
+ """Cap-bounded staging area for prefetched bundles.
+
+ ``put`` evicts the oldest entry when over capacity (counted as
+ ``prefetch_wasted`` by the caller — the build was spent, the bundle was
+ never used). ``pop`` removes a bundle when the layer actually consumes
+ it (it is then promoted into the shared LRU by the caller).
+ """
+
+ def __init__(self, cap: int = 48):
+ self.cap = max(0, cap)
+ self._buf = OrderedDict()
+ self._lock = threading.Lock()
+
+ def put(self, key, bundle):
+ with self._lock:
+ self._buf[key] = bundle
+ evicted = None
+ if self.cap > 0 and len(self._buf) > self.cap:
+ evicted = self._buf.popitem(last=False)
+ return evicted[0] if evicted is not None else None
+
+ def pop(self, key):
+ with self._lock:
+ return self._buf.pop(key, None)
+
+ def contains(self, key) -> bool:
+ with self._lock:
+ return key in self._buf
+
+ def set_cap(self, cap: int):
+ self.cap = max(0, cap)
diff --git a/src/edge0/streaming/install.py b/src/edge0/streaming/install.py
new file mode 100644
index 0000000..37b0e7e
--- /dev/null
+++ b/src/edge0/streaming/install.py
@@ -0,0 +1,106 @@
+"""Install streaming MoE blocks into a loaded base model.
+
+Generalizes the deployment's ``install_streaming_experts``: instead of
+hard-coding a walk of ``model.layers``, the walk is driven by ``MoESpec``
+(``block_path`` template + num layers). The returned list of
+``StreamingSwitchGLU`` objects is what the engine, prerouter stager, and
+prefill hooks drive.
+"""
+
+from __future__ import annotations
+
+from edge0.backends import core
+
+from edge0.moe.spec import MoESpec
+from edge0.streaming.cache import PrefetchBuffer, SharedExpertCache
+from edge0.streaming.layer import StreamingSwitchGLU
+from edge0.streaming.mmap import SafetensorsMmap
+from edge0.streaming.options import LayerOptions
+
+
+def install_streaming_experts(
+ model,
+ shards: list[SafetensorsMmap],
+ spec: MoESpec,
+ options: LayerOptions | None = None,
+ num_layers: int | None = None,
+ shared_cache: SharedExpertCache | None = None,
+ prefetch_buffer: PrefetchBuffer | None = None,
+) -> list[StreamingSwitchGLU]:
+ """Replace every routed-expert block in ``model`` with its streaming
+ twin and return the twins in layer order.
+
+ ``num_layers`` defaults to ``spec``-independent discovery: the block
+ path template is probed with increasing layer indices until it stops
+ resolving. The original blocks are kept on the objects as
+ ``_edge0_resident`` (used by the bit-parity tests); nothing else in the
+ base model is touched.
+ """
+ if num_layers is None:
+ n = 0
+ while True:
+ try:
+ spec.block_of(model, n)
+ except AttributeError:
+ break
+ n += 1
+ if n == 0:
+ raise ValueError(
+ f"block_path {spec.block_path!r} resolves no layer 0 — "
+ "check MoESpec.block_path")
+ num_layers = n
+
+ opts = options or LayerOptions()
+ cache = shared_cache or SharedExpertCache(opts.cache_slots)
+ buf = prefetch_buffer or PrefetchBuffer(opts.prefetch_cap)
+
+ stream_layers: list[StreamingSwitchGLU | None] = []
+ for i in range(num_layers):
+ block = spec.block_of(model, i)
+ if not hasattr(block, "experts") and not hasattr(
+ block, "switch_mlp"):
+ # Dense (non-routed) layer — nothing to stream.
+ stream_layers.append(None)
+ continue
+ twin = StreamingSwitchGLU(
+ shards, i, spec, options=opts,
+ shared_cache=cache, prefetch_buffer=buf)
+ # keep the resident block reachable (bit-parity tests, debugging)
+ block._edge0_resident = block
+ # swap the twin into the block, following the family convention:
+ # qwen-style blocks call ``switch_mlp(x, inds)``; ling-style blocks
+ # hold ``experts = SwitchGLU(...)`` and call ``experts(x, idx)``.
+ # The original submodule is stashed for parity tests.
+ if hasattr(block, "switch_mlp"):
+ block._edge0_resident_switch = block.switch_mlp
+ block.switch_mlp = twin
+ else:
+ block._edge0_resident_switch = block.experts
+ block.experts = twin
+ if opts.top_k is not None:
+ res = getattr(block, "_edge0_resident", block)
+ if hasattr(res, "top_k"):
+ res.top_k = opts.top_k
+ stream_layers.append(twin)
+ return stream_layers
+
+
+def attach_streaming(block, twin) -> None:
+ """Point a MoE block's ``__call__`` at its streaming twin.
+
+ For vendored models whose layer code calls ``self.mlp(...)`` directly,
+ the installer swaps the block object itself (``setattr`` on the layer);
+ for models whose blocks carry a ``__call__`` the engine may patch it
+ class-level. This helper exists so both paths share one spelling.
+ """
+ twin._edge0_resident = getattr(block, "_edge0_resident", block)
+ block._edge0_stream = twin
+
+
+def route_indices(mx_array) -> core.array:
+ """Sanity guard: indices must be a GPU mx array when they leave the
+ block (the whole staged design assumes no per-layer host syncs)."""
+ assert isinstance(mx_array, core.array), (
+ "router indices must stay on GPU (core.array), got "
+ f"{type(mx_array).__name__}")
+ return mx_array
diff --git a/src/edge0/streaming/layer.py b/src/edge0/streaming/layer.py
new file mode 100644
index 0000000..9ac9199
--- /dev/null
+++ b/src/edge0/streaming/layer.py
@@ -0,0 +1,1395 @@
+"""StreamingSwitchGLU: the SSD-offloaded MoE expert block.
+
+Faithful port of the deployment's ``streaming_experts_qwen35.py``
+(itself a port of the ling ``stream_experts.py`` design), generalized so
+ONE class serves every supported model:
+
+* expert weight layout (separate vs fused gate+up projections) and
+ safetensors key prefix come from ``MoESpec``;
+* every behavior knob (staged decode, hot pins, prefetch, compile,
+ incremental stack, whole-layer prefill) comes from ``LayerOptions``
+ instead of env vars.
+
+Public surface consumed by the engine / prerouter / prefill hooks:
+
+ load_full_layer / clear_full_layer / load_hot_layer /
+ materialize_hot / dematerialize_hot / clear_hot_layer
+ stage_experts / wait_staged / swap_staged / sync_actuals /
+ stage_from_prefill / stage_from_history
+ prefetch / prefetch_all / warm_pages
+ stats / reset
+
+The ``__call__(x, indices)`` routing:
+
+1. whole-layer prefill path — full stacked weights resident
+ (``_full_weights``), sorted gather, bit-identical to the resident
+ math; also records usage + last-token top-k for staging.
+2. hot-stack prefill path — resident top-N stack
+ (``_hot_weights``) with exact per-expert scatter-add for misses.
+3. staged decode path — routing through fixed staged slots with
+ a single GPU-side expert->slot table (``core.take``), no per-layer
+ host syncs; missing experts map to an all-zero overflow row
+ (production prerouter consumers have zero misses by construction:
+ the staged set IS the prerouter's prediction).
+4. exact decode path — per-expert bundles (pinned / shared LRU
+ / prefetch buffer / pool builds), sorted gather for large batches.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+
+from edge0.backends import core, quant
+from edge0.backends import nn
+import numpy as np
+
+from mlx_lm.models.switch_layers import _gather_sort, _scatter_unsort
+
+from edge0.moe.spec import MoESpec
+from edge0.streaming.cache import PrefetchBuffer, SharedExpertCache
+from edge0.streaming.mmap import SafetensorsMmap, u32_view
+from edge0.streaming.options import LayerOptions
+
+
+def _swiglu(up, gate):
+ return nn.silu(gate) * up
+
+
+class StreamingSwitchGLU:
+ """Streaming version of a routed-expert GLU block.
+
+ Only the routed experts are streamed. Shared experts, routers, and
+ attention stay resident in the base model.
+ """
+
+ def __init__(
+ self,
+ shards: list[SafetensorsMmap],
+ layer_idx: int,
+ spec: MoESpec,
+ options: LayerOptions | None = None,
+ shared_cache: SharedExpertCache | None = None,
+ prefetch_buffer: PrefetchBuffer | None = None,
+ ):
+ self.layer_idx = layer_idx
+ self.spec = spec
+ self.num_experts = spec.num_experts
+ self.group_size = spec.quant.group_size
+ self.bits = spec.quant.bits
+ self.mode = spec.quant.mode
+ self._fuse_gu = spec.fuse_gu
+ self._bundle_projs = spec.bundle_projs
+
+ o = options or LayerOptions()
+ self._staged_mode = o.staged
+ self._staged_replace = o.staged_replace
+ self._staged_n = max(1, o.staged_n)
+ # Router top-k that triggers the staged path (single-token decode);
+ # set by the engine to the routed top_k (usually the staged_n).
+ self._staged_trigger = o.staged_trigger
+ self._staged_sync = o.staged_sync
+ self.hot_per_layer = o.hot_per_layer
+ self.hot_update_interval = o.hot_update_interval
+ self.hot_decay = o.hot_decay
+ self._pin_bonus = o.pin_bonus
+ self._use_compile = o.use_compile
+
+ self._pool = ThreadPoolExecutor(max_workers=o.load_threads)
+ self._prefetch_pool = ThreadPoolExecutor(
+ max_workers=max(1, o.prefetch_threads))
+
+ self._shards = shards
+ self._shard_idx = {
+ name: i
+ for i, s in enumerate(shards)
+ for name in s.entries
+ }
+ self._prefix = spec.key_template.format(layer=layer_idx)
+ self._shape = {}
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ for part in ("weight", "scales", "biases"):
+ name = f"{self._prefix}.{proj}.{part}"
+ self._shape[(proj, part)] = self._shards[
+ self._shard_idx[name]].entries[name]["shape"]
+ # Bundle-format shapes: what _build produces / caches store / math
+ # consumes. Fused gate_up = gate rows on top of up rows along axis 1.
+ if self._fuse_gu:
+ self._bundle_shape = {}
+ for part in ("weight", "scales", "biases"):
+ gs = self._shape[("gate_proj", part)]
+ self._bundle_shape[("gate_up_proj", part)] = (
+ gs[0], 2 * gs[1]) + tuple(gs[2:])
+ self._bundle_shape[("down_proj", part)] = self._shape[
+ ("down_proj", part)]
+ else:
+ self._bundle_shape = self._shape
+
+ self._pinned = {}
+ self._hot_counts = {}
+ self._calls = 0
+ self._full_weights: dict | None = None
+ self._hot_weights: dict | None = None
+ self._hot_backing: dict | None = None
+ self._hot_key: tuple | None = None
+ self._hot_set: set | None = None
+ self.predicted_experts: set[int] = set()
+ self._lock = threading.Lock()
+ self.last_used: list[int] = []
+ self._stats = {"loads": 0, "hits": 0, "load_wall": 0.0}
+ self.shared_cache = shared_cache or SharedExpertCache(o.cache_slots)
+ self._prefetch_buf = prefetch_buffer or PrefetchBuffer(o.prefetch_cap)
+ self._inflight: dict[int, object] = {}
+ self._stats.update({
+ "prefetch_submitted": 0,
+ "prefetch_hits": 0,
+ "prefetch_wasted": 0,
+ "prefetch_wait": 0.0,
+ })
+
+ # ---- MoE math (core.compile variants) ------------------------------
+
+ def _make_moe_math():
+ gs, bt, md = self.group_size, self.bits, self.mode
+
+ @core.compile
+ def fn(x, w_u, s_u, b_u, w_g, s_g, b_g, w_d, s_d, b_d, local):
+ xe = core.expand_dims(x, (-2, -3))
+ x_up = quant.gather_qmm(
+ xe, w_u, s_u, b_u, rhs_indices=local,
+ transpose=True, group_size=gs, bits=bt, mode=md,
+ sorted_indices=False)
+ x_gate = quant.gather_qmm(
+ xe, w_g, s_g, b_g, rhs_indices=local,
+ transpose=True, group_size=gs, bits=bt, mode=md,
+ sorted_indices=False)
+ z = quant.gather_qmm(
+ _swiglu(x_up, x_gate), w_d, s_d, b_d,
+ rhs_indices=local, transpose=True, group_size=gs,
+ bits=bt, mode=md, sorted_indices=False)
+ return z.squeeze(-2)
+ return fn
+
+ def _make_moe_math_sorted():
+ gs, bt, md = self.group_size, self.bits, self.mode
+
+ @core.compile
+ def fn(x, w_u, s_u, b_u, w_g, s_g, b_g, w_d, s_d, b_d,
+ local, inv_order, out_shape):
+ x_up = quant.gather_qmm(
+ x, w_u, s_u, b_u, rhs_indices=local,
+ transpose=True, group_size=gs, bits=bt, mode=md,
+ sorted_indices=True)
+ x_gate = quant.gather_qmm(
+ x, w_g, s_g, b_g, rhs_indices=local,
+ transpose=True, group_size=gs, bits=bt, mode=md,
+ sorted_indices=True)
+ z = quant.gather_qmm(
+ _swiglu(x_up, x_gate), w_d, s_d, b_d,
+ rhs_indices=local, transpose=True, group_size=gs,
+ bits=bt, mode=md, sorted_indices=True)
+ return _scatter_unsort(z, inv_order, out_shape).squeeze(-2)
+ return fn
+
+ def _make_moe_math_fused():
+ gs, bt, md = self.group_size, self.bits, self.mode
+
+ @core.compile
+ def fn(x, w_gu, s_gu, b_gu, w_d, s_d, b_d, local):
+ xe = core.expand_dims(x, (-2, -3))
+ x_gu = quant.gather_qmm(
+ xe, w_gu, s_gu, b_gu, rhs_indices=local,
+ transpose=True, group_size=gs, bits=bt, mode=md,
+ sorted_indices=False)
+ x_gate, x_up = core.split(x_gu, 2, axis=-1)
+ z = quant.gather_qmm(
+ _swiglu(x_up, x_gate), w_d, s_d, b_d,
+ rhs_indices=local, transpose=True, group_size=gs,
+ bits=bt, mode=md, sorted_indices=False)
+ return z.squeeze(-2)
+ return fn
+
+ def _make_moe_math_sorted_fused():
+ gs, bt, md = self.group_size, self.bits, self.mode
+
+ @core.compile
+ def fn(x, w_gu, s_gu, b_gu, w_d, s_d, b_d,
+ local, inv_order, out_shape):
+ x_gu = quant.gather_qmm(
+ x, w_gu, s_gu, b_gu, rhs_indices=local,
+ transpose=True, group_size=gs, bits=bt, mode=md,
+ sorted_indices=True)
+ x_gate, x_up = core.split(x_gu, 2, axis=-1)
+ z = quant.gather_qmm(
+ _swiglu(x_up, x_gate), w_d, s_d, b_d,
+ rhs_indices=local, transpose=True, group_size=gs,
+ bits=bt, mode=md, sorted_indices=True)
+ return _scatter_unsort(z, inv_order, out_shape).squeeze(-2)
+ return fn
+
+ self._moe_math = _make_moe_math()
+ self._moe_math_sorted = _make_moe_math_sorted()
+ if self._fuse_gu:
+ # Fused gate+up: replace the math fns with the fused variants
+ # (same pattern as the original streaming_experts_qwen35.py) —
+ # every call site passes its wargs through unchanged.
+ self._moe_math = _make_moe_math_fused()
+ self._moe_math_sorted = _make_moe_math_sorted_fused()
+
+ # ---- staged decode ------------------------------------------------
+ # Port of ling stream_experts.py R1 fixed-slot staging. The router's
+ # indices stay on GPU: a CPU-built [num_experts] expert->slot table
+ # maps them into the staged tensor via core.take, so a whole decode
+ # step builds/evaluates with a single sync (the final logits eval)
+ # instead of 40 per-layer tolist barriers. Missing experts map to
+ # the overflow zero slot (contribution dropped) — with a prerouter
+ # the staged set IS the routing, so this never fires in production.
+ # Double-buffered staged slots: `_staged_state` is the ACTIVE set
+ # read by the current decode step's __call__ (never disturbed
+ # mid-step); `_staged_state_next` is the fill target written by
+ # stage_experts, swapped in at the step boundary by swap_staged().
+ self._staged_state: tuple | None = None
+ self._staged_state_next: tuple | None = None
+ self._staging_lock = threading.Lock()
+ self._staging_future = None
+ self._staged_pending: list[int] | None = None
+ self._pending_indices: core.array | None = None
+ self._staged_at_use: set[int] | None = None
+ self._last_prefill_topk: list[int] | None = None
+ # Small cache of already-materialized (GPU) expert->slot tables.
+ self._slot_table_cache: dict[tuple, core.array] = {}
+ # Cache of the assembled (slot_of, stacked-wargs) pair per staged
+ # expert set, so decode steps whose staged set repeats don't rebuild
+ # the 9 core.stack nodes + slot table on every layer every step.
+ if not o.asm_cache:
+ class _NoAsmCache(dict):
+ def __setitem__(self, k, v): # never store
+ pass
+ self._staged_asm_cache = _NoAsmCache()
+ else:
+ self._staged_asm_cache = {}
+ # Persistent all-zero overflow slot rows (slot index = _staged_n),
+ # so missing experts map to a real zero row.
+ self._zero_slot = {}
+ for proj in self._bundle_projs:
+ for part in ("weight", "scales", "biases"):
+ shape = self._bundle_shape[(proj, part)][1:]
+ dtype = core.uint32 if part == "weight" else core.bfloat16
+ self._zero_slot[(proj, part)] = core.zeros(shape, dtype)
+ # ---- incremental stack --------------------------------------------
+ # Sticky-slot persistent staged tensors: the [n+1, ...] stacked
+ # weights are allocated ONCE per layer and only the slots whose
+ # expert changed are rewritten in place each step, replacing the
+ # 9 core.stack graph nodes per layer per step. Safe only when the
+ # previous forward's eval has finished (staged_sync, the
+ # production default) — the async double-buffered fill would race
+ # the writes.
+ self._incr_mode = (
+ o.incr_stack and self._staged_mode and self._staged_sync
+ and not self._staged_replace)
+ # Write-back LRU (dedup staged set from the shared LRU): while
+ # staged, an expert's bundle is held only by the incremental stack
+ # machinery; it enters the LRU only when evicted from the stack.
+ self._incr_writeback = (
+ self._incr_mode and o.incr_writeback)
+ self._incr_tensors = None # {(proj,part): core.array [n+1, ...]}
+ self._incr_slot_table = None # core.array [num_experts] int32
+ self._incr_occupancy = None # list[slot] -> expert id | None
+ self._incr_bundles = {} # expert -> bundle (current set)
+ self._stats.update({
+ "staged_used": 0,
+ "staged_dropped": 0,
+ "staged_fallback": 0,
+ "stage_wall": 0.0,
+ "stage_wait_ms": 0.0,
+ "stage_delta_hits": 0,
+ "stage_cache_hits": 0,
+ "stage_builds": 0,
+ "incr_writes": 0,
+ })
+
+ # ---- loading ---------------------------------------------------------
+
+ def _shard_for(self, name: str) -> SafetensorsMmap:
+ return self._shards[self._shard_idx[name]]
+
+ def _build(self, expert: int):
+ """Load one expert's 9 tensors (gate/up/down x weight/scales/biases).
+
+ Hot-stack fast path: if the expert is a member of the resident
+ hot stack, slice its rows from the stack instead of reading the
+ shard — zero IO, zero page-cache pressure."""
+ if self._hot_backing is not None and expert in self._hot_set:
+ idx = self._hot_key.index(expert)
+ shape = self._bundle_shape
+ b = {}
+ for (proj, part), buf in self._hot_backing.items():
+ sh = shape[(proj, part)]
+ per = buf.size // len(self._hot_key)
+ sl = buf[idx * per:(idx + 1) * per]
+ if part == "weight":
+ b[(proj, part)] = core.array(
+ sl.view("<u4").reshape(sh[1:]))
+ else:
+ b[(proj, part)] = core.array(sl.view("<u2")).view(
+ core.bfloat16).reshape(sh[1:])
+ return b
+ per_w = {}
+
+ def _read_rows(proj, part):
+ name = f"{self._prefix}.{proj}.{part}"
+ raw = self._shard_for(name).raw(name)
+ shape = self._shape[(proj, part)]
+ per = raw.size // self.num_experts
+ return raw[expert * per:(expert + 1) * per]
+
+ def _to_mx(sl, key):
+ part = key[1]
+ shape = self._bundle_shape[key]
+ if part == "weight":
+ return core.array(u32_view(sl, shape[1:]))
+ return core.array(sl.view("<u2")).view(
+ core.bfloat16).reshape(shape[1:])
+
+ if self._fuse_gu:
+ # gate rows on top of up rows (matches split(x_gu, 2) order);
+ # numpy-level concat of the raw byte slices before the single
+ # core.array per part (pool thread, off the critical path).
+ for part in ("weight", "scales", "biases"):
+ sl = np.concatenate([
+ _read_rows("gate_proj", part),
+ _read_rows("up_proj", part)])
+ per_w[("gate_up_proj", part)] = _to_mx(
+ sl, ("gate_up_proj", part))
+ for part in ("weight", "scales", "biases"):
+ per_w[("down_proj", part)] = _to_mx(
+ _read_rows("down_proj", part), ("down_proj", part))
+ return per_w
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ for part in ("weight", "scales", "biases"):
+ per_w[(proj, part)] = _to_mx(
+ _read_rows(proj, part), (proj, part))
+ return per_w
+
+ def warm_pages(self, experts):
+ """Fault expert mmap pages into the OS page cache without retaining
+ mx arrays (MLX active memory stays flat)."""
+ for e in experts:
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ for part in ("weight", "scales", "biases"):
+ name = f"{self._prefix}.{proj}.{part}"
+ raw = self._shard_for(name).raw(name)
+ per = raw.size // self.num_experts
+ sl = raw[e * per:(e + 1) * per]
+ _ = sl.sum()
+
+ def _get_bundles(self, experts):
+ """Resolve bundles: pinned/LRU hits synchronously, misses via pool."""
+ bundles = {}
+ missing = []
+ with self._lock:
+ for e in experts:
+ p = self._pinned.get(e)
+ if p is not None:
+ bundles[e] = p
+ continue
+ bundles[e] = None
+ missing.append(e)
+ still_missing = []
+ for e in missing:
+ cached = self.shared_cache.get((self.layer_idx, e))
+ if cached is not None:
+ self._stats["hits"] += 1
+ bundles[e] = cached
+ else:
+ still_missing.append(e)
+ missing = still_missing
+ resolved = set()
+ for e in missing:
+ with self._lock:
+ bundle = self._prefetch_buf.pop((self.layer_idx, e))
+ if bundle is not None:
+ self._stats["prefetch_hits"] += 1
+ self._stats["hits"] += 1
+ with self._lock:
+ self._hot_counts[e] = (
+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)
+ self.shared_cache.put((self.layer_idx, e), bundle)
+ bundles[e] = bundle
+ resolved.add(e)
+ missing = [e for e in missing if e not in resolved]
+ resolved = set()
+ for e in missing:
+ with self._lock:
+ fut = self._inflight.get(e)
+ if fut is not None:
+ t0 = time.perf_counter()
+ tensors = fut.result()
+ with self._lock:
+ self._inflight.pop(e, None)
+ self._stats["prefetch_wait"] += time.perf_counter() - t0
+ self._hot_counts[e] = (
+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)
+ self._stats["prefetch_hits"] += 1
+ self._stats["hits"] += 1
+ self.shared_cache.put((self.layer_idx, e), tensors)
+ bundles[e] = tensors
+ resolved.add(e)
+ missing = [e for e in missing if e not in resolved]
+ if missing:
+ t0 = time.perf_counter()
+ built = list(self._pool.map(self._build, missing))
+ wall = time.perf_counter() - t0
+ with self._lock:
+ for e, tensors in zip(missing, built):
+ self._stats["loads"] += 1
+ self._hot_counts[e] = (
+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)
+ bundles[e] = tensors
+ self._stats["load_wall"] += wall
+ for e, tensors in zip(missing, built):
+ self.shared_cache.put((self.layer_idx, e), tensors)
+ return bundles
+
+ # ---- prefetch --------------------------------------------------------
+
+ def prefetch(self, experts):
+ unique = sorted({int(e) for e in experts})
+ if not unique:
+ return
+ submitted = []
+ with self._lock:
+ missing = []
+ for e in unique:
+ if (
+ e in self._pinned
+ or e in self._inflight
+ or self.shared_cache.peek((self.layer_idx, e)) is not None
+ or self._prefetch_buf.contains((self.layer_idx, e))
+ ):
+ continue
+ missing.append(e)
+ for e in missing:
+ fut = self._prefetch_pool.submit(self._build, e)
+ self._inflight[e] = fut
+ submitted.append((e, fut))
+ self._stats["prefetch_submitted"] += len(missing)
+ # Register the done-callback OUTSIDE the lock: if the build finished
+ # before submit() returned, add_done_callback fires synchronously on
+ # this thread, and _on_prefetch_done re-acquires the same non-reentrant
+ # lock — deadlock. Moving the registration out fixes it.
+ for e, fut in submitted:
+ fut.add_done_callback(partial(self._on_prefetch_done, e))
+
+ def prefetch_all(self):
+ n = self.num_experts
+ if self._prefetch_buf.cap < n + 32:
+ self._prefetch_buf.set_cap(n + 32)
+ self.prefetch(range(n))
+
+ def _on_prefetch_done(self, expert, fut):
+ try:
+ bundle = fut.result()
+ except Exception:
+ with self._lock:
+ self._inflight.pop(expert, None)
+ return
+ with self._lock:
+ self._inflight.pop(expert, None)
+ if self.shared_cache.peek((self.layer_idx, expert)) is not None:
+ return
+ evicted = self._prefetch_buf.put((self.layer_idx, expert), bundle)
+ if evicted is not None:
+ self._stats["prefetch_wasted"] += 1
+
+ # ---- fixed-slot staging (staged decode) ------------------------------
+
+ def _build_staged_bundles(self, experts: list[int]):
+ """Resolve per-expert bundles for the next staged slots.
+
+ Runs on the thread pool. Reuses the previous staged state by
+ reference (delta reuse), then pinned / shared-LRU mx arrays; only
+ genuinely cold experts go through the raw mmap build. The core.stack
+ into fixed [n+1, ...] tensors is done on the main thread in __call__
+ because MLX lazy-op construction is not safe concurrently with eval.
+ """
+ _t0 = time.perf_counter()
+ n = self._staged_n
+ exp = sorted({int(e) for e in experts})[:n]
+ prev = {}
+ st = self._staged_state
+ if st is not None:
+ pbundles, _, pexp = st
+ for i, e in enumerate(pexp):
+ prev[e] = pbundles[i]
+ slot_of = [n] * self.num_experts
+ for i, e in enumerate(exp):
+ slot_of[e] = i
+ bundles = []
+ for e in exp:
+ b = prev.get(e)
+ if b is not None:
+ with self._lock:
+ self._stats["stage_delta_hits"] += 1
+ if b is None:
+ with self._lock:
+ b = self._pinned.get(e)
+ if b is None:
+ b = self.shared_cache.get((self.layer_idx, e))
+ if b is not None:
+ with self._lock:
+ self._stats["stage_cache_hits"] += 1
+ if b is None:
+ b = self._build(e)
+ # put the fresh build into the shared LRU so later steps
+ # hit the cache instead of rebuilding: the staged resolver
+ # above checks shared_cache BEFORE _build, so this closes
+ # the loop (delta miss -> LRU hit, no rebuild).
+ self.shared_cache.put((self.layer_idx, e), b)
+ with self._lock:
+ self._stats["stage_builds"] += 1
+ bundles.append(b)
+ wall = time.perf_counter() - _t0
+ return tuple(bundles), slot_of, exp, wall
+
+ def _stage_incr(self, experts: list[int]):
+ """Incremental-stack resolver: sticky slot assignment + in-place
+ delta writes into the persistent stacked tensors.
+
+ Keeps every surviving expert in its current slot (zero work), frees
+ the slots of evicted experts, and assigns newly arriving experts to
+ the freed slots — writing ONLY those slots' rows in place (one
+ __setitem__ per (proj, part) per changed slot) plus slot-table
+ updates. Bundle resolution cascade is identical to
+ _build_staged_bundles (pinned -> shared LRU -> raw build).
+ Runs on the main thread (sync staged mode only)."""
+ _t0 = time.perf_counter()
+ n = self._staged_n
+ exp = sorted({int(e) for e in experts})[:n]
+ exp_set = set(exp)
+
+ # Lazy one-time allocation of the persistent tensors. Slot n is the
+ # all-zero overflow row and is NEVER written after this init.
+ if self._incr_tensors is None:
+ self._incr_tensors = {}
+ for proj in self._bundle_projs:
+ for part in ("weight", "scales", "biases"):
+ shape = self._bundle_shape[(proj, part)][1:]
+ dtype = core.uint32 if part == "weight" else core.bfloat16
+ t = core.zeros((n + 1, *shape), dtype)
+ core.eval(t)
+ self._incr_tensors[(proj, part)] = t
+ self._incr_slot_table = core.full(
+ (self.num_experts,), n, dtype=core.int32)
+ core.eval(self._incr_slot_table)
+ self._incr_occupancy = [None] * n
+
+ occ = self._incr_occupancy
+ # Sticky assignment: survivors keep slots; evicted slots free up.
+ free = [i for i in range(n) if occ[i] is not None
+ and occ[i] not in exp_set]
+ for i in free:
+ occ[i] = None
+ free = [i for i in range(n) if occ[i] is None]
+ # slot-of list for compatibility consumers (_staged_state / stats).
+ slot_of_list = [n] * self.num_experts
+ for i in range(n):
+ e = occ[i]
+ if e is not None:
+ slot_of_list[e] = i
+
+ changed = [e for e in exp if e not in self._incr_bundles]
+ new_slots = []
+ for j, e in enumerate(changed):
+ slot = free[j] if j < len(free) else None
+ new_slots.append((e, slot))
+
+ writes = []
+ # Mirror the occupancy exactly: only the CURRENT staged set keeps
+ # bundles. A stale entry would hide a returning expert from
+ # `changed` (no slot assigned, no row write -> its contribution
+ # silently maps to the zero row).
+ bundles = {e: self._incr_bundles[e] for e in exp
+ if e in self._incr_bundles}
+ for e, slot in new_slots:
+ if slot is None:
+ # |exp| <= n guarantees a free slot for every changed
+ # expert (survivors <= |exp|), so this is unreachable;
+ # guard anyway: drop the expert rather than corrupt a slot.
+ continue
+ b = self._pinned.get(e)
+ if b is None:
+ b = self.shared_cache.get((self.layer_idx, e))
+ if b is not None:
+ with self._lock:
+ self._stats["stage_cache_hits"] += 1
+ if b is None:
+ b = self._build(e)
+ if not self._incr_writeback:
+ # eager mode: share the fresh build via the LRU
+ self.shared_cache.put((self.layer_idx, e), b)
+ with self._lock:
+ self._stats["stage_builds"] += 1
+ bundles[e] = b
+ occ[slot] = e
+ slot_of_list[e] = slot
+ writes.append((slot, b))
+ # Evicted experts: table back to the zero overflow slot.
+ evicted = [e for e in self._incr_bundles if e not in exp_set]
+ if self._incr_writeback:
+ # Write-back LRU: while an expert is staged its canonical bundle
+ # lives ONLY in _incr_bundles (the persistent stack rows are the
+ # math copy) — no LRU duplication of the staged set. On eviction
+ # from the stack the bundle is written back so returning experts
+ # still hit the LRU.
+ for e in evicted:
+ self.shared_cache.put((self.layer_idx, e),
+ self._incr_bundles[e])
+ self._incr_bundles = bundles
+
+ # ---- in-place delta writes (main thread, previous graph already
+ # evaluated: stage runs at the step boundary in sync mode) ----
+ for slot, b in writes:
+ for proj in self._bundle_projs:
+ for part in ("weight", "scales", "biases"):
+ self._incr_tensors[(proj, part)][slot] = b[(proj, part)]
+ tbl = self._incr_slot_table
+ for e in evicted:
+ tbl[e] = n
+ for e, slot in new_slots:
+ if slot is not None:
+ tbl[e] = slot
+ # FIX(2026-09-02): force-evaluate the slot table after in-place
+ # mutations so the consuming forward's lazy graph (core.take /
+ # _moe_math) reads the UPDATED mapping — without this, near-100%
+ # slot churn reads stale slot tables and the output collapses.
+ if writes or evicted or new_slots:
+ core.eval(self._incr_slot_table)
+ with self._lock:
+ self._stats["incr_writes"] += len(writes)
+ self._stats["stage_delta_hits"] += len(exp) - len(writes)
+ self._stats["stage_wall"] += time.perf_counter() - _t0
+
+ # Compatibility state: bundles tuple ordered like the old resolver
+ # (sync_actuals promotion + fallback paths read it).
+ return (tuple(bundles[e] for e in exp), slot_of_list, exp)
+
+ def stage_experts(self, experts):
+ """Async-fill the NEXT slot set for the following decode step.
+
+ Writes to `_staged_state_next` (never the active `_staged_state`), so
+ the fill for token t+1 can run during token t without disturbing the
+ set token t is reading. swap_staged() promotes it at the step
+ boundary. If the fill cannot land in time, the consuming __call__
+ falls back to the exact per-layer path (staged_fallback).
+ """
+ if not self._staged_mode:
+ return
+ unique = sorted({int(e) for e in experts})
+ if not unique:
+ return
+ self._drain_rearm()
+ with self._staging_lock:
+ if self._staging_future is not None:
+ self._staged_pending = unique
+ return
+ if self._staged_sync:
+ if self._incr_mode:
+ self._staged_state_next = self._stage_incr(unique)
+ else:
+ bundles, slot_of, exp, wall = self._build_staged_bundles(
+ unique)
+ self._staged_state_next = (tuple(bundles), slot_of, exp)
+ with self._lock:
+ self._stats["stage_wall"] += wall
+ return
+ self._submit_stage_locked(unique)
+
+ def wait_staged(self):
+ """Block until the in-flight staged fill lands, then promote it to
+ the active state (ling stream_experts.py:441-455 semantics: the
+ fill's completion IS the promotion — no separate blocking
+ step-boundary swap; each layer waits lazily at consume time so the
+ pool's SSD builds overlap the forward's graph building)."""
+ self._drain_rearm()
+ with self._staging_lock:
+ fut = self._staging_future
+ if fut is not None:
+ t0 = time.perf_counter()
+ try:
+ fut.result()
+ except Exception:
+ pass
+ with self._lock:
+ self._stats["stage_wait_ms"] += (
+ time.perf_counter() - t0) * 1000
+ with self._staging_lock:
+ if self._staged_state_next is not None:
+ self._staged_state = self._staged_state_next
+ self._staged_state_next = None
+
+ def swap_staged(self):
+ """Promote the filled next set to active at the step boundary.
+
+ Blocks until the in-flight fill lands (the exposed remainder of the
+ hidden fill), then promotes. Layers with no pending fill keep their
+ current active set (or None on the very first step)."""
+ self.wait_staged()
+ with self._staging_lock:
+ if self._staged_state_next is not None:
+ self._staged_state = self._staged_state_next
+ self._staged_state_next = None
+ self._staged_pending = None
+
+ def _submit_stage_locked(self, unique):
+ try:
+ fut = self._pool.submit(self._build_staged_bundles, unique)
+ except RuntimeError:
+ self._staging_future = None
+ return
+ self._staging_future = fut
+ # If the future is ALREADY done (idle pool + tiny build), the done
+ # callback fires synchronously INSIDE this call — but we hold
+ # _staging_lock here and _on_stage_done needs it => deadlock.
+ # Fix: only attach the callback while still holding the lock if the
+ # future is pending; for an already-done one, detach-then-handle
+ # after release via _staged_rearm (picked up at the next entry).
+ if not fut.done():
+ fut.add_done_callback(self._on_stage_done)
+ return
+ self._staged_rearm = fut
+
+ def _drain_rearm(self):
+ """Handle a synchronously-completed fill outside _staging_lock."""
+ fut = getattr(self, "_staged_rearm", None)
+ if fut is None:
+ return
+ self._staged_rearm = None
+ self._on_stage_done(fut)
+
+ def _on_stage_done(self, fut):
+ try:
+ arrays, slot_of, exp, wall = fut.result()
+ except Exception:
+ with self._staging_lock:
+ self._staging_future = None
+ return
+ with self._staging_lock:
+ self._staged_state_next = (arrays, slot_of, exp)
+ self._staging_future = None
+ with self._lock:
+ self._stats["stage_wall"] += wall
+ if self._staged_pending:
+ p = self._staged_pending
+ self._staged_pending = None
+ self._submit_stage_locked(p)
+
+ def sync_actuals(self):
+ """After a decode step: record this step's actual router indices so
+ they can be staged for the next step (the "current router as the
+ prerouter" substitution). No correctness handling: missing experts are
+ simply dropped by the staged path."""
+ idx = self._pending_indices
+ if idx is None:
+ return
+ try:
+ flat = idx.reshape(-1).tolist()
+ except Exception:
+ return
+ self._pending_indices = None
+ unique = sorted(set(int(v) for v in flat))
+ if not unique:
+ return
+ with self._lock:
+ self.last_used = unique
+ for e in unique:
+ self._hot_counts[e] = (
+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)
+ used_exp = self._staged_at_use
+ if used_exp is not None:
+ dropped = [e for e in unique if e not in used_exp]
+ self._stats["staged_dropped"] += len(dropped)
+ # Promote bundles the router actually used into the shared LRU
+ # so they survive beyond the staging state. Skipped in
+ # write-back mode: staged bundles enter the LRU only on stack
+ # eviction (avoiding staged-set duplication).
+ if self._incr_writeback:
+ return
+ st = self._staged_state
+ if st is not None:
+ sbundles, _, sexp = st
+ smap = {e: sbundles[i] for i, e in enumerate(sexp)}
+ for e in unique:
+ if e in self._pinned:
+ continue
+ b = smap.get(e)
+ if b is not None:
+ self.shared_cache.put((self.layer_idx, e), b)
+
+ def stage_from_prefill(self):
+ """After prefill: stage the last prefill token's actual top-k so the
+ first decode step is already staged (router used as the predictor)."""
+ if not self._staged_mode:
+ return
+ topk = getattr(self, "_last_prefill_topk", None)
+ if not topk:
+ return
+ unique = sorted(set(int(v) for v in topk))
+ if unique:
+ self.last_used = unique
+ self.stage_experts(unique)
+
+ def _refresh_hot_pins(self):
+ if self.hot_per_layer <= 0:
+ return
+ cands = set(self._hot_counts) | set(self.predicted_experts)
+ scores = {e: self._hot_counts.get(e, 0.0) for e in cands}
+ for e in self.predicted_experts:
+ scores[e] = scores.get(e, 0.0) + self._pin_bonus
+ top = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
+ top = top[:self.hot_per_layer]
+ target = {e for e, _ in top}
+ with self._lock:
+ self._pinned = {e: t for e, t in self._pinned.items()
+ if e in target}
+ for e in target:
+ cached = self.shared_cache.pop((self.layer_idx, e))
+ if cached is not None:
+ self._pinned[e] = cached
+ elif e not in self._pinned:
+ self._pinned[e] = self._build(e)
+ self._hot_target = target
+
+ def refresh_hot_pins(self):
+ if self.hot_per_layer <= 0:
+ return
+ try:
+ self._refresh_hot_pins()
+ except Exception: # best-effort
+ pass
+
+ def stats(self):
+ with self._lock:
+ s = dict(self._stats)
+ s["pinned"] = len(self._pinned)
+ s["shared_size"] = self.shared_cache.size()
+ s["calls"] = self._calls
+ return s
+
+ # ---- whole-layer prefill (load full layer once, then drop) ------------
+
+ def load_full_layer(self) -> None:
+ if self._full_weights is not None:
+ return
+ # The checkpoint stores experts ALREADY STACKED per layer
+ # (switch_mlp.gate_proj.weight [256,512,256], one tensor) — so
+ # this is 9 direct whole-tensor loads, NOT 256x9 per-expert
+ # builds. The core.array.view bitcast makes it ~9ms/layer; with
+ # before_layer_cb + async_eval_per_layer, the CPU load hides
+ # under the previous layer's GPU execution.
+ w = {}
+
+ def _read_full(proj, part):
+ name = f"{self._prefix}.{proj}.{part}"
+ raw = self._shard_for(name).raw(name)
+ return raw
+
+ for proj in self._bundle_projs:
+ for part in ("weight", "scales", "biases"):
+ if self._fuse_gu and proj == "gate_up_proj":
+ # Fused rows = gate rows on top of up rows PER EXPERT
+ # (the [E, 2*INTER, in] layout _build produces): the
+ # shard stores gate/up as two separate [E, ...]
+ # tensors, so interleave per-expert slices — a flat
+ # concatenate of both tensors would misalign rows.
+ raw_g = _read_full("gate_proj", part)
+ raw_u = _read_full("up_proj", part)
+ per = raw_g.size // self.num_experts
+ shape = self._bundle_shape[(proj, part)]
+ rows = np.concatenate([
+ np.concatenate([
+ raw_g[e * per:(e + 1) * per],
+ raw_u[e * per:(e + 1) * per]])
+ for e in range(self.num_experts)])
+ if part == "weight":
+ arr = core.array(u32_view(rows, shape))
+ else:
+ arr = core.array(rows.view("<u2")).view(
+ core.bfloat16).reshape(shape)
+ else:
+ raw = _read_full(proj, part)
+ shape = self._bundle_shape[(proj, part)]
+ if part == "weight":
+ arr = core.array(u32_view(raw, shape))
+ else:
+ arr = core.array(raw.view("<u2")).view(
+ core.bfloat16).reshape(shape)
+ w[(proj, part)] = arr
+ self._full_weights = w
+
+ def clear_full_layer(self) -> None:
+ self._full_weights = None
+
+ # ---- hot-layer prefill (LRU-resident partial stack) -------------------
+ # Middle state between on-demand (re-reads the union every request)
+ # and whole-layer E3b (thrashes the page cache): keep the layer's
+ # top-N most frequent experts as a resident stacked tensor; misses go
+ # through the exact per-expert path and are scatter-added.
+
+ def load_hot_layer(self, n_hot: int = 128) -> None:
+ """Load/refresh the layer's hot-expert weights.
+
+ Backing store = one NUMPY array per (proj, part) holding the
+ stacked rows of the top-n_hot experts (page cache, survives across
+ requests, counts ZERO toward MLX active). The mx window
+ (materialize_hot) exposes a few layers at a time so peak MLX
+ memory stays at window_size * n_hot * ~1.2MB instead of
+ 40 * n_hot * 1.2MB.
+
+ Selection = top-n_hot by _hot_counts (decayed frequency). The
+ backing is rebuilt only when membership changes by more than
+ 12.5%; small churn is absorbed by the exact miss path."""
+ counts = self._hot_counts
+ if counts:
+ hot = sorted(counts, key=lambda e: counts[e], reverse=True)
+ hot = hot[:n_hot]
+ else:
+ hot = list(range(n_hot))
+ key = tuple(hot)
+ if self._hot_backing is not None:
+ if self._hot_key == key:
+ return
+ old_set = set(self._hot_key)
+ changed = len([e for e in key if e not in old_set])
+ if changed <= max(1, n_hot // 8):
+ return
+ backing = {}
+ for proj in self._bundle_projs:
+ for part in ("weight", "scales", "biases"):
+ if self._fuse_gu and proj == "gate_up_proj":
+ name_g = f"{self._prefix}.gate_proj.{part}"
+ name_u = f"{self._prefix}.up_proj.{part}"
+ raw_g = self._shard_for(name_g).raw(name_g)
+ raw_u = self._shard_for(name_u).raw(name_u)
+ per = raw_g.size // self.num_experts
+ rows = []
+ for e in hot:
+ rows.append(np.concatenate([
+ raw_g[e * per:(e + 1) * per],
+ raw_u[e * per:(e + 1) * per]]))
+ else:
+ name = f"{self._prefix}.{proj}.{part}"
+ raw = self._shard_for(name).raw(name)
+ per = raw.size // self.num_experts
+ rows = [raw[e * per:(e + 1) * per] for e in hot]
+ # concatenated numpy backing (page cache, not GPU)
+ backing[(proj, part)] = np.concatenate(rows)
+ self._hot_backing = backing
+ self._hot_key = key
+ self._hot_set = set(hot)
+ # invalidate any materialized mx window content
+ self._hot_weights = None
+
+ def materialize_hot(self) -> None:
+ """Materialize the numpy backing into mx arrays (GPU) for this
+ layer. Called by the engine's before_layer_cb under the sliding
+ window; the engine dematerializes layers outside the window."""
+ if self._hot_weights is not None or self._hot_backing is None:
+ return
+ w = {}
+ shape0 = self._bundle_shape
+ n_e = len(self._hot_key)
+ for (proj, part), buf in self._hot_backing.items():
+ shape = shape0[(proj, part)]
+ if part == "weight":
+ # uint8 bytes -> u32 view, then [N, ...] stack
+ arr = core.array(buf.view("<u4")).reshape(
+ (n_e,) + shape[1:])
+ else:
+ # uint8 bytes -> bf16 bitcast view, then [N, ...] stack
+ arr = core.array(buf.view("<u2")).view(
+ core.bfloat16).reshape((n_e,) + shape[1:])
+ w[(proj, part)] = arr
+ self._hot_weights = w
+
+ def dematerialize_hot(self) -> None:
+ """Release the GPU copy; the numpy backing survives."""
+ self._hot_weights = None
+
+ def clear_hot_layer(self) -> None:
+ self._hot_weights = None
+ self._hot_backing = None
+ self._hot_key = None
+ self._hot_set = None
+
+ # ---- forward ---------------------------------------------------------
+
+ def _gather_gate_up(self, x, w, local, sorted_indices):
+ """Up/gate projections over a stacked weight dict, fused or not."""
+ if self._fuse_gu:
+ x_gu = quant.gather_qmm(
+ x, w[("gate_up_proj", "weight")],
+ w[("gate_up_proj", "scales")],
+ w[("gate_up_proj", "biases")], rhs_indices=local,
+ transpose=True, group_size=self.group_size,
+ bits=self.bits, mode=self.mode,
+ sorted_indices=sorted_indices)
+ x_gate, x_up = core.split(x_gu, 2, axis=-1)
+ return x_up, x_gate
+ x_up = quant.gather_qmm(
+ x, w[("up_proj", "weight")], w[("up_proj", "scales")],
+ w[("up_proj", "biases")], rhs_indices=local, transpose=True,
+ group_size=self.group_size, bits=self.bits, mode=self.mode,
+ sorted_indices=sorted_indices)
+ x_gate = quant.gather_qmm(
+ x, w[("gate_proj", "weight")], w[("gate_proj", "scales")],
+ w[("gate_proj", "biases")], rhs_indices=local, transpose=True,
+ group_size=self.group_size, bits=self.bits, mode=self.mode,
+ sorted_indices=sorted_indices)
+ return x_up, x_gate
+
+ def __call__(self, x: core.array, indices: core.array | None = None) -> core.array:
+ self._calls += 1
+ # Record the router indices regardless of which path runs below, so
+ # sync_actuals() can stage the next step's slots even after
+ # fallbacks. In real-prerouter mode there are no router indices (the
+ # staged set is the routing), so sync_actuals records the staged set.
+ self._pending_indices = indices
+ if self._calls % self.hot_update_interval == 0:
+ self._refresh_hot_pins()
+
+ # HOT-layer prefill path (multi-token chunks only): indices remapped
+ # to the resident top-N stack; misses (~3%) contribute zero from the
+ # fast gather (overflow row) and are computed separately below via
+ # the exact per-expert path, then scatter-added. Numerically exact.
+ if (self._hot_weights is not None and indices.size > 8
+ and self._full_weights is None):
+ with self._lock:
+ flat = indices.reshape(-1).tolist()
+ for e in set(int(v) for v in flat):
+ self._hot_counts[e] = (
+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)
+ if self._staged_mode:
+ k = indices.shape[-1]
+ self._last_prefill_topk = flat[-k:]
+ w = self._hot_weights
+ pos = {e: i for i, e in enumerate(self._hot_key)}
+ n_rows = len(self._hot_key)
+ # normalize shapes: x may arrive [1, T, H] (batched) — flatten
+ # to [T, H]; indices likewise [1, T, k] -> [T, k].
+ x_flat = x.reshape(-1, x.shape[-1])
+ ind_flat = indices.reshape(-1, indices.shape[-1])
+ tok_n = ind_flat.shape[0]
+ k_n = ind_flat.shape[-1]
+ flat = ind_flat.reshape(-1).tolist()
+ flat_idx = [int(v) for v in flat]
+ g_list = []
+ miss_rows = [] # flat positions needing exact add
+ for i, e in enumerate(flat_idx):
+ p = pos.get(e)
+ if p is None:
+ g_list.append(n_rows) # overflow zero row
+ miss_rows.append(i)
+ else:
+ g_list.append(p)
+ remapped = core.array(g_list, dtype=core.int32).reshape(
+ ind_flat.shape)
+ x3 = core.expand_dims(x_flat, (-2, -3))
+ xs, local, inv_order = _gather_sort(x3, remapped)
+ x_up, x_gate = self._gather_gate_up(xs, w, local, True)
+ z = quant.gather_qmm(
+ _swiglu(x_up, x_gate),
+ w[("down_proj", "weight")], w[("down_proj", "scales")],
+ w[("down_proj", "biases")], rhs_indices=local, transpose=True,
+ group_size=self.group_size, bits=self.bits, mode=self.mode,
+ sorted_indices=True)
+ out = _scatter_unsort(z, inv_order)
+ # Miss contribution: switch_mlp's caller applies scores after
+ # us, so only RAW per-expert outputs [T, k, H] at miss slots are
+ # needed. Reuse _moe_math (same kernel as the staged path) with
+ # a single-expert "stack" built from the bundle.
+ if miss_rows:
+ miss_pairs = [(i // k_n, i % k_n, flat_idx[i])
+ for i in miss_rows]
+ by_expert = {}
+ for t, kk, e in miss_pairs:
+ by_expert.setdefault(e, []).append((t, kk))
+ for e, tk in by_expert.items():
+ b = self._build(e)
+ rows = core.array([t for t, _ in tk], dtype=core.int32)
+ xe = core.take(x_flat, rows, axis=0) # [m, H]
+ xe2 = core.expand_dims(xe, (-2, -3)) # [m, 1, 1, H]
+ # bundle tensors lack the expert dim -> add it once
+ # (view-like reshape, no copy of the 4bit payload)
+ def _e1(t):
+ return t.reshape((1,) + t.shape)
+ one = core.zeros((1,), dtype=core.int32)
+ if self._fuse_gu:
+ ggu = quant.gather_qmm(
+ xe2, _e1(b[("gate_up_proj", "weight")]),
+ _e1(b[("gate_up_proj", "scales")]),
+ _e1(b[("gate_up_proj", "biases")]),
+ rhs_indices=one, transpose=True,
+ group_size=self.group_size, bits=self.bits,
+ mode=self.mode, sorted_indices=True)
+ gg, gu = core.split(ggu, 2, axis=-1)
+ else:
+ gu = quant.gather_qmm(
+ xe2, _e1(b[("up_proj", "weight")]),
+ _e1(b[("up_proj", "scales")]),
+ _e1(b[("up_proj", "biases")]),
+ rhs_indices=one, transpose=True,
+ group_size=self.group_size, bits=self.bits,
+ mode=self.mode, sorted_indices=True)
+ gg = quant.gather_qmm(
+ xe2, _e1(b[("gate_proj", "weight")]),
+ _e1(b[("gate_proj", "scales")]),
+ _e1(b[("gate_proj", "biases")]),
+ rhs_indices=one, transpose=True,
+ group_size=self.group_size, bits=self.bits,
+ mode=self.mode, sorted_indices=True)
+ ze = quant.gather_qmm(
+ _swiglu(gu, gg),
+ _e1(b[("down_proj", "weight")]),
+ _e1(b[("down_proj", "scales")]),
+ _e1(b[("down_proj", "biases")]),
+ rhs_indices=one, transpose=True,
+ group_size=self.group_size, bits=self.bits,
+ mode=self.mode, sorted_indices=True
+ ) # [m, 1, 1, H]
+ ze = ze.squeeze(-2).squeeze(-2) # [m, H]
+ flat_pos = core.array(
+ [t * k_n + kk for t, kk in tk], dtype=core.int32)
+ out = out.at[flat_pos].add(ze[:, None, :])
+ out = out.reshape(*ind_flat.shape, *out.shape[-2:])
+ out = out.squeeze(-2) # [T, k, H]
+ if x.ndim == 3: # restore batch dim
+ out = out.reshape(x.shape[0], tok_n, k_n, -1)
+ return out
+
+ # Whole-layer prefill path (multi-token chunks only): the full
+ # stacked weights are already loaded (by the engine's
+ # before_layer_cb load_full_layer / clear_full_layer), so the router
+ # indices go straight into the sorted gather — same ops as the exact
+ # path (bit-identical), no tolist / no bundles / no remap.
+ if self._full_weights is not None and indices.size > 8:
+ # Record usage so hot pins survive prefill (the whole-layer path
+ # never touches _get_bundles, so without this _hot_counts stays
+ # empty and PREFILL_PIN has nothing to select from). Also keep
+ # the last token's top-k so the first decode step can be staged.
+ with self._lock:
+ flat = indices.reshape(-1).tolist()
+ unique = set(int(v) for v in flat)
+ for e in unique:
+ self._hot_counts[e] = (
+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)
+ if self._staged_mode:
+ k = indices.shape[-1]
+ self._last_prefill_topk = flat[-k:]
+ w = self._full_weights
+ x = core.expand_dims(x, (-2, -3))
+ x, local, inv_order = _gather_sort(x, indices)
+ x_up, x_gate = self._gather_gate_up(x, w, local, True)
+ z = quant.gather_qmm(
+ _swiglu(x_up, x_gate),
+ w[("down_proj", "weight")], w[("down_proj", "scales")],
+ w[("down_proj", "biases")], rhs_indices=local, transpose=True,
+ group_size=self.group_size, bits=self.bits, mode=self.mode,
+ sorted_indices=True)
+ # Matches the resident SwitchGLU contract: 4D [B, T, k, H] (the
+ # caller's SparseMoeBlock sums over axis -2). NO batch restore
+ # here — _gather_sort/_scatter_unsort already keep [B, T, k, H]
+ # (verified against the deployment's whole-layer path, which
+ # returns the same 4D shape).
+ return _scatter_unsort(z, inv_order, indices.shape).squeeze(-2)
+
+ # Staged decode path (prerouter-style, single-token): consume the
+ # fixed staged slots; router indices never leave the GPU (expert->slot
+ # via core.take). Missing experts map to the overflow zero slot and
+ # their contribution is dropped. Falls back to the exact path only if
+ # the slots are not ready yet.
+
+ def _build_asm(bundles, slot_of_list, exp_key):
+ slot_of = self._slot_table_cache.get(exp_key)
+ if slot_of is None:
+ slot_of = core.array(slot_of_list, dtype=core.int32)
+ core.eval(slot_of)
+ if len(self._slot_table_cache) > 8:
+ self._slot_table_cache.pop(next(
+ iter(self._slot_table_cache)))
+ self._slot_table_cache[exp_key] = slot_of
+ # Stack on the main thread (MLX lazy-op construction must not
+ # race eval). Order matches the math fn's signature directly:
+ # fused (w_gu, s_gu, b_gu, w_d, s_d, b_d); unfused
+ # (w_u, s_u, b_u, w_g, s_g, b_g, w_d, s_d, b_d).
+ wargs = []
+ for proj in self._bundle_projs:
+ for part in ("weight", "scales", "biases"):
+ rows = [b[(proj, part)] for b in bundles]
+ rows.extend([self._zero_slot[(proj, part)]] * (
+ self._staged_n + 1 - len(rows)))
+ wargs.append(core.stack(rows))
+ return slot_of, tuple(wargs)
+
+ # Real-prerouter routing: no router indices, route through the WHOLE
+ # staged set (routing == prediction, no drops).
+ if self._staged_replace and indices is None:
+ self.wait_staged()
+ st = self._staged_state
+ if st is None:
+ with self._lock:
+ self._stats["staged_fallback"] += 1
+ self._staged_at_use = None
+ # First-step guard: zero contribution (nothing staged yet).
+ return core.expand_dims(core.zeros_like(x), -2)
+ bundles, slot_of_list, staged_exp = st
+ self._staged_at_use = staged_exp
+ self.last_used = list(staged_exp)
+ exp_key = tuple(staged_exp)
+ asm = self._staged_asm_cache.get(exp_key)
+ if asm is None:
+ slot_of, wargs = _build_asm(bundles, slot_of_list, exp_key)
+ if len(self._staged_asm_cache) > 1:
+ self._staged_asm_cache.pop(next(
+ iter(self._staged_asm_cache)))
+ self._staged_asm_cache[exp_key] = (slot_of, wargs)
+ else:
+ slot_of, wargs = asm
+ k = len(staged_exp)
+ local2d = core.arange(k, dtype=core.int32)[None, None, :]
+ with self._lock:
+ self._stats["staged_used"] += k
+ return self._moe_math(x, *wargs, local2d)
+
+ if self._staged_mode and indices is not None and indices.size == self._staged_trigger:
+ self.wait_staged()
+ st = self._staged_state
+ if st is None:
+ with self._lock:
+ self._stats["staged_fallback"] += 1
+ self._staged_at_use = None
+ elif self._incr_mode and self._incr_tensors is not None:
+ # Incremental stack: the persistent tensors were updated
+ # in place at stage time (sticky slots); consume them
+ # directly — no core.stack, no asm cache. Order matches the
+ # math fn's signature exactly as _build_asm produces it.
+ _, _, staged_exp = st
+ self._staged_at_use = staged_exp
+ wargs = tuple(
+ self._incr_tensors[(proj, part)]
+ for proj in self._bundle_projs
+ for part in ("weight", "scales", "biases"))
+ local2d = core.take(self._incr_slot_table, indices)
+ with self._lock:
+ self._stats["staged_used"] += indices.size
+ return self._moe_math(x, *wargs, local2d)
+ else:
+ bundles, slot_of_list, staged_exp = st
+ self._staged_at_use = staged_exp
+ exp_key = tuple(staged_exp)
+ asm = self._staged_asm_cache.get(exp_key)
+ if asm is None:
+ slot_of, wargs = _build_asm(
+ bundles, slot_of_list, exp_key)
+ if len(self._staged_asm_cache) > 1:
+ self._staged_asm_cache.pop(next(
+ iter(self._staged_asm_cache)))
+ self._staged_asm_cache[exp_key] = (slot_of, wargs)
+ else:
+ slot_of, wargs = asm
+ local2d = core.take(slot_of, indices)
+ with self._lock:
+ self._stats["staged_used"] += indices.size
+ return self._moe_math(x, *wargs, local2d)
+
+ # Exact decode path: resolve bundles for unique experts.
+ _t0 = time.perf_counter()
+ flat = indices.reshape(-1)
+ unique = sorted(set(int(v) for v in flat.tolist()))
+ with self._lock:
+ self.last_used = unique
+ _t1 = time.perf_counter()
+ bundles = self._get_bundles(unique)
+ _t2 = time.perf_counter()
+ remap = {e: i for i, e in enumerate(unique)}
+ local2d = core.array(
+ [remap[e] for e in flat.tolist()]
+ ).reshape(indices.shape)
+ _t3 = time.perf_counter()
+
+ vals = list(bundles.values())
+ wargs = []
+ for proj in self._bundle_projs:
+ for part in ("weight", "scales", "biases"):
+ wargs.append(core.stack([b[(proj, part)] for b in vals]))
+
+ orig_x = x
+ x = core.expand_dims(x, (-2, -3))
+ do_sort = local2d.size >= 64
+ inv_order = None
+ if do_sort:
+ if self._use_compile:
+ x_s, local, inv_order = _gather_sort(x, local2d)
+ return self._moe_math_sorted(
+ x_s, *wargs, local, inv_order, indices.shape)
+ x, local, inv_order = _gather_sort(x, local2d)
+ elif self._use_compile:
+ return self._moe_math(orig_x, *wargs, local2d)
+ else:
+ local = local2d
+
+ if self._fuse_gu:
+ w_gu, s_gu, b_gu = wargs[0], wargs[1], wargs[2]
+ w_d, s_d, b_d = wargs[3], wargs[4], wargs[5]
+ x_gu = quant.gather_qmm(
+ x, w_gu, s_gu, b_gu, rhs_indices=local,
+ transpose=True, group_size=self.group_size, bits=self.bits,
+ mode=self.mode, sorted_indices=do_sort)
+ x_gate, x_up = core.split(x_gu, 2, axis=-1)
+ else:
+ # wargs order matches the math contract: up first, then gate,
+ # then down (see spec.bundle_projs).
+ w_u, s_u, b_u = wargs[0], wargs[1], wargs[2]
+ w_g, s_g, b_g = wargs[3], wargs[4], wargs[5]
+ w_d, s_d, b_d = wargs[6], wargs[7], wargs[8]
+ x_up = quant.gather_qmm(
+ x, w_u, s_u, b_u, rhs_indices=local,
+ transpose=True, group_size=self.group_size, bits=self.bits,
+ mode=self.mode, sorted_indices=do_sort)
+ x_gate = quant.gather_qmm(
+ x, w_g, s_g, b_g, rhs_indices=local,
+ transpose=True, group_size=self.group_size, bits=self.bits,
+ mode=self.mode, sorted_indices=do_sort)
+ z = quant.gather_qmm(
+ _swiglu(x_up, x_gate), w_d, s_d, b_d, rhs_indices=local,
+ transpose=True, group_size=self.group_size, bits=self.bits,
+ mode=self.mode, sorted_indices=do_sort)
+ if do_sort:
+ z = _scatter_unsort(z, inv_order, indices.shape)
+ return z.squeeze(-2)
+
+ # ---- lifecycle --------------------------------------------------------
+
+ def reset(self):
+ """Per-request reset: drop staged/prefill working state so the next
+ request starts clean (the shared LRU, hot counts, and pools survive
+ across requests — they are the cross-request memory)."""
+ self._pending_indices = None
+ self._staged_at_use = None
+ self._last_prefill_topk = None
+ self._staged_state = None
+ self._staged_state_next = None
+ with self._staging_lock:
+ self._staged_pending = None
+ self.last_used = []
+ self._full_weights = None
+
+ def close(self):
+ """Release thread pools (idempotent)."""
+ for pool in (self._pool, self._prefetch_pool):
+ try:
+ pool.shutdown(wait=False, cancel_futures=True)
+ except Exception:
+ pass
diff --git a/src/edge0/streaming/mmap.py b/src/edge0/streaming/mmap.py
new file mode 100644
index 0000000..3ea9441
--- /dev/null
+++ b/src/edge0/streaming/mmap.py
@@ -0,0 +1,63 @@
+"""Zero-copy byte-range access into safetensors shards.
+
+A single expert is a contiguous byte slice of one stacked
+[num_experts, ...] tensor; the streaming layer reads those slices straight
+out of an mmap with no full-tensor materialization.
+"""
+
+from __future__ import annotations
+
+import json
+import mmap
+import struct
+
+import numpy as np
+
+
+class SafetensorsMmap:
+ """One safetensors shard, opened as a byte-range mmap.
+
+ ``entries[name]`` holds ``{"offset", "size", "dtype", "shape"}``;
+ ``raw(name)`` returns a zero-copy uint8 view of the tensor's bytes.
+ """
+
+ def __init__(self, path: str):
+ self.path = path
+ self._file = open(path, "rb")
+ self._mm = mmap.mmap(self._file.fileno(), 0, access=mmap.ACCESS_READ)
+ header_len = struct.unpack("<Q", self._mm[:8])[0]
+ header = json.loads(self._mm[8:8 + header_len])
+ self.payload_base = 8 + header_len
+ self.entries = {}
+ for name, meta in header.items():
+ if name == "__metadata__":
+ continue
+ begin, end = meta["data_offsets"]
+ self.entries[name] = {
+ "offset": self.payload_base + begin,
+ "size": end - begin,
+ "dtype": meta["dtype"],
+ "shape": tuple(meta["shape"]),
+ }
+
+ def raw(self, name: str) -> np.ndarray:
+ e = self.entries[name]
+ return np.frombuffer(
+ self._mm, dtype=np.uint8, count=e["size"], offset=e["offset"]
+ )
+
+ def close(self):
+ self._mm.close()
+ self._file.close()
+
+
+def bf16_bits_to_f32(data: np.ndarray) -> np.ndarray:
+ """Raw BF16 bytes -> float32 array (bit pattern shifted into fp32)."""
+ u16 = data.view("<u2")
+ return (u16.astype(np.uint32) << 16).view(np.float32)
+
+
+def u32_view(data: np.ndarray, shape) -> np.ndarray:
+ """uint8 bytes -> uint32 view reshaped to ``shape`` (quantized payloads
+ are stored as packed uint32 words)."""
+ return data.view("<u4").reshape(shape)
diff --git a/src/edge0/streaming/options.py b/src/edge0/streaming/options.py
new file mode 100644
index 0000000..115130e
--- /dev/null
+++ b/src/edge0/streaming/options.py
@@ -0,0 +1,134 @@
+"""Streaming layer options.
+
+One ``LayerOptions`` instance describes the streaming behavior of one MoE
+layer (all layers usually share one instance). This is the typed
+replacement for the deployment's env-var knobs; profiles (see
+``edge0.models``) pick the option sets for a given tier.
+
+Defaults reproduce the deployment's production behavior for the edge0-35b
+K=4 tier (staged decode on, sync fills, hot pins on prefill).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class LayerOptions:
+ """Options for one streaming MoE layer.
+
+ Attributes:
+ staged: enable fixed-slot staged decode (routing through staged
+ slots with zero per-layer host syncs).
+ staged_replace: staged set REPLACES the router (routing set ==
+ staged set; used with a prerouter head — no drops).
+ staged_n: number of staged slots.
+ staged_trigger: router top-k that activates the staged path
+ (usually the routed top_k).
+ staged_sync: fill staged slots synchronously at the step boundary
+ (as opposed to purely async double-buffered fills).
+ asm_cache: cache assembled (slot table, stacked-wargs) per staged
+ expert set so repeated sets don't rebuild graph nodes.
+ incr_stack: incremental sticky-slot stack (in-place row writes
+ instead of re-stacking; sync staged mode only).
+ incr_writeback: write evicted bundles back into the shared LRU
+ (dedups the staged set from the LRU while staged).
+ history_prefetch: prefetch the union of recent actual expert sets.
+ hot_per_layer: resident hot-expert pins per layer (0 disables).
+ hot_update_interval: refresh hot pins every N forward calls.
+ hot_decay: decay factor for hot-expert usage counts.
+ pin_bonus: extra score for prerouter-predicted experts when
+ selecting hot pins.
+ cache_slots: shared LRU capacity across all layers.
+ prefetch_cap: prefetch buffer capacity.
+ load_threads: threads for on-demand expert builds.
+ prefetch_threads: threads for eager prefetch builds.
+ use_compile: wrap the MoE math in ``mx.compile``.
+ top_k: override the resident block's routed top-k (both prefill
+ and decode); None leaves the model config value.
+ full_layer_prefill: whole-layer load-drop prefill (E3b).
+ prefill_full_layers: number of LEADING layers that get whole-layer
+ loads (N*~310MB of page cache for qwen); 0 = every layer.
+ prefill_hot: resident hot-stack size used during prefill
+ (0 disables the hot-stack path).
+ """
+
+ staged: bool = False
+ staged_replace: bool = False
+ staged_n: int = 8
+ staged_trigger: int = 8
+ staged_sync: bool = True
+ asm_cache: bool = True
+ incr_stack: bool = False
+ incr_writeback: bool = False
+ history_prefetch: bool = True
+ hot_per_layer: int = 0
+ hot_update_interval: int = 4
+ hot_decay: float = 0.75
+ pin_bonus: float = 2.0
+ cache_slots: int = 64
+ prefetch_cap: int = 48
+ load_threads: int = 8
+ prefetch_threads: int = 4
+ use_compile: bool = True
+ top_k: int | None = None
+ full_layer_prefill: bool = False
+ prefill_full_layers: int = 0
+ prefill_hot: int = 0
+
+ # ---- presets ----------------------------------------------------------
+
+ @classmethod
+ def staged_k4(cls, **overrides) -> "LayerOptions":
+ """edge0-35b K=4 tier: staged decode + hot pins + whole-layer
+ prefill (production profile).
+
+ ``staged_replace`` stays False: the trained prerouter head supplies
+ the routing (the MoE block routes via the prerouter logits), so the
+ staged set == the routing set exactly — the slot table maps without
+ drops."""
+ return cls(
+ staged=True, staged_replace=False, staged_n=4,
+ staged_trigger=4, staged_sync=True, history_prefetch=True,
+ hot_per_layer=32, hot_update_interval=4, hot_decay=0.75,
+ cache_slots=64, prefetch_cap=48, load_threads=8,
+ prefetch_threads=4, use_compile=True, top_k=4,
+ full_layer_prefill=True, prefill_full_layers=12,
+ prefill_hot=32, **overrides)
+
+ @classmethod
+ def staged_k8(cls, **overrides) -> "LayerOptions":
+ """edge0-10b tier: staged decode with K=8 (native routing width),
+ no hot pins, E3b whole-layer prefill."""
+ return cls(
+ staged=True, staged_replace=False, staged_n=8,
+ staged_trigger=8, staged_sync=True, history_prefetch=True,
+ hot_per_layer=0, cache_slots=64, prefetch_cap=48,
+ load_threads=8, prefetch_threads=4, use_compile=True, top_k=8,
+ full_layer_prefill=True, prefill_hot=0, **overrides)
+
+ @classmethod
+ def prod_k8(cls, **overrides) -> "LayerOptions":
+ """edge0-10b tier: deployment production profile.
+
+ Mirrors start_server.sh on the ling v7 deploy: STAGED_DECODE=0
+ (staged decode is OFF — the deployment verified staged decode on
+ ling degrades output, see the comment in start_server.sh),
+ STAGED_SYNC=0, STAGED_SLOTS=8, EXPERT_CACHE_SLOTS=64,
+ MLX_COMPILE_MOE=1, PREFILL_FULL_LAYER=1. The hybrid prerouter
+ still stages each next token's expert set into the shared LRU
+ (engine.py: stage_all at the step boundary AND right after
+ prefill), and the vendored MoE blocks re-select from the cached
+ prerouter logits."""
+ return cls(
+ staged=False, staged_replace=False, staged_n=8,
+ staged_trigger=8, staged_sync=False, history_prefetch=False,
+ hot_per_layer=0, cache_slots=64, prefetch_cap=48,
+ load_threads=8, prefetch_threads=4, use_compile=True, top_k=8,
+ full_layer_prefill=True, prefill_hot=0, **overrides)
+
+ def for_layer(self, layer_idx: int) -> "LayerOptions":
+ """Per-layer copy (staged slots are per-layer state, but options
+ are shared; this hook exists for future per-layer overrides)."""
+ return self
diff --git a/tests/test_moe_spec.py b/tests/test_moe_spec.py
new file mode 100644
index 0000000..b3dd18f
--- /dev/null
+++ b/tests/test_moe_spec.py
@@ -0,0 +1,98 @@
+"""MoESpec path resolution and key templating tests."""
+
+from __future__ import annotations
+
+import pytest
+
+from edge0.moe.spec import (MoESpec, QuantSpec, RouterKind, WeightLayout)
+
+
+class _FakeLayer:
+ def __init__(self, mlp=None):
+ self.mlp = mlp
+
+
+class _FakeSwitchMLP:
+ def __init__(self):
+ self.name = "switch"
+
+
+class _FakeBlock:
+ def __init__(self):
+ self.switch_mlp = _FakeSwitchMLP()
+
+
+class _FakeModel:
+ def __init__(self, n=4):
+ self.language_model = type("LM", (), {})()
+ self.language_model.model = type("M", (), {})()
+ self.language_model.model.layers = [
+ _FakeLayer(_FakeBlock()) for _ in range(n)
+ ]
+ # ling-style plain module list (no attribute access)
+ self.layers = [_FakeLayer(_FakeBlock()) for _ in range(n)]
+
+
+def _spec(**kw):
+ base = dict(
+ num_experts=8, top_k=2, intermediate_size=64,
+ key_template="language_model.model.layers.{layer}.mlp.switch_mlp",
+ block_path="language_model.model.layers.{layer}.mlp.switch_mlp",
+ )
+ base.update(kw)
+ return MoESpec(**base)
+
+
+def test_keys_templating():
+ s = _spec()
+ assert s.keys(3, "gate_proj", "weight") == (
+ "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight")
+ assert s.keys(12, "down_proj", "biases") == (
+ "language_model.model.layers.12.mlp.switch_mlp.down_proj.biases")
+
+
+def test_block_of_digit_segments():
+ s = _spec()
+ m = _FakeModel()
+ block = s.block_of(m, 2)
+ assert block.name == "switch"
+ assert s.block_of(m, 0) is m.language_model.model.layers[0].mlp.switch_mlp
+
+
+def test_layer_of_defaults_from_block_path():
+ s = _spec()
+ m = _FakeModel()
+ layer = s.layer_of(m, 1)
+ assert layer is m.language_model.model.layers[1]
+
+
+def test_layer_of_plain_list_path():
+ s = _spec(block_path="layers.{layer}.mlp",
+ layer_path="layers.{layer}",
+ key_template="model.layers.{layer}.mlp.experts")
+ m = _FakeModel()
+ assert s.block_of(m, 3) is m.layers[3].mlp
+ assert s.layer_of(m, 2) is m.layers[2]
+
+
+def test_bundle_projs_layouts():
+ # Stack order matches the math signature directly: up, gate, down
+ # (_swiglu(up, gate)) — same convention as the original
+ # streaming_experts_qwen35.py ("up_proj", "gate_proj", "down_proj").
+ sep = _spec()
+ assert sep.bundle_projs == ("up_proj", "gate_proj", "down_proj")
+ assert sep.fuse_gu is False
+ fused = _spec(layout=WeightLayout.FUSED_GATE_UP)
+ assert fused.bundle_projs == ("gate_up_proj", "down_proj")
+ assert fused.fuse_gu is True
+
+
+def test_quant_defaults():
+ assert QuantSpec().bits == 4
+ assert QuantSpec().group_size == 64
+ assert QuantSpec().mode == "affine"
+
+
+def test_router_kinds():
+ assert RouterKind.SOFTMAX_TOPK.value == "softmax_topk"
+ assert RouterKind.SIGMOID_GROUP.value == "sigmoid_group"
diff --git a/tests/test_registry.py b/tests/test_registry.py
new file mode 100644
index 0000000..adf9998
--- /dev/null
+++ b/tests/test_registry.py
@@ -0,0 +1,97 @@
+"""Registry / AutoConfig / AutoModel resolution tests (no GPU needed)."""
+
+from __future__ import annotations
+
+import pytest
+
+import edge0.models # noqa: F401 (populates MODEL_REGISTRY on import)
+from edge0 import AutoConfig
+from edge0.registry import MODEL_REGISTRY, TYPE_ALIASES
+
+
+def test_both_tiers_registered():
+ assert set(MODEL_REGISTRY) == {"edge0-35b", "edge0-10b"}
+
+
+def test_type_aliases_resolve():
+ assert TYPE_ALIASES["qwen3_5_moe"] == "edge0-35b"
+ assert TYPE_ALIASES["qwen3_5_moe_text"] == "edge0-35b"
+ assert TYPE_ALIASES["bailing_hybrid"] == "edge0-10b"
+ assert TYPE_ALIASES["bailing_moe_linear"] == "edge0-10b"
+
+
[email protected]("name", ["edge0-35b", "edge0-10b"])
+def test_auto_config_defaults(name):
+ cfg = AutoConfig.from_pretrained(name=name)
+ assert cfg.name == name
+ assert cfg.moe_spec.num_experts > 0
+ assert cfg.moe_spec.top_k in (4, 8)
+ # qwen tier defaults to staged decode; the ling tier mirrors the
+ # deployment production profile (STAGED_DECODE=0 — staged decode on
+ # ling degrades output, see start_server.sh) with the prerouter
+ # still staging each next token's expert set.
+ assert cfg.options.staged is (name == "edge0-35b")
+ assert cfg.prerouter is not None
+ assert cfg.prerouter.weights_file.endswith(".safetensors")
+ assert cfg.prerouter_top_k == cfg.moe_spec.top_k
+
+
+def test_qwen35_profile():
+ cfg = AutoConfig.from_pretrained(name="edge0-35b")
+ assert cfg.moe_spec.num_experts == 256
+ assert cfg.moe_spec.top_k == 4
+ assert cfg.moe_spec.intermediate_size == 512
+ assert cfg.moe_spec.norm_topk_prob is True
+ assert cfg.moe_spec.shared_experts == 1
+ assert cfg.moe_spec.quant.bits == 4
+ assert cfg.moe_spec.quant.group_size == 64
+ assert cfg.moe_spec.layout.value == "separate"
+ assert "language_model.model.layers" in cfg.moe_spec.key_template
+ assert cfg.options.staged_n == 4
+ assert cfg.options.prefill_full_layers == 12
+ assert cfg.options.hot_per_layer == 32
+ assert cfg.prerouter.start_layer == 7
+ assert cfg.prerouter.hidden == 512
+ assert cfg.prerouter.dtype == "fp16"
+ assert cfg.prerouter.feature_topk == "executed"
+ assert cfg.gen.temperature == 0.6
+ assert 248046 in cfg.gen.eos_ids
+ assert cfg.port == 8085
+
+
+def test_ling10b_profile():
+ cfg = AutoConfig.from_pretrained(name="edge0-10b")
+ assert cfg.moe_spec.num_experts == 128
+ assert cfg.moe_spec.top_k == 8
+ assert cfg.moe_spec.router.value == "sigmoid_group"
+ assert cfg.moe_spec.routed_scaling == 2.5
+ assert cfg.moe_spec.n_group == 8
+ assert cfg.moe_spec.topk_group == 4
+ assert cfg.options.staged_n == 8
+ assert cfg.prerouter.start_layer == 1
+ assert cfg.prerouter.owners == tuple(range(1, 23))
+ assert cfg.prerouter.patch_call is False
+ assert 156895 in cfg.gen.eos_ids
+ assert cfg.port == 8083
+
+
+def test_override_and_reject():
+ cfg = AutoConfig.from_pretrained(name="edge0-10b", prerouter=None,
+ lora="", prerouter_top_k=0)
+ assert cfg.prerouter is None
+ assert cfg.lora == ""
+ assert cfg.prerouter_top_k == 0
+ with pytest.raises(TypeError, match="unknown"):
+ AutoConfig.from_pretrained(name="edge0-10b", bogus_field=1)
+
+
+def test_unknown_name_lists_registry():
+ with pytest.raises(KeyError, match="edge0-35b"):
+ AutoConfig.from_pretrained(name="edge0-99b")
+
+
+def test_adapter_module_exposes_config():
+ for name, mod in MODEL_REGISTRY.items():
+ assert hasattr(mod, "Config")
+ assert hasattr(mod, "build_model")
+ assert hasattr(mod, "build_engine")
diff --git a/tests/test_sampling.py b/tests/test_sampling.py
new file mode 100644
index 0000000..965c01c
--- /dev/null
+++ b/tests/test_sampling.py
@@ -0,0 +1,64 @@
+"""Sampler unit tests: shape-independent draws and truncation behavior."""
+
+from __future__ import annotations
+
+import mlx.core as mx
+import pytest
+
+from edge0.sampling import _mask_logits, sample
+
+
+def test_mask_identity_for_sane_kwargs():
+ logits = mx.arange(10, dtype=mx.float32)
+ out = _mask_logits(logits, temperature=1.0, top_k=None, top_p=None)
+ assert mx.allclose(out, logits).item()
+
+
+def test_mask_topk_keeps_only_top_k():
+ logits = mx.arange(10, dtype=mx.float32) # 0..9
+ out = _mask_logits(logits, temperature=1.0, top_k=3, top_p=None)
+ vals = mx.sort(out, axis=-1)[-3:].tolist()
+ assert vals == [7.0, 8.0, 9.0]
+ assert int(mx.sum(mx.isfinite(out)).item()) == 3
+
+
+def test_sample_returns_int_and_in_range():
+ logits = mx.ones((5,), dtype=mx.float32)
+ for _ in range(5):
+ t = sample(logits, temperature=1.0)
+ assert isinstance(t, int)
+ assert 0 <= t < 5
+
+
+def test_sample_seed_deterministic():
+ logits = mx.arange(8, dtype=mx.float32) * 2.0
+ a = sample(logits, temperature=0.5, seed=42)
+ b = sample(logits, temperature=0.5, seed=42)
+ assert a == b
+
+
+def test_repetition_penalty_changes_distribution():
+ logits = mx.array([10.0, 9.0, 0.0], dtype=mx.float32)
+ # history token 0 is penalized -> token 1 becomes likelier
+ t = sample(logits, temperature=1.0, repetition_penalty=2.0,
+ history=(0,))
+ assert isinstance(t, int)
+ # with penalty 2x on a +10 logit: 5 vs 9 -> argmax flips to 1
+ out = _mask_logits(logits, 1.0, None, None)
+ assert out.dtype == mx.float32
+
+
+def test_sample_temperature_zero_is_argmax():
+ logits = mx.array([1.0, 5.0, 3.0], dtype=mx.float32)
+ # temperature <= 0 is deterministic decode (HF / llama.cpp semantics):
+ # greedy — the draw must ALWAYS be the argmax, whatever the seed.
+ for seed in range(20):
+ assert sample(logits, temperature=0.0, seed=seed) == 1
+
+
+def test_sample_temperature_zero_greedy_with_topk():
+ # top-k truncation still applies before the greedy argmax
+ logits = mx.array([1.0, 5.0, 3.0, 4.0, 2.0], dtype=mx.float32)
+ # top-2 = {5.0 (idx 1), 4.0 (idx 3)} -> greedy picks idx 1
+ for seed in range(5):
+ assert sample(logits, temperature=0.0, top_k=2, seed=seed) == 1
diff --git a/tests/test_server.py b/tests/test_server.py
new file mode 100644
index 0000000..7c7a373
--- /dev/null
+++ b/tests/test_server.py
@@ -0,0 +1,317 @@
+"""Server-layer tests: request parsing, chat session, SSE framing and the
+stdlib HTTP transport, all against a fake engine (no real checkpoints).
+
+Covers the OpenAI-compatible surface the deployment servers expose:
+/healthz, /v1/models, POST /v1/chat/completions (non-streaming and SSE
+streaming), and the /v1/completions rejection path.
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+import time
+from http.server import ThreadingHTTPServer
+from types import SimpleNamespace
+from urllib import request as urlrequest
+
+import pytest
+
+from edge0.config import GenerationConfig
+from edge0.server.app import (
+ _chat_once,
+ _chat_stream,
+ _StdlibHandler,
+ build_app_handlers,
+ run_stdlib,
+)
+from edge0.server.chat import (
+ ChatMessage,
+ ChatRequest,
+ ChatSession,
+ QueueServer,
+ decode_tokens,
+ parse_chat_request,
+ sse_format,
+)
+
+
+# ---- fake engine ----------------------------------------------------------
+
+
+class FakeTok:
+ """Minimal tokenizer stub (transformers-like surface)."""
+
+ def __init__(self, ids=(7, 8, 9)):
+ self.ids = list(ids)
+ self.bos_token_id = 0
+
+ def apply_chat_template(self, messages, tokenize=False,
+ add_generation_prompt=True,
+ enable_thinking=False):
+ text = "<tpl>" + json.dumps(messages)
+ if add_generation_prompt:
+ text += "<|im_start|>assistant\n"
+ return text
+
+ def encode(self, text):
+ return self.ids
+
+ def decode(self, tokens):
+ return "".join(f"T{t}" for t in tokens)
+
+
+class PlainTok(FakeTok):
+ """Tokenizer without apply_chat_template (stdlib fallback path)."""
+
+ def apply_chat_template(self, *a, **k): # pragma: no cover - unused
+ raise AttributeError
+
+
+class FakeEngine:
+ """Duck-typed engine: enough of Edge0Engine for the server layer."""
+
+ name = "fake"
+
+ def __init__(self, tok=None):
+ self._tok = tok or FakeTok()
+ self.pos = 3
+ self.cfg = SimpleNamespace(
+ gen=GenerationConfig(temperature=0.7, top_p=0.95))
+ self.generated = []
+
+ def generate(self, ids, gen_config=None, on_token=None, **kw):
+ toks = [11, 12, 13]
+ self.generated.append((list(ids), gen_config))
+ if on_token is not None:
+ for t in toks:
+ on_token(t)
+ return toks
+
+ def stats(self):
+ return {"ok": 1}
+
+
+def _req(**kw) -> ChatRequest:
+ payload = {
+ "model": "fake",
+ "messages": [{"role": "user", "content": "hi"}],
+ **kw,
+ }
+ return parse_chat_request(payload)
+
+
+# ---- parse_chat_request ---------------------------------------------------
+
+
+def test_parse_basic_defaults():
+ req = _req()
+ assert req.model == "fake"
+ assert len(req.messages) == 1
+ assert req.messages[0].role == "user"
+ assert req.messages[0].content == "hi"
+ assert req.temperature is None
+ assert req.stream is False
+
+
+def test_parse_multipart_content_joins_text():
+ req = _req(messages=[{
+ "role": "user",
+ "content": [{"type": "text", "text": "a"},
+ {"type": "image_url", "image_url": {"url": "x"}},
+ {"type": "text", "text": "b"}],
+ }])
+ assert req.messages[0].content == "ab"
+
+
+def test_parse_stream_flag_and_sampling():
+ req = _req(temperature=0.3, top_p=0.9, top_k=32, max_tokens=16,
+ stream=True)
+ assert req.temperature == 0.3
+ assert req.top_p == 0.9
+ assert req.top_k == 32
+ assert req.max_tokens == 16
+ assert req.stream is True
+
+
+# ---- sse / decode ---------------------------------------------------------
+
+
+def test_sse_format():
+ line = sse_format({"id": "1", "x": "中文"})
+ assert line.startswith("data: ")
+ assert line.endswith("\n\n")
+ assert json.loads(line[6:-2]) == {"id": "1", "x": "中文"}
+
+
+def test_decode_tokens():
+ eng = FakeEngine()
+ assert decode_tokens(eng, [1, 2]) == "T1T2"
+ eng._tok = None
+ assert decode_tokens(eng, [1]) == ""
+
+
+# ---- ChatSession ----------------------------------------------------------
+
+
+def test_chat_session_prompt_ids_uses_template():
+ eng = FakeEngine()
+ sess = ChatSession(eng, _req())
+ ids = sess.prompt_ids()
+ assert ids == [7, 8, 9]
+ # The template was called with enable_thinking=False (serve.py parity).
+ text = eng._tok.apply_chat_template([{"role": "user", "content": "hi"}])
+ assert "<|im_start|>assistant" in text
+
+
+def test_chat_session_prompt_ids_fallback_text():
+ eng = FakeEngine(tok=PlainTok())
+ sess = ChatSession(eng, _req())
+ ids = sess.prompt_ids()
+ assert ids == [7, 8, 9]
+
+
+def test_chat_session_fallback_text_format():
+ eng = FakeEngine(tok=PlainTok())
+ req = parse_chat_request({"messages": [
+ {"role": "system", "content": "sys"},
+ {"role": "user", "content": "q"},
+ {"role": "assistant", "content": "a"},
+ ]})
+ sess = ChatSession(eng, req)
+ text = sess._chat_text()
+ assert "<|im_start|>system\nsys<|im_end|>\n" in text
+ assert "<|im_start|>user\nq<|im_end|>\n" in text
+ assert "<|im_start|>assistant\na<|im_end|>\n" in text
+ assert text.endswith("<|im_start|>assistant\n")
+
+
+def test_gen_config_overrides_request_fields():
+ eng = FakeEngine()
+ sess = ChatSession(eng, _req(temperature=0.1, max_tokens=5))
+ gen = sess.gen_config()
+ assert gen.temperature == 0.1
+ assert gen.max_new_tokens == 5
+ assert gen.top_p == 0.95 # engine default preserved
+
+
+def test_chat_session_run_usage_and_callback():
+ eng = FakeEngine()
+ seen = []
+ sess = ChatSession(eng, _req())
+ tokens, meta = sess.run(on_token=seen.append)
+ assert tokens == [11, 12, 13]
+ assert seen == [11, 12, 13]
+ assert meta["usage"] == {
+ "prompt_tokens": 3,
+ "completion_tokens": 3,
+ "total_tokens": 6,
+ }
+ assert "wall_s" in meta
+
+
+# ---- QueueServer ----------------------------------------------------------
+
+
+def test_queue_server_health():
+ eng = FakeEngine()
+ srv = QueueServer(eng)
+ h = srv.health()
+ assert h["status"] == "ok"
+ assert h["model"] == "fake"
+ assert h["pos"] == 3
+
+
+def test_queue_server_chat_serializes():
+ eng = FakeEngine()
+ srv = QueueServer(eng)
+ out = srv.chat(_req())
+ assert out["usage"]["completion_tokens"] == 3
+ assert len(eng.generated) == 1
+ gen = eng.generated[0][1]
+ assert gen.temperature == 0.7
+
+
+# ---- handlers -------------------------------------------------------------
+
+
+def test_handlers_healthz_and_models():
+ srv = QueueServer(FakeEngine())
+ handlers = build_app_handlers(srv)
+ assert handlers["GET /healthz"]()["status"] == "ok"
+ models = handlers["GET /v1/models"]()
+ assert models["object"] == "list"
+ assert models["data"][0]["id"] == "fake"
+
+
+def test_handlers_completions_rejected():
+ srv = QueueServer(FakeEngine())
+ handlers = build_app_handlers(srv)
+ body, status = handlers["POST /v1/completions"]({})
+ assert status == 400
+
+
+def test_chat_once_response_shape():
+ srv = QueueServer(FakeEngine())
+ out = _chat_once(srv, {"messages": [{"role": "user", "content": "hi"}]})
+ assert out["object"] == "chat.completion"
+ assert out["model"] == "fake"
+ ch = out["choices"][0]
+ assert ch["message"]["role"] == "assistant"
+ assert ch["message"]["content"] == "T11T12T13"
+ assert ch["finish_reason"] == "stop"
+ assert out["usage"]["completion_tokens"] == 3
+
+
+def test_chat_stream_events_sequence():
+ srv = QueueServer(FakeEngine())
+ raw = _chat_stream(srv, {
+ "messages": [{"role": "user", "content": "hi"}], "stream": True,
+ })
+ body = raw if isinstance(raw, bytes) else raw.get_data()
+ events = [e for e in body.decode("utf-8").split("\n\n") if e]
+ chunks = [json.loads(e[6:]) for e in events[:-1]] # drop [DONE]
+ deltas = [c["choices"][0]["delta"]["content"] for c in chunks]
+ assert deltas == ["T11", "T12", "T13"]
+ assert chunks[-1]["choices"][0]["finish_reason"] == "stop"
+ assert events[-1] == "data: [DONE]"
+
+
+# ---- stdlib HTTP transport ------------------------------------------------
+
+
+def test_stdlib_http_end_to_end():
+ eng = FakeEngine()
+ srv = QueueServer(eng, model_name="fake")
+ handler = type("Edge0Handler", (_StdlibHandler,), {"server_q": srv})
+ httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
+ port = httpd.server_address[1]
+ t = threading.Thread(target=httpd.serve_forever, daemon=True)
+ t.start()
+ base = f"http://127.0.0.1:{port}"
+ try:
+ with urlrequest.urlopen(f"{base}/healthz", timeout=10) as r:
+ assert r.status == 200
+ assert json.loads(r.read())["status"] == "ok"
+ with urlrequest.urlopen(f"{base}/v1/models", timeout=10) as r:
+ assert json.loads(r.read())["data"][0]["id"] == "fake"
+ payload = json.dumps({
+ "model": "fake",
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_tokens": 8,
+ }).encode("utf-8")
+ req = urlrequest.Request(f"{base}/v1/chat/completions",
+ data=payload,
+ headers={"Content-Type": "application/json"})
+ with urlrequest.urlopen(req, timeout=10) as r:
+ assert r.status == 200
+ out = json.loads(r.read())
+ assert out["choices"][0]["message"]["content"] == "T11T12T13"
+ # unknown route -> 404
+ with pytest.raises(Exception) as exc:
+ urlrequest.urlopen(f"{base}/nope", timeout=10)
+ assert getattr(exc.value, "code", None) == 404
+ finally:
+ httpd.shutdown()
+ httpd.server_close()
+ t.join(timeout=5)
diff --git a/tests/test_streaming_math.py b/tests/test_streaming_math.py
new file mode 100644
index 0000000..e77154e
--- /dev/null
+++ b/tests/test_streaming_math.py
@@ -0,0 +1,301 @@
+"""StreamingSwitchGLU path math tests: every streaming path must produce
+outputs identical to the exact per-expert path (same gather_qmm kernel),
+and the exact path must match a plain fp16 reference implementation.
+
+No real checkpoints needed: small random 4-bit weights are written to a
+temp safetensors shard with the exact layout the checkpoint uses
+(gate_proj / up_proj / down_proj stacked along the expert axis).
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+import mlx.core as mx
+import mlx.nn as nn
+from safetensors.numpy import save_file
+
+from edge0.moe.spec import MoESpec, QuantSpec, WeightLayout
+from edge0.streaming.cache import SharedExpertCache
+from edge0.streaming.layer import StreamingSwitchGLU
+from edge0.streaming.mmap import SafetensorsMmap
+from edge0.streaming.options import LayerOptions
+
+N_EXPERTS = 8
+DIM = 64
+INTER = 128
+
+
+def _write_shard(path, fuse_gu: bool, seed: int = 0):
+ """Random fp16 weights -> 4-bit shard; returns the fp16 originals.
+
+ Layout matches the checkpoint convention [out, in/8] packed u32:
+ gate/up project hidden (DIM) to intermediate (INTER); down projects
+ INTER back to DIM — so down rows run along DIM with INTER as the
+ packed axis.
+ """
+ mx.random.seed(seed)
+ w = {}
+ w["gate_proj"] = mx.random.normal((N_EXPERTS, INTER, DIM)).astype(mx.float16)
+ w["up_proj"] = mx.random.normal((N_EXPERTS, INTER, DIM)).astype(mx.float16)
+ w["down_proj"] = mx.random.normal((N_EXPERTS, DIM, INTER)).astype(mx.float16)
+
+ def _parts(proj):
+ w16 = w[proj]
+ # transpose to [E, out, in] -> quantize per-expert rows along last
+ # axis; scales are stored as bf16 bit patterns (u16), weights as
+ # packed u32.
+ q, scales, biases = mx.quantize(w16, group_size=64, bits=4)
+ # mx.quantize returns fp16 scales; the checkpoint stores bf16 bit
+ # patterns — convert numerically, then bitcast to u16.
+ return {
+ "weight": np.asarray(q), # uint32
+ "scales": np.asarray(scales.astype(mx.bfloat16).view(mx.uint16)),
+ "biases": np.asarray(biases.astype(mx.bfloat16).view(mx.uint16)),
+ }
+
+ tensors = {}
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ for part, arr in _parts(proj).items():
+ tensors[f"layers.0.mlp.switch_mlp.{proj}.{part}"] = arr
+ save_file(tensors, str(path))
+ return {k: np.asarray(v) for k, v in w.items()}
+
+
+def _spec(fuse_gu: bool) -> MoESpec:
+ return MoESpec(
+ num_experts=N_EXPERTS, top_k=4, intermediate_size=INTER,
+ quant=QuantSpec(bits=4, group_size=64),
+ layout=(WeightLayout.FUSED_GATE_UP if fuse_gu
+ else WeightLayout.SEPARATE),
+ key_template="layers.{layer}.mlp.switch_mlp",
+ block_path="layers.{layer}.mlp.switch_mlp",
+ )
+
+
+def _options(**kw) -> LayerOptions:
+ base = dict(staged=False, staged_n=4, staged_trigger=4,
+ staged_sync=True, load_threads=2, prefetch_threads=1)
+ base.update(kw)
+ return LayerOptions(**base)
+
+
[email protected](scope="module", params=[False, True],
+ ids=["separate", "fused_gate_up"])
+def layer(request, tmp_path_factory):
+ fuse = request.param
+ path = tmp_path_factory.mktemp("shard") / "w.safetensors"
+ _write_shard(path, fuse)
+ mm = SafetensorsMmap(str(path))
+ lay = StreamingSwitchGLU(
+ [mm], 0, _spec(fuse), _options(),
+ shared_cache=SharedExpertCache(64))
+ return lay, mm, fuse
+
+
+def ref_moe(x, inds, shard: SafetensorsMmap, fuse_gu: bool):
+ """Bit-level reference: dequantize the SAME 4-bit payloads the
+ streaming paths consume, then compute the GLU in float32. This pins
+ the file layout + dequant math; kernel rounding aside it must match
+ every streaming path exactly."""
+ xf = x.astype(mx.float32)
+ wcache = {}
+
+ def w(e, proj):
+ key = (e, proj)
+ if key not in wcache:
+ name = f"layers.0.mlp.switch_mlp.{proj}.weight"
+ raw = shard.raw(name)
+ per = raw.size // N_EXPERTS
+ sl = raw[e * per:(e + 1) * per]
+ # [out, in/8] packed u32: gate/up out=INTER in=DIM;
+ # down out=DIM in=INTER.
+ if proj == "down_proj":
+ q = mx.array(sl.view("<u4")).reshape(DIM, INTER // 8)
+ else:
+ q = mx.array(sl.view("<u4")).reshape(INTER, DIM // 8)
+ parts = {"weight": q}
+ n_in = INTER if proj == "down_proj" else DIM
+ for part in ("scales", "biases"):
+ name = f"layers.0.mlp.switch_mlp.{proj}.{part}"
+ raw = shard.raw(name)
+ per = raw.size // N_EXPERTS
+ sl = raw[e * per:(e + 1) * per]
+ parts[part] = mx.array(sl.view("<u2")).view(
+ mx.bfloat16).reshape(-1, n_in // 64)
+ wcache[key] = mx.dequantize(
+ parts["weight"], parts["scales"], parts["biases"],
+ group_size=64, bits=4) # [out, in]
+ return wcache[key]
+
+ T, k = inds.shape
+ rows = []
+ for t in range(T):
+ per = []
+ for j in range(k):
+ e = int(inds[t, j])
+ gate = xf[t] @ w(e, "gate_proj").T
+ up = xf[t] @ w(e, "up_proj").T
+ z = nn.silu(gate) * up # swiglu
+ per.append(z @ w(e, "down_proj").T)
+ rows.append(mx.stack(per))
+ return mx.stack(rows)
+
+
+def _inds(tokens=1, k=4):
+ return mx.array(np.random.default_rng(0).integers(
+ 0, N_EXPERTS, size=(tokens, k)), dtype=mx.int32)
+
+
+def test_exact_matches_reference(layer):
+ lay, mm, fuse = layer
+ x = mx.random.normal((1, 2, DIM)).astype(mx.float16)
+ inds = _inds(tokens=2)
+ out = lay(x, inds)
+ assert out.shape == (1, 2, 4, DIM)
+ ref = ref_moe(x[0], inds, mm, fuse)
+ # gather_qmm evaluates with internal bf16 precision: a few elements
+ # carry ~4.0 absolute error where |ref| is small, while the overall
+ # error is only ≈0.24% relative L2. A per-element allclose would need
+ # a tolerance loose enough to mask real bugs, so assert on relative
+ # L2 instead: a gate/up swap or misaligned row layout (the bugs this
+ # pins) shows up as ≈100% error, ~400x above the kernel noise floor.
+ err = mx.sqrt(mx.sum((out[0].astype(mx.float32) - ref) ** 2)
+ / mx.sum(ref ** 2))
+ assert err < 1e-2, f"exact path diverges from dequant reference: {err}"
+
+
+def test_staged_matches_exact(layer):
+ lay, _, _ = layer
+ opts = _options(staged=True)
+ # fresh layer, staged mode
+ import tempfile, os
+ with tempfile.TemporaryDirectory() as td:
+ path = os.path.join(td, "w.safetensors")
+ fp16 = _write_shard(path, lay._fuse_gu)
+ mm = SafetensorsMmap(path)
+ s = StreamingSwitchGLU([mm], 0, _spec(lay._fuse_gu), opts,
+ shared_cache=SharedExpertCache(64))
+ x = mx.random.normal((1, 1, DIM)).astype(mx.float16)
+ inds = _inds(tokens=1)
+ # exact reference from the same layer
+ exact = lay(x, inds)
+ # stage the same expert set synchronously, then consume via slots
+ s.stage_experts(list(map(int, inds[0].tolist())))
+ s.wait_staged()
+ out = s(x, inds)
+ assert out.shape == exact.shape
+ assert mx.allclose(out, exact).item(), "staged != exact"
+
+
+def test_staged_replace_matches_exact(layer):
+ lay, _, _ = layer
+ import tempfile, os
+ with tempfile.TemporaryDirectory() as td:
+ path = os.path.join(td, "w.safetensors")
+ _write_shard(path, lay._fuse_gu)
+ mm = SafetensorsMmap(path)
+ s = StreamingSwitchGLU(
+ [mm], 0, _spec(lay._fuse_gu),
+ _options(staged=True, staged_replace=True),
+ shared_cache=SharedExpertCache(64))
+ x = mx.random.normal((1, 1, DIM)).astype(mx.float16)
+ staged = [0, 2, 5, 7]
+ s.stage_experts(staged)
+ s.wait_staged()
+ out = s(x, None) # route through the whole set
+ inds = mx.array([staged], dtype=mx.int32)
+ exact = lay(x, inds)
+ assert out.shape == exact.shape
+ assert mx.allclose(out, exact).item(), "staged_replace != exact"
+
+
+def test_staged_fallback_when_empty(layer):
+ lay, _, _ = layer
+ import tempfile, os
+ with tempfile.TemporaryDirectory() as td:
+ path = os.path.join(td, "w.safetensors")
+ _write_shard(path, lay._fuse_gu)
+ mm = SafetensorsMmap(path)
+ s = StreamingSwitchGLU([mm], 0, _spec(lay._fuse_gu),
+ _options(staged=True),
+ shared_cache=SharedExpertCache(64))
+ x = mx.random.normal((1, 1, DIM)).astype(mx.float16)
+ inds = _inds(tokens=1)
+ # nothing staged -> falls back to the exact path
+ out = s(x, inds)
+ exact = lay(x, inds)
+ assert mx.allclose(out, exact).item()
+ st = s.stats()
+ assert st["staged_fallback"] == 1
+
+
+def test_full_layer_prefill_matches_exact(layer):
+ lay, _, _ = layer
+ import tempfile, os
+ with tempfile.TemporaryDirectory() as td:
+ path = os.path.join(td, "w.safetensors")
+ _write_shard(path, lay._fuse_gu)
+ mm = SafetensorsMmap(path)
+ s = StreamingSwitchGLU([mm], 0, _spec(lay._fuse_gu),
+ _options(staged=True),
+ shared_cache=SharedExpertCache(64))
+ x = mx.random.normal((1, 4, DIM)).astype(mx.float16)
+ # 3D router indices like the model forward ([B, T, k]): the FULL
+ # branch keeps the [B, T, k, H] SwitchGLU contract without a batch
+ # restore (matching the deployment's whole-layer path), so the
+ # comparison must use the router's 3D shape, not the 2D _inds form.
+ inds = mx.expand_dims(_inds(tokens=4), 0) # size 16 > 8 -> prefill path
+ s.load_full_layer()
+ out = s(x, inds)
+ s.clear_full_layer()
+ exact = lay(x, inds)
+ assert out.shape == exact.shape
+ assert mx.allclose(out, exact).item(), "full-layer != exact"
+
+
+def test_hot_stack_matches_exact(layer):
+ lay, _, _ = layer
+ import tempfile, os
+ with tempfile.TemporaryDirectory() as td:
+ path = os.path.join(td, "w.safetensors")
+ _write_shard(path, lay._fuse_gu)
+ mm = SafetensorsMmap(path)
+ s = StreamingSwitchGLU([mm], 0, _spec(lay._fuse_gu),
+ _options(staged=True, hot_per_layer=0),
+ shared_cache=SharedExpertCache(64))
+ x = mx.random.normal((1, 4, DIM)).astype(mx.float16)
+ inds = _inds(tokens=4)
+ s.load_hot_layer(n_hot=N_EXPERTS) # full coverage -> no misses
+ s.materialize_hot()
+ out = s(x, inds)
+ exact = lay(x, inds)
+ assert out.shape == exact.shape
+ assert mx.allclose(out, exact).item(), "hot path != exact"
+
+
+def test_double_buffered_swap(layer):
+ """stage_experts fills next; swap promotes; consume uses the new set."""
+ lay, _, _ = layer
+ import tempfile, os
+ with tempfile.TemporaryDirectory() as td:
+ path = os.path.join(td, "w.safetensors")
+ _write_shard(path, lay._fuse_gu)
+ mm = SafetensorsMmap(path)
+ s = StreamingSwitchGLU(
+ [mm], 0, _spec(lay._fuse_gu),
+ _options(staged=True, staged_sync=False),
+ shared_cache=SharedExpertCache(64))
+ x = mx.random.normal((1, 1, DIM)).astype(mx.float16)
+ first = [1, 3, 4, 6]
+ second = [0, 2, 5, 7]
+ s.stage_experts(first)
+ s.swap_staged()
+ out1 = s(x, mx.array([first], dtype=mx.int32))
+ s.stage_experts(second)
+ s.swap_staged()
+ out2 = s(x, mx.array([second], dtype=mx.int32))
+ ref1 = lay(x, mx.array([first], dtype=mx.int32))
+ ref2 = lay(x, mx.array([second], dtype=mx.int32))
+ assert mx.allclose(out1, ref1).item()
+ assert mx.allclose(out2, ref2).item()